xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into hotfix/taos-tools 

* develop: (118 commits)
  fix td-320
  set handle when report broken link to server
  add debug info
  add rpcCancelRequest API
  [td-541] add string length overflow check.
  [td-225] fix error in scripts
  add stream cases to jenkins
  add numpy install to travis
  [TD-605]
  move more pytest cases to regression test.
  [TD-350]: sort test case for both columns and tags all - supporting types
  move more cases from full test to regression test.
  [td-225] update the tests scripts
  [TD-605]
  [td-225] fix bugs in group by normal columns
  [td-225] fix bugs in group by normal columns
  [td-225]
  [td-225] fix bugs in restore from core dump
  [td-225] do not execute query if the link is already broken.
  [td-225] fix bugs in group by normal columns
  ...
This commit is contained in:
Shuaiqiang Chang 2020-06-15 14:06:09 +08:00
parent 1ff6e7c7dd 352cb2863f
commit 21cdf62293
219 changed files with 8613 additions and 1617 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.travis.yml
View File

@ -45,11 +45,13 @@ matrix:
cd ${TRAVIS_BUILD_DIR}/debug
make install > /dev/null || travis_terminate $?
pip install numpy
pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
pip3 install numpy
pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
cd ${TRAVIS_BUILD_DIR}/tests
./test-all.sh $TRAVIS_EVENT_TYPE || travis_terminate $?
./test-all.sh smoke || travis_terminate $?
cd ${TRAVIS_BUILD_DIR}/tests/pytest
./valgrind-test.sh 2>&1 > mem-error-out.log
@ -164,12 +166,14 @@ matrix:
cd ${TRAVIS_BUILD_DIR}/debug
make install > /dev/null || travis_terminate $?
pip install numpy
pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
pip3 install numpy
pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
cd ${TRAVIS_BUILD_DIR}/tests
./test-all.sh $TRAVIS_EVENT_TYPE COVER
./test-all.sh smoke COVER
TEST_RESULT=$?
@ -178,7 +182,7 @@ matrix:
cd ${TRAVIS_BUILD_DIR}
lcov -d . --capture --rc lcov_branch_coverage=1 -o coverage.info
lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' -o coverage.info
lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' -o coverage.info
lcov -l --rc lcov_branch_coverage=1 coverage.info || travis_terminate $?
gem install coveralls-lcov

4
src/client/inc/tsclient.h
View File

@ -285,8 +285,8 @@ typedef struct STscObj {
void * pTimer;
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
char acctId[TSDB_DB_NAME_LEN];
char db[TSDB_TABLE_ID_LEN];
char acctId[TSDB_ACCT_LEN];
char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
char sversion[TSDB_VERSION_LEN];
char writeAuth : 1;
char superAuth : 1;

21
src/client/src/tscAsync.c
View File

@ -449,10 +449,13 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
tscTrace("%p update table meta in local cache, continue to process sql and send corresponding subquery", pSql);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
if (pTableMetaInfo->pTableMeta == NULL){
code = tscGetTableMeta(pSql, pTableMetaInfo);
code = tscGetTableMeta(pSql, pTableMetaInfo);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return;
} else {
assert(code == TSDB_CODE_SUCCESS);
}
assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pTableMetaInfo->vgroupIndex >= 0 && pSql->param != NULL);
@ -473,7 +476,11 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
code = tscGetTableMeta(pSql, pTableMetaInfo);
assert(code == TSDB_CODE_SUCCESS);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return;
} else {
assert(code == TSDB_CODE_SUCCESS);
}
// if failed to process sql, go to error handler
if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) {
@ -483,7 +490,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
// // 1. table uid, 2. ip address
// code = tscSendMsgToServer(pSql);
// if (code == TSDB_CODE_SUCCESS) return;
// }
} else {
tscTrace("%p continue parse sql after get table meta", pSql);
@ -491,8 +497,11 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STMT_INSERT)) {
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
code = tscGetTableMeta(pSql, pTableMetaInfo);
assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->pTableMeta != NULL);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return;
} else {
assert(code == TSDB_CODE_SUCCESS);
}
(*pSql->fp)(pSql->param, pSql, code);
return;
}

14
src/client/src/tscFunctionImpl.c
View File

@ -151,7 +151,6 @@ typedef struct SRateInfo {
double sum; // for sum/avg
} SRateInfo;
int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable) {
if (!isValidDataType(dataType, dataBytes)) {
@ -700,7 +699,7 @@ static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end,
}
static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
if (pCtx->order == TSDB_ORDER_ASC) {
if (pCtx->order != pCtx->param[0].i64Key) {
return BLK_DATA_NO_NEEDED;
}
@ -728,7 +727,7 @@ static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY
}
static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
if (pCtx->order == TSDB_ORDER_ASC) {
if (pCtx->order != pCtx->param[0].i64Key) {
return BLK_DATA_NO_NEEDED;
}
@ -1594,7 +1593,7 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) {
if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
return;
}
if (pCtx->order == TSDB_ORDER_DESC) {
return;
}
@ -1653,7 +1652,7 @@ static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) {
* least one data in this block that is not null.(TODO opt for this case)
*/
static void last_function(SQLFunctionCtx *pCtx) {
if (pCtx->order == TSDB_ORDER_ASC) {
if (pCtx->order != pCtx->param[0].i64Key) {
return;
}
@ -1682,7 +1681,6 @@ static void last_function(SQLFunctionCtx *pCtx) {
}
static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) {
assert(pCtx->order != TSDB_ORDER_ASC);
void *pData = GET_INPUT_CHAR_INDEX(pCtx, index);
if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
return;
@ -1726,7 +1724,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) {
* 1. for scan data in asc order, no need to check data
* 2. for data blocks that are not loaded, no need to check data
*/
if (pCtx->order == TSDB_ORDER_ASC) {
if (pCtx->order != pCtx->param[0].i64Key) {
return;
}
@ -1764,7 +1762,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) {
* 1. for scan data in asc order, no need to check data
* 2. for data blocks that are not loaded, no need to check data
*/
if (pCtx->order == TSDB_ORDER_ASC) {
if (pCtx->order != pCtx->param[0].i64Key) {
return;
}

23
src/client/src/tscLocal.c
View File

@ -132,7 +132,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
for (int32_t i = 0; i < numOfRows; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, TSDB_COL_NAME_LEN);
STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, TSDB_COL_NAME_LEN - 1);
char *type = tDataTypeDesc[pSchema[i].type].aName;
@ -171,7 +171,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
// field name
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, TSDB_COL_NAME_LEN);
STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, TSDB_COL_NAME_LEN - 1);
// type name
pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1);
@ -211,18 +211,18 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
pQueryInfo->order.order = TSDB_ORDER_ASC;
TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE};
strncpy(f.name, "Field", TSDB_COL_NAME_LEN);
TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE};
tstrncpy(f.name, "Field", sizeof(f.name));
SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE, TSDB_COL_NAME_LEN, false);
(TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, (TSDB_COL_NAME_LEN - 1), false);
rowLen += (TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE);
rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE);
f.bytes = typeColLength;
f.type = TSDB_DATA_TYPE_BINARY;
strncpy(f.name, "Type", TSDB_COL_NAME_LEN);
tstrncpy(f.name, "Type", sizeof(f.name));
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength,
@ -232,7 +232,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
f.bytes = sizeof(int32_t);
f.type = TSDB_DATA_TYPE_INT;
strncpy(f.name, "Length", TSDB_COL_NAME_LEN);
tstrncpy(f.name, "Length", sizeof(f.name));
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t),
@ -242,7 +242,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
f.bytes = noteColLength;
f.type = TSDB_DATA_TYPE_BINARY;
strncpy(f.name, "Note", TSDB_COL_NAME_LEN);
tstrncpy(f.name, "Note", sizeof(f.name));
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength,
@ -279,14 +279,15 @@ static void tscProcessCurrentUser(SSqlObj *pSql) {
pExpr->resType = TSDB_DATA_TYPE_BINARY;
char* vx = calloc(1, pExpr->resBytes);
STR_WITH_MAXSIZE_TO_VARSTR(vx, pSql->pTscObj->user, TSDB_USER_LEN);
size_t size = sizeof(pSql->pTscObj->user);
STR_WITH_MAXSIZE_TO_VARSTR(vx, pSql->pTscObj->user, size);
tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes);
free(vx);
}
static void tscProcessCurrentDB(SSqlObj *pSql) {
char db[TSDB_DB_NAME_LEN + 1] = {0};
char db[TSDB_DB_NAME_LEN] = {0};
extractDBName(pSql->pTscObj->db, db);
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);

10
src/client/src/tscParseInsert.c
View File

@ -97,7 +97,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1
useconds = str2int64(pToken->z);
} else {
// strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm);
if (taosParseTime(pToken->z, time, pToken->n, timePrec) != TSDB_CODE_SUCCESS) {
if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
return tscInvalidSQLErrMsg(error, "invalid timestamp format", pToken->z);
}
@ -795,7 +795,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX);
tscSetTableFullName(pSTableMeterMetaInfo, &sToken, pSql);
strncpy(pTag->name, pSTableMeterMetaInfo->name, TSDB_TABLE_ID_LEN);
tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name));
code = tscGetTableMeta(pSql, pSTableMeterMetaInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
@ -989,7 +989,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
int validateTableName(char *tblName, int len) {
char buf[TSDB_TABLE_ID_LEN] = {0};
strncpy(buf, tblName, len);
tstrncpy(buf, tblName, sizeof(buf));
SSQLToken token = {.n = len, .type = TK_ID, .z = buf};
tSQLGetToken(buf, &token.type);
@ -1512,7 +1512,7 @@ void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) {
}
pCmd->count = 1;
strncpy(path, pDataBlock->filename, PATH_MAX);
tstrncpy(path, pDataBlock->filename, sizeof(path));
FILE *fp = fopen(path, "r");
if (fp == NULL) {
@ -1520,7 +1520,7 @@ void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) {
continue;
}
strncpy(pTableMetaInfo->name, pDataBlock->tableId, TSDB_TABLE_ID_LEN);
tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
memset(pDataBlock->pData, 0, pDataBlock->nAllocSize);
int32_t ret = tscGetTableMeta(pSql, pTableMetaInfo);

6
src/client/src/tscProfile.c
View File

@ -239,8 +239,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
continue;
}
strncpy(pQdesc->sql, pSql->sqlstr, TSDB_SHOW_SQL_LEN - 1);
pQdesc->sql[TSDB_SHOW_SQL_LEN - 1] = 0;
tstrncpy(pQdesc->sql, pSql->sqlstr, sizeof(pQdesc->sql));
pQdesc->stime = htobe64(pSql->stime);
pQdesc->queryId = htonl(pSql->queryId);
pQdesc->useconds = htobe64(pSql->res.useconds);
@ -256,8 +255,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
SSqlStream *pStream = pObj->streamList;
while (pStream) {
strncpy(pSdesc->sql, pStream->pSql->sqlstr, TSDB_SHOW_SQL_LEN - 1);
pSdesc->sql[TSDB_SHOW_SQL_LEN - 1] = 0;
tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql));
pSdesc->streamId = htonl(pStream->streamId);
pSdesc->num = htobe64(pStream->num);

148
src/client/src/tscSQLParser.c
View File

@ -138,7 +138,7 @@ static int setColumnFilterInfoForTimestamp(SQueryInfo* pQueryInfo, tVariant* pVa
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
if (seg != NULL) {
if (taosParseTime(pVar->pz, &time, pVar->nLen, tinfo.precision) != TSDB_CODE_SUCCESS) {
if (taosParseTime(pVar->pz, &time, pVar->nLen, tinfo.precision, tsDaylight) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(pQueryInfo->msg, msg);
}
} else {
@ -169,7 +169,7 @@ static int32_t handlePassword(SSqlCmd* pCmd, SSQLToken* pPwd) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
if (pPwd->n > TSDB_PASSWORD_LEN) {
if (pPwd->n >= TSDB_PASSWORD_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
@ -232,7 +232,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
pzName->n = strdequote(pzName->z);
strncpy(pTableMetaInfo->name, pzName->z, pzName->n);
} else { // drop user
if (pzName->n > TSDB_USER_LEN) {
if (pzName->n >= TSDB_USER_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
@ -317,7 +317,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return TSDB_CODE_TSC_INVALID_SQL;
}
if (pName->n > TSDB_USER_LEN) {
if (pName->n >= TSDB_USER_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
@ -348,7 +348,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
if (pToken->n > TSDB_TABLE_NAME_LEN) {
if (!tscValidateTableNameLength(pToken->n)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
@ -401,7 +401,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
SSQLToken* pName = &pUser->user;
SSQLToken* pPwd = &pUser->passwd;
if (pName->n > TSDB_USER_LEN) {
if (pName->n >= TSDB_USER_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
@ -543,7 +543,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression");
}
pSql->cmd.parseFinished = true;
pSql->cmd.parseFinished = 1;
return tscBuildMsg[pCmd->command](pSql, pInfo);
}
@ -933,7 +933,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
SSchema* pSchema = tscGetTableSchema(pTableMeta);
for (int32_t i = 0; i < numOfTags + numOfCols; ++i) {
if (strncasecmp(pTagField->name, pSchema[i].name, TSDB_COL_NAME_LEN) == 0) {
if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
return false;
}
@ -993,7 +993,7 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
// field name must be unique
for (int32_t i = 0; i < numOfTags + numOfCols; ++i) {
if (strncasecmp(pColField->name, pSchema[i].name, TSDB_COL_NAME_LEN) == 0) {
if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
return false;
}
@ -1005,7 +1005,8 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
/* is contained in pFieldList or not */
static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) {
for (int32_t j = startIdx; j < pFieldList->nField; ++j) {
if (strncasecmp(name, pFieldList->p[j].name, TSDB_COL_NAME_LEN) == 0) return true;
TAOS_FIELD* field = pFieldList->p + j;
if (strncasecmp(name, field->name, sizeof(field->name) - 1) == 0) return true;
}
return false;
@ -1041,7 +1042,7 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL
/* db name is not specified, the tableName dose not include db name */
if (pDB != NULL) {
if (pDB->n > TSDB_DB_NAME_LEN) {
if (pDB->n >= TSDB_ACCT_LEN + TSDB_DB_NAME_LEN) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -1055,12 +1056,12 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL
totalLen += 1;
/* here we only check the table name length limitation */
if (tableName->n > TSDB_TABLE_NAME_LEN) {
if (!tscValidateTableNameLength(tableName->n)) {
return TSDB_CODE_TSC_INVALID_SQL;
}
} else { // pDB == NULL, the db prefix name is specified in tableName
/* the length limitation includes tablename + dbname + sep */
if (tableName->n > TSDB_TABLE_NAME_LEN + TSDB_DB_NAME_LEN + tListLen(TS_PATH_DELIMITER)) {
if (tableName->n >= TSDB_TABLE_NAME_LEN + TSDB_DB_NAME_LEN) {
return TSDB_CODE_TSC_INVALID_SQL;
}
}
@ -1077,7 +1078,7 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL
fullName[totalLen] = 0;
}
return (totalLen <= TSDB_TABLE_ID_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL;
return (totalLen < TSDB_TABLE_ID_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL;
}
static void extractColumnNameFromString(tSQLExprItem* pItem) {
@ -1166,7 +1167,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
/* todo alias name should use the original sql string */
char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr;
strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN);
tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName));
tExprNode* pNode = NULL;
SArray* colList = taosArrayInit(10, sizeof(SColIndex));
@ -1308,7 +1309,7 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName;
strncpy(pExpr->aliasName, colName, tListLen(pExpr->aliasName));
tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName));
SColumnList ids = {0};
ids.num = 1;
@ -1358,7 +1359,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum
for (int32_t j = 0; j < numOfTotalColumns; ++j) {
SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos + j, j, pIndex->tableIndex);
strncpy(pExpr->aliasName, pSchema[j].name, tListLen(pExpr->aliasName));
tstrncpy(pExpr->aliasName, pSchema[j].name, sizeof(pExpr->aliasName));
pIndex->columnIndex = j;
SColumnList ids = {0};
@ -1401,9 +1402,7 @@ int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pI
}
if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE};
strcpy(colSchema.name, TSQL_TBNAME_L);
SSchema colSchema = tGetTableNameColumnSchema();
tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, true);
} else {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
@ -1448,11 +1447,18 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema,
if (aliasName != NULL) {
strcpy(columnName, aliasName);
} else {
getRevisedName(columnName, functionID, TSDB_COL_NAME_LEN, pSchema[pColIndex->columnIndex].name);
getRevisedName(columnName, functionID, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name);
}
SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false);
strncpy(pExpr->aliasName, columnName, tListLen(pExpr->aliasName));
tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName));
// set reverse order scan data blocks for last query
if (functionID == TSDB_FUNC_LAST) {
pExpr->numOfParams = 1;
pExpr->param[0].i64Key = TSDB_ORDER_DESC;
pExpr->param[0].nType = TSDB_DATA_TYPE_INT;
}
// for all queries, the timestamp column needs to be loaded
SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
@ -1536,7 +1542,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
}
memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
getColumnName(pItem, pExpr->aliasName, TSDB_COL_NAME_LEN);
getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1);
SColumnList ids = getColumnList(1, index.tableIndex, index.columnIndex);
if (finalResult) {
@ -1651,7 +1657,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
ids.ids[0] = index;
memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
getColumnName(pItem, pExpr->aliasName, TSDB_COL_NAME_LEN);
getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1);
if (finalResult) {
int32_t numOfOutput = tscNumOfFields(pQueryInfo);
@ -1725,6 +1731,22 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i, &index) != 0) {
return TSDB_CODE_TSC_INVALID_SQL;
}
if (optr == TK_LAST) { // todo refactor
SSqlGroupbyExpr* pGroupBy = &pQueryInfo->groupbyExpr;
if (pGroupBy->numOfGroupCols > 0) {
for(int32_t k = 0; k < pGroupBy->numOfGroupCols; ++k) {
SColIndex* pIndex = taosArrayGet(pGroupBy->columnInfo, k);
if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { // group by normal columns
SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, colIndex + i);
pExpr->numOfParams = 1;
pExpr->param->i64Key = TSDB_ORDER_ASC;
break;
}
}
}
}
}
}
@ -1851,7 +1873,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
}
memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
getColumnName(pItem, pExpr->aliasName, TSDB_COL_NAME_LEN);
getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1);
SColumnList ids = getColumnList(1, 0, index.columnIndex);
if (finalResult) {
@ -1913,9 +1935,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
SSchema s = {0};
if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
s.bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
s.type = TSDB_DATA_TYPE_BINARY;
s.colId = TSDB_TBNAME_COLUMN_INDEX;
s = tGetTableNameColumnSchema();
} else {
s = pTagSchema[index.columnIndex];
}
@ -1993,6 +2013,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken
if (strncasecmp(pSchema[i].name, pToken->z, pToken->n) == 0) {
columnIndex = i;
break;
}
}
@ -2051,7 +2072,7 @@ int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColum
}
pIndex->tableIndex = COLUMN_INDEX_INITIAL_VAL;
char tableName[TSDB_TABLE_ID_LEN + 1] = {0};
char tableName[TSDB_TABLE_ID_LEN] = {0};
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
@ -2202,7 +2223,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
if (pDbPrefixToken->type != 0) {
assert(pDbPrefixToken->n >= 0);
if (pDbPrefixToken->n > TSDB_DB_NAME_LEN) { // db name is too long
if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
@ -2229,7 +2250,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
if (pCmd->payloadLen > TSDB_TABLE_NAME_LEN) {
if (!tscValidateTableNameLength(pCmd->payloadLen)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
}
@ -2240,7 +2261,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
// show vnodes may be ip addr of dnode in payload
SSQLToken* pDnodeIp = &pShowInfo->prefix;
if (pDnodeIp->n > TSDB_IPv4ADDR_LEN) { // ip addr is too long
if (pDnodeIp->n >= TSDB_IPv4ADDR_LEN) { // ip addr is too long
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
@ -2589,9 +2610,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
tscColumnListInsert(pQueryInfo->colList, &index);
SColIndex colIndex = {
.colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId,
};
SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId };
taosArrayPush(pGroupExpr->columnInfo, &colIndex);
pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC;
@ -2860,7 +2879,7 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, SStringBuilder* sb) {
taosStringBuilderAppendString(sb, TBNAME_LIST_SEP);
}
if (pSub->val.nLen <= 0 || pSub->val.nLen > TSDB_TABLE_NAME_LEN) {
if (pSub->val.nLen <= 0 || !tscValidateTableNameLength(pSub->val.nLen)) {
return TSDB_CODE_TSC_INVALID_SQL;
}
}
@ -2889,7 +2908,8 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
const char* msg1 = "non binary column not support like operator";
const char* msg2 = "binary column not support this operator";
const char* msg2 = "binary column not support this operator";
const char* msg3 = "bool column not support this operator";
SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex);
SColumnFilterInfo* pColFilter = NULL;
@ -2923,6 +2943,12 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn
if (pExpr->nSQLOptr == TK_LIKE) {
return invalidSqlErrMsg(pQueryInfo->msg, msg1);
}
if (pSchema->type == TSDB_DATA_TYPE_BOOL) {
if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE) {
return invalidSqlErrMsg(pQueryInfo->msg, msg3);
}
}
}
pColumn->colIndex = *pIndex;
@ -3615,7 +3641,7 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac
taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1);
}
char idBuf[TSDB_TABLE_ID_LEN + 1] = {0};
char idBuf[TSDB_TABLE_ID_LEN] = {0};
int32_t xlen = strlen(segments[i]);
SSQLToken t = {.z = segments[i], .n = xlen, .type = TK_STRING};
@ -3924,7 +3950,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t
char* seg = strnchr(pRight->val.pz, '-', pRight->val.nLen, false);
if (seg != NULL) {
if (taosParseTime(pRight->val.pz, &val, pRight->val.nLen, TSDB_TIME_PRECISION_MICRO) == TSDB_CODE_SUCCESS) {
if (taosParseTime(pRight->val.pz, &val, pRight->val.nLen, TSDB_TIME_PRECISION_MICRO, tsDaylight) == TSDB_CODE_SUCCESS) {
parsed = true;
} else {
return TSDB_CODE_TSC_INVALID_SQL;
@ -3997,7 +4023,7 @@ int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name;
for (int32_t j = 0; j < TSDB_COL_NAME_LEN && fieldName[j] != 0; ++j) {
for (int32_t j = 0; j < (TSDB_COL_NAME_LEN - 1) && fieldName[j] != 0; ++j) {
for (int32_t k = 0; k < tListLen(rep); ++k) {
if (fieldName[j] == rep[k]) {
fieldName[j] = '_';
@ -4013,7 +4039,7 @@ int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name;
for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutput; ++j) {
if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, TSDB_COL_NAME_LEN) == 0) {
if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, (TSDB_COL_NAME_LEN - 1)) == 0) {
const char* msg = "duplicated column name in new table";
return invalidSqlErrMsg(pQueryInfo->msg, msg);
}
@ -4368,7 +4394,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
tVariantListItem* pItem = &pAlterSQL->varList->a[0];
if (pItem->pVar.nLen > TSDB_COL_NAME_LEN) {
if (pItem->pVar.nLen >= TSDB_COL_NAME_LEN) {
return invalidSqlErrMsg(pQueryInfo->msg, msg9);
}
@ -4379,9 +4405,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return TSDB_CODE_TSC_INVALID_SQL;
}
if (index.columnIndex < tscGetNumOfColumns(pTableMeta)) {
int32_t numOfCols = tscGetNumOfColumns(pTableMeta);
if (index.columnIndex < numOfCols) {
return invalidSqlErrMsg(pQueryInfo->msg, msg10);
} else if (index.columnIndex == 0) {
} else if (index.columnIndex == numOfCols) {
return invalidSqlErrMsg(pQueryInfo->msg, msg11);
}
@ -4420,7 +4447,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return TSDB_CODE_TSC_INVALID_SQL;
}
char name[TSDB_COL_NAME_LEN + 1] = {0};
char name[TSDB_COL_NAME_LEN] = {0};
strncpy(name, pVarList->a[0].pVar.pz, pVarList->a[0].pVar.nLen);
TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
@ -4522,8 +4549,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(pQueryInfo->msg, msg18);
}
char name1[TSDB_COL_NAME_LEN + 1] = {0};
strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen);
char name1[TSDB_COL_NAME_LEN] = {0};
tstrncpy(name1, pItem->pVar.pz, sizeof(name1));
TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
}
@ -5231,9 +5258,10 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) {
int16_t colIndex = pColIndex->colIndex;
if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
type = TSDB_DATA_TYPE_BINARY;
bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; // todo extract method
name = TSQL_TBNAME_L;
SSchema s = tGetTableNameColumnSchema();
type = s.type;
bytes = s.bytes;
name = s.name;
} else {
if (TSDB_COL_IS_TAG(pColIndex->flag)) {
SSchema* tagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
@ -5254,8 +5282,8 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) {
SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex};
SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true);
memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN);
memset(pExpr->aliasName, 0, sizeof(pExpr->aliasName));
tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName));
pExpr->colInfo.flag = TSDB_COL_TAG;
@ -5419,7 +5447,7 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false);
const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name;
strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
tstrncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
return TSDB_CODE_SUCCESS;
}
@ -5601,7 +5629,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
}
// get meter meta from mnode
strncpy(pCreateTable->usingInfo.tagdata.name, pStableMeterMetaInfo->name, TSDB_TABLE_ID_LEN);
tstrncpy(pCreateTable->usingInfo.tagdata.name, pStableMeterMetaInfo->name, sizeof(pCreateTable->usingInfo.tagdata.name));
tVariantList* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals;
int32_t code = tscGetTableMeta(pSql, pStableMeterMetaInfo);
@ -6019,7 +6047,7 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray*
if (pCols != NULL) { // record the involved columns
SColIndex colIndex = {0};
strncpy(colIndex.name, pSchema->name, TSDB_COL_NAME_LEN);
tstrncpy(colIndex.name, pSchema->name, sizeof(colIndex.name));
colIndex.colId = pSchema->colId;
colIndex.colIndex = index.columnIndex;
@ -6053,6 +6081,16 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray*
}
}
}
if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) {
if (pRight->nodeType == TSQL_NODE_VALUE) {
if ( pRight->pVal->nType == TSDB_DATA_TYPE_BOOL
|| pRight->pVal->nType == TSDB_DATA_TYPE_BINARY
|| pRight->pVal->nType == TSDB_DATA_TYPE_NCHAR) {
return TSDB_CODE_TSC_INVALID_SQL;
}
}
}
}
return TSDB_CODE_SUCCESS;

10
src/client/src/tscSchemaUtil.c
View File

@ -50,14 +50,6 @@ int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) {
SSchema *tscGetTableSchema(const STableMeta *pTableMeta) {
assert(pTableMeta != NULL);
// if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
// STableMeta* pSTableMeta = pTableMeta->pSTable;
// assert (pSTableMeta != NULL);
//
// return pSTableMeta->schema;
// }
return (SSchema*) pTableMeta->schema;
}
@ -115,7 +107,7 @@ bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols) {
// 3. valid column names
for (int32_t j = i + 1; j < numOfCols; ++j) {
if (strncasecmp(pSchema[i].name, pSchema[j].name, TSDB_COL_NAME_LEN) == 0) {
if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) {
return false;
}
}

51
src/client/src/tscSecondaryMerge.c
View File

@ -217,7 +217,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
pReducer->numOfBuffer = numOfFlush;
pReducer->numOfVnode = numOfBuffer;
pReducer->pDesc = pDesc;
tscTrace("%p the number of merged leaves is: %d", pSql, pReducer->numOfBuffer);
@ -324,7 +324,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
tfree(pReducer->discardData);
tfree(pReducer->pResultBuf);
tfree(pReducer->pFinalRes);
// tfree(pReducer->pBufForInterpo);
tfree(pReducer->prevRowOfInput);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -363,7 +362,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
SFillColInfo* pFillCol = createFillColInfo(pQueryInfo);
pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
4096, numOfCols, pQueryInfo->slidingTime, pQueryInfo->fillType, pFillCol);
4096, numOfCols, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit,
tinfo.precision, pQueryInfo->fillType, pFillCol);
}
int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
@ -494,7 +494,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
tscTrace("%p waiting for delete procedure, status: %d", pSql, status);
}
taosDestoryFillInfo(pLocalReducer->pFillInfo);
pLocalReducer->pFillInfo = taosDestoryFillInfo(pLocalReducer->pFillInfo);
if (pLocalReducer->pCtx != NULL) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
@ -604,7 +604,7 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage
tOrderDescriptor *pOrderDesc = pReducer->pDesc;
SColumnOrderInfo* orderInfo = &pOrderDesc->orderInfo;
// no group by columns, all data belongs to one group
int32_t numOfCols = orderInfo->numOfCols;
if (numOfCols <= 0) {
@ -627,7 +627,7 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage
// only one row exists
int32_t index = orderInfo->pData[0];
int32_t offset = (pOrderDesc->pColumnModel)->pFields[index].offset;
int32_t ret = memcmp(pPrev + offset, tmpBuffer->data + offset, pOrderDesc->pColumnModel->rowSize - offset);
return ret == 0;
}
@ -980,8 +980,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
}
/* all output for current group are completed */
int32_t totalRemainRows =
taosGetNumOfResultWithFill(pFillInfo, rpoints, pFillInfo->slidingTime, actualETime);
int32_t totalRemainRows = getFilledNumOfRes(pFillInfo, actualETime, pLocalReducer->resColModel->capacity);
if (totalRemainRows <= 0) {
break;
}
@ -1041,7 +1040,7 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer)
static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, bool needInit) {
// the tag columns need to be set before all functions execution
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
size_t size = tscSqlExprNumOfExprs(pQueryInfo);
for (int32_t j = 0; j < size; ++j) {
SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[j];
@ -1183,7 +1182,7 @@ int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {
*/
bool needToMerge(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) {
int32_t ret = 0; // merge all result by default
int16_t functionId = pLocalReducer->pCtx[0].functionId;
// todo opt performance
@ -1267,13 +1266,7 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no
SFillInfo* pFillInfo = pLocalReducer->pFillInfo;
if (pFillInfo != NULL) {
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
TSKEY ekey = taosGetRevisedEndKey(pQueryInfo->window.ekey, pFillInfo->order, pFillInfo->slidingTime,
pQueryInfo->slidingTimeUnit, tinfo.precision);
taosFillSetStartInfo(pFillInfo, pResBuf->num, ekey);
taosFillSetStartInfo(pFillInfo, pResBuf->num, pQueryInfo->window.ekey);
taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf);
}
@ -1327,23 +1320,15 @@ static bool doBuildFilledResultForGroup(SSqlObj *pSql) {
SLocalReducer *pLocalReducer = pRes->pLocalReducer;
SFillInfo *pFillInfo = pLocalReducer->pFillInfo;
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
int8_t p = tinfo.precision;
if (pFillInfo != NULL && taosNumOfRemainRows(pFillInfo) > 0) {
assert(pQueryInfo->fillType != TSDB_FILL_NONE);
tFilePage *pFinalDataBuf = pLocalReducer->pResultBuf;
int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1));
int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1));
int32_t remain = taosNumOfRemainRows(pFillInfo);
TSKEY ekey = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit, p);
// the first column must be the timestamp column
int32_t rows = taosGetNumOfResultWithFill(pFillInfo, remain, ekey, pLocalReducer->resColModel->capacity);
if (rows > 0) { // do interpo
int32_t rows = getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
if (rows > 0) { // do fill gap
doFillResult(pSql, pLocalReducer, false);
}
@ -1362,10 +1347,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow;
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL ||
prevGroupCompleted) {
@ -1373,9 +1355,8 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey;
etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime,
pQueryInfo->slidingTimeUnit, tinfo.precision);
int32_t rows = taosGetNumOfResultWithFill(pFillInfo, 0, etime, pLocalReducer->resColModel->capacity);
assert(pFillInfo->numOfRows == 0);
int32_t rows = getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
if (rows > 0) { // do interpo
doFillResult(pSql, pLocalReducer, true);
}

54
src/client/src/tscServer.c
View File

@ -209,6 +209,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
tscError("%p sql is already released", pSql->signature);
return;
}
if (pSql->signature != pSql) {
tscError("%p sql is already released, signature:%p", pSql, pSql->signature);
return;
@ -217,10 +218,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
STscObj *pObj = pSql->pTscObj;
// tscTrace("%p msg:%s is received from server", pSql, taosMsg[rpcMsg->msgType]);
if (pObj->signature != pObj) {
tscTrace("%p sql is already released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed,
if (pObj->signature != pObj || pSql->freed == 1) {
tscTrace("%p sqlObj needs to be released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed,
pObj, pObj->signature);
tscFreeSqlObj(pSql);
rpcFreeCont(rpcMsg->pCont);
@ -375,7 +375,7 @@ int tscProcessSql(SSqlObj *pSql) {
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = NULL;
uint16_t type = 0;
uint32_t type = 0;
if (pQueryInfo != NULL) {
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@ -424,13 +424,13 @@ void tscKillSTableQuery(SSqlObj *pSql) {
* sub-queries not correctly released and master sql object of super table query reaches an abnormal state.
*/
pSql->pSubs[i]->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
//taosStopRpcConn(pSql->pSubs[i]->thandle);
// taosStopRpcConn(pSql->pSubs[i]->);
}
/*
* 1. if the subqueries are not launched or partially launched, we need to waiting the launched
* query return to successfully free allocated resources.
* 2. if no any subqueries are launched yet, which means the metric query only in parse sql stage,
* 2. if no any subqueries are launched yet, which means the super table query only in parse sql stage,
* set the res.code, and return.
*/
const int64_t MAX_WAITING_TIME = 10000; // 10 Sec.
@ -866,7 +866,7 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
assert(pCmd->numOfClause == 1);
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strncpy(pCreateDbMsg->db, pTableMetaInfo->name, tListLen(pCreateDbMsg->db));
tstrncpy(pCreateDbMsg->db, pTableMetaInfo->name, sizeof(pCreateDbMsg->db));
return TSDB_CODE_SUCCESS;
}
@ -1018,7 +1018,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strcpy(pDrop->ep, pTableMetaInfo->name);
tstrncpy(pDrop->ep, pTableMetaInfo->name, sizeof(pDrop->ep));
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE;
return TSDB_CODE_SUCCESS;
@ -1036,7 +1036,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strcpy(pDropMsg->user, pTableMetaInfo->name);
tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));
return TSDB_CODE_SUCCESS;
}
@ -1091,9 +1091,9 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
size_t nameLen = strlen(pTableMetaInfo->name);
if (nameLen > 0) {
strcpy(pShowMsg->db, pTableMetaInfo->name); // prefix is set here
tstrncpy(pShowMsg->db, pTableMetaInfo->name, sizeof(pShowMsg->db)); // prefix is set here
} else {
strcpy(pShowMsg->db, pObj->db);
tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db));
}
SShowInfo *pShowInfo = &pInfo->pDCLInfo->showOpt;
@ -1289,7 +1289,10 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) (pCmd->payload + tsRpcHeadSize);
pCmd->payloadLen = htonl(pUpdateMsg->head.contLen);
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
tscSetDnodeIpList(pSql, &pTableMetaInfo->pTableMeta->vgroupInfo);
return TSDB_CODE_SUCCESS;
}
@ -1300,7 +1303,7 @@ int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strcpy(pAlterDbMsg->db, pTableMetaInfo->name);
tstrncpy(pAlterDbMsg->db, pTableMetaInfo->name, sizeof(pAlterDbMsg->db));
return TSDB_CODE_SUCCESS;
}
@ -1430,9 +1433,9 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
char *db; // ugly code to move the space
db = strstr(pObj->db, TS_PATH_DELIMITER);
db = (db == NULL) ? pObj->db : db + 1;
strcpy(pConnect->db, db);
strcpy(pConnect->clientVersion, version);
strcpy(pConnect->msgVersion, "");
tstrncpy(pConnect->db, db, sizeof(pConnect->db));
tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion));
tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion));
return TSDB_CODE_SUCCESS;
}
@ -1705,8 +1708,9 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i);
strncpy(pMsg, pTableMetaInfo->name, TSDB_TABLE_ID_LEN);
pMsg += TSDB_TABLE_ID_LEN;
size_t size = sizeof(pTableMetaInfo->name);
tstrncpy(pMsg, pTableMetaInfo->name, size);
pMsg += size;
}
pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP;
@ -2150,11 +2154,11 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
SSqlRes *pRes = &pSql->res;
SCMConnectRsp *pConnect = (SCMConnectRsp *)pRes->pRsp;
strcpy(pObj->acctId, pConnect->acctId); // copy acctId from response
tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response
int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db);
assert(len <= tListLen(pObj->db));
strncpy(pObj->db, temp, tListLen(pObj->db));
assert(len <= sizeof(pObj->db));
tstrncpy(pObj->db, temp, sizeof(pObj->db));
if (pConnect->ipList.numOfIps > 0)
tscSetMgmtIpList(&pConnect->ipList);
@ -2172,7 +2176,7 @@ int tscProcessUseDbRsp(SSqlObj *pSql) {
STscObj * pObj = pSql->pTscObj;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
strcpy(pObj->db, pTableMetaInfo->name);
tstrncpy(pObj->db, pTableMetaInfo->name, sizeof(pObj->db));
return 0;
}
@ -2197,7 +2201,7 @@ int tscProcessDropTableRsp(SSqlObj *pSql) {
* The cached information is expired, however, we may have lost the ref of original meter. So, clear whole cache
* instead.
*/
tscTrace("%p force release metermeta after drop table:%s", pSql, pTableMetaInfo->name);
tscTrace("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name);
taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true);
if (pTableMetaInfo->pTableMeta) {
@ -2220,9 +2224,7 @@ int tscProcessAlterTableMsgRsp(SSqlObj *pSql) {
if (pTableMetaInfo->pTableMeta) {
bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true);
if (isSuperTable) { // if it is a super table, reset whole query cache
tscTrace("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name);
@ -2338,7 +2340,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo);
assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
strncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, tListLen(pNewMeterMetaInfo->name));
tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name));
memcpy(pNew->cmd.payload, pSql->cmd.payload, pSql->cmd.payloadLen); // tag information if table does not exists.
pNew->cmd.payloadLen = pSql->cmd.payloadLen;
tscTrace("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated);

89
src/client/src/tscSql.c
View File

@ -45,11 +45,11 @@ static bool validImpl(const char* str, size_t maxsize) {
}
static bool validUserName(const char* user) {
return validImpl(user, TSDB_USER_LEN);
return validImpl(user, TSDB_USER_LEN - 1);
}
static bool validPassword(const char* passwd) {
return validImpl(passwd, TSDB_PASSWORD_LEN);
return validImpl(passwd, TSDB_PASSWORD_LEN - 1);
}
SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *db, uint16_t port,
@ -86,21 +86,21 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
pObj->signature = pObj;
strncpy(pObj->user, user, TSDB_USER_LEN);
tstrncpy(pObj->user, user, sizeof(pObj->user));
taosEncryptPass((uint8_t *)pass, strlen(pass), pObj->pass);
if (db) {
int32_t len = strlen(db);
/* db name is too long */
if (len > TSDB_DB_NAME_LEN) {
if (len >= TSDB_DB_NAME_LEN) {
terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH;
rpcClose(pDnodeConn);
free(pObj);
return NULL;
}
char tmp[TSDB_DB_NAME_LEN + 1] = {0};
strcpy(tmp, db);
char tmp[TSDB_DB_NAME_LEN] = {0};
tstrncpy(tmp, db, sizeof(tmp));
strdequote(tmp);
strtolower(pObj->db, tmp);
@ -219,6 +219,11 @@ void waitForQueryRsp(void *param, TAOS_RES *tres, int code) {
sem_post(&pSql->rspSem);
}
static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) {
SSqlObj* pSql = (SSqlObj*) tres;
sem_post(&pSql->rspSem);
}
TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) {
STscObj *pObj = (STscObj *)taos;
if (pObj == NULL || pObj->signature != pObj) {
@ -369,11 +374,6 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
return (pQueryInfo->order.order == TSDB_ORDER_DESC) ? pRes->numOfRows : -pRes->numOfRows;
}
static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) {
SSqlObj* pSql = (SSqlObj*) tres;
sem_post(&pSql->rspSem);
}
TAOS_ROW taos_fetch_row(TAOS_RES *res) {
SSqlObj *pSql = (SSqlObj *)res;
if (pSql == NULL || pSql->signature != pSql) {
@ -475,6 +475,42 @@ int taos_select_db(TAOS *taos, const char *db) {
return code;
}
// send free message to vnode to free qhandle and corresponding resources in vnode
static bool tscFreeQhandleInVnode(SSqlObj* pSql) {
SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && !tscIsTwoStageSTableQuery(pQueryInfo, 0) &&
(pCmd->command == TSDB_SQL_SELECT ||
pCmd->command == TSDB_SQL_SHOW ||
pCmd->command == TSDB_SQL_RETRIEVE ||
pCmd->command == TSDB_SQL_FETCH) &&
(pCmd->command == TSDB_SQL_SELECT && pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
tscTrace("%p start to send msg to free qhandle in dnode, command:%s", pSql, sqlCmd[pCmd->command]);
pSql->freed = 1;
tscProcessSql(pSql);
// in case of sync model query, waits for response and then goes on
// if (pSql->fp == waitForQueryRsp || pSql->fp == waitForRetrieveRsp) {
// sem_wait(&pSql->rspSem);
// tscFreeSqlObj(pSql);
// tscTrace("%p sqlObj is freed by app", pSql);
// } else {
tscTrace("%p sqlObj will be freed while rsp received", pSql);
// }
return true;
}
return false;
}
void taos_free_result(TAOS_RES *res) {
SSqlObj *pSql = (SSqlObj *)res;
tscTrace("%p start to free result", res);
@ -484,10 +520,8 @@ void taos_free_result(TAOS_RES *res) {
return;
}
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
// The semaphore can not be changed while freeing async sub query objects.
SSqlRes *pRes = &pSql->res;
if (pRes == NULL || pRes->qhandle == 0) {
tscTrace("%p SqlObj is freed by app, qhandle is null", pSql);
tscFreeSqlObj(pSql);
@ -502,31 +536,10 @@ void taos_free_result(TAOS_RES *res) {
}
pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
/*
* If the query process is cancelled by user in stable query, tscProcessSql should not be called
* for each subquery. Because the failure of execution tsProcessSql may trigger the callback function
* be executed, and the retry efforts may result in double free the resources, e.g.,SRetrieveSupport
*/
if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false &&
(pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_SHOW ||
pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_FETCH) &&
(pCmd->command == TSDB_SQL_SELECT && pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
tscTrace("%p start to send msg to free qhandle in dnode, command:%s", pSql, sqlCmd[pCmd->command]);
pSql->freed = 1;
tscProcessSql(pSql);
// in case of sync model query, waits for response and then goes on
if (pSql->fp == waitForQueryRsp || pSql->fp == waitForRetrieveRsp) {
sem_wait(&pSql->rspSem);
}
if (!tscFreeQhandleInVnode(pSql)) {
tscFreeSqlObj(pSql);
tscTrace("%p sqlObj is freed by app", pSql);
}
tscFreeSqlObj(pSql);
tscTrace("%p sql result is freed by app", pSql);
}
// todo should not be used in async query

8
src/client/src/tscSubquery.c
View File

@ -1662,12 +1662,13 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
*/
if (code != TSDB_CODE_SUCCESS) {
if (trsupport->numOfRetry++ >= MAX_NUM_OF_SUBQUERY_RETRY) {
tscTrace("%p sub:%p reach the max retry times, set global code:%d", pParentSql, pSql, code);
tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code));
atomic_val_compare_exchange_32(&pState->code, 0, code);
} else { // does not reach the maximum retry time, go on
tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry);
SSqlObj *pNew = tscCreateSqlObjForSubquery(pParentSql, trsupport, pSql);
if (pNew == NULL) {
tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vgId:%d, orderOfSub:%d",
trsupport->pParentSqlObj, pSql, pVgroup->vgId, trsupport->subqueryIndex);
@ -1677,7 +1678,8 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
} else {
SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
assert(pNewQueryInfo->pTableMetaInfo[0]->pTableMeta != NULL);
taos_free_result(pSql);
tscProcessSql(pNew);
return;
}
@ -1921,7 +1923,7 @@ static char *getArithemicInputSrc(void *param, const char *name, int32_t colId)
for (int32_t i = 0; i < pSupport->numOfCols; ++i) {
pExpr = taosArrayGetP(pSupport->exprList, i);
if (strncmp(name, pExpr->aliasName, TSDB_COL_NAME_LEN) == 0) {
if (strncmp(name, pExpr->aliasName, sizeof(pExpr->aliasName) - 1) == 0) {
index = i;
break;
}

16
src/client/src/tscSystem.c
View File

@ -144,11 +144,11 @@ void taos_init_imp() {
}
int64_t refreshTime = tsTableMetaKeepTimer;
refreshTime = refreshTime > 2 ? 2 : refreshTime;
refreshTime = refreshTime < 1 ? 1 : refreshTime;
refreshTime = refreshTime > 10 ? 10 : refreshTime;
refreshTime = refreshTime < 10 ? 10 : refreshTime;
if (tscCacheHandle == NULL) {
tscCacheHandle = taosCacheInit(tscTmr, refreshTime);
tscCacheHandle = taosCacheInit(refreshTime);
}
tscTrace("client is initialized successfully");
@ -180,7 +180,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
assert(cfg != NULL);
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
strncpy(configDir, pStr, TSDB_FILENAME_LEN);
tstrncpy(configDir, pStr, TSDB_FILENAME_LEN);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscPrint("set config file directory:%s", pStr);
} else {
@ -234,7 +234,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
tscPrint("failed to set locale:%s, current locale:%s", pStr, tsLocale);
}
strncpy(tsLocale, locale, tListLen(tsLocale));
tstrncpy(tsLocale, locale, sizeof(tsLocale));
char *charset = strrchr(tsLocale, sep);
if (charset != NULL) {
@ -249,7 +249,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
tscPrint("charset changed from %s to %s", tsCharset, charset);
}
strncpy(tsCharset, charset, tListLen(tsCharset));
tstrncpy(tsCharset, charset, sizeof(tsCharset));
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else {
@ -286,7 +286,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
tscPrint("charset changed from %s to %s", tsCharset, pStr);
}
strncpy(tsCharset, pStr, tListLen(tsCharset));
tstrncpy(tsCharset, pStr, sizeof(tsCharset));
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else {
tscPrint("charset:%s not valid", pStr);
@ -324,7 +324,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
// return -1;
// }
strncpy(tsSocketType, pStr, tListLen(tsSocketType));
tstrncpy(tsSocketType, pStr, sizeof(tsSocketType));
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscPrint("socket type is set:%s", tsSocketType);
}

31
src/client/src/tscUtil.c
View File

@ -134,24 +134,6 @@ void tscGetDBInfoFromMeterId(char* tableId, char* db) {
db[0] = 0;
}
//STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) {
// if (pSidList == NULL) {
// tscError("illegal sidlist");
// return 0;
// }
//
// if (idx < 0 || idx >= pSidList->numOfSids) {
// int32_t sidRange = (pSidList->numOfSids > 0) ? (pSidList->numOfSids - 1) : 0;
//
// tscError("illegal sidIdx:%d, reset to 0, sidIdx range:%d-%d", idx, 0, sidRange);
// idx = 0;
// }
//
// assert(pSidList->pSidExtInfoList[idx] >= 0);
//
// return (STableIdInfo*)(pSidList->pSidExtInfoList[idx] + (char*)pSidList);
//}
bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
if (pQueryInfo == NULL) {
return false;
@ -176,8 +158,7 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
return false;
}
if (((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) != TSDB_QUERY_TYPE_STABLE_SUBQUERY) &&
pQueryInfo->command == TSDB_SQL_SELECT) {
if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->command == TSDB_SQL_SELECT) {
return UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
}
@ -593,7 +574,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
dataBuf->size = startOffset;
dataBuf->tsSource = -1;
strncpy(dataBuf->tableId, name, TSDB_TABLE_ID_LEN);
tstrncpy(dataBuf->tableId, name, sizeof(dataBuf->tableId));
/*
* The table meta may be released since the table meta cache are completed clean by other thread
@ -801,7 +782,7 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) {
TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) {
TAOS_FIELD f = { .type = type, .bytes = bytes, };
strncpy(f.name, name, TSDB_COL_NAME_LEN);
tstrncpy(f.name, name, sizeof(f.name));
return f;
}
@ -966,12 +947,12 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
if (isTagCol) {
SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId;
strncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, TSDB_COL_NAME_LEN);
tstrncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, sizeof(pExpr->colInfo.name));
} else if (pTableMetaInfo->pTableMeta != NULL) {
// in handling select database/version/server_status(), the pTableMeta is NULL
SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex);
pExpr->colInfo.colId = pSchema->colId;
strncpy(pExpr->colInfo.name, pSchema->name, TSDB_COL_NAME_LEN);
tstrncpy(pExpr->colInfo.name, pSchema->name, sizeof(pExpr->colInfo.name));
}
}
@ -1666,7 +1647,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
assert(pTableMetaInfo != NULL);
if (name != NULL) {
strncpy(pTableMetaInfo->name, name, TSDB_TABLE_ID_LEN);
tstrncpy(pTableMetaInfo->name, name, sizeof(pTableMetaInfo->name));
}
pTableMetaInfo->pTableMeta = pTableMeta;

17
src/common/inc/tdataformat.h
View File

@ -31,7 +31,7 @@ extern "C" {
do { \
VarDataLenT __len = strlen(str); \
*(VarDataLenT *)(x) = __len; \
strncpy(varDataVal(x), (str), __len); \
memcpy(varDataVal(x), (str), __len); \
} while (0);
#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \
@ -43,7 +43,7 @@ extern "C" {
#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \
do { \
*(VarDataLenT *)(x) = (_size); \
strncpy(varDataVal(x), (str), (_size)); \
memcpy(varDataVal(x), (str), (_size)); \
} while (0);
// ----------------- TSDB COLUMN DEFINITION
@ -69,7 +69,8 @@ typedef struct {
int version; // version
int numOfCols; // Number of columns appended
int tlen; // maximum length of a SDataRow without the header part
int flen; // First part length in a SDataRow after the header part
uint16_t flen; // First part length in a SDataRow after the header part
uint16_t vlen; // pure value part length, excluded the overhead
STColumn columns[];
} STSchema;
@ -77,6 +78,7 @@ typedef struct {
#define schemaVersion(s) ((s)->version)
#define schemaTLen(s) ((s)->tlen)
#define schemaFLen(s) ((s)->flen)
#define schemaVLen(s) ((s)->vlen)
#define schemaColAt(s, i) ((s)->columns + i)
#define tdFreeSchema(s) tfree((s))
@ -105,7 +107,8 @@ typedef struct {
int tCols;
int nCols;
int tlen;
int flen;
uint16_t flen;
uint16_t vlen;
int version;
STColumn *columns;
} STSchemaBuilder;
@ -122,16 +125,16 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
* |<--------------------+--------------------------- len ---------------------------------->|
* |<-- Head -->|<--------- flen -------------->| |
* +---------------------+---------------------------------+---------------------------------+
* | int16_t | int16_t | | |
* | uint16_t | int16_t | | |
* +----------+----------+---------------------------------+---------------------------------+
* | len | sversion | First part | Second part |
* +----------+----------+---------------------------------+---------------------------------+
*/
typedef void *SDataRow;
#define TD_DATA_ROW_HEAD_SIZE sizeof(int16_t)*2
#define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
#define dataRowLen(r) (*(int16_t *)(r))
#define dataRowLen(r) (*(uint16_t *)(r))
#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))
#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))

2
src/common/inc/tglobal.h
View File

@ -53,6 +53,7 @@ extern int64_t tsMsPerDay[3];
extern char tsFirst[];
extern char tsSecond[];
extern char tsLocalFqdn[];
extern char tsLocalEp[];
extern uint16_t tsServerPort;
extern uint16_t tsDnodeShellPort;
@ -169,6 +170,7 @@ extern char gitinfo[];
extern char gitinfoOfInternal[];
extern char buildinfo[];
extern int8_t tsDaylight;
extern char tsTimezone[64];
extern char tsLocale[64];
extern char tsCharset[64]; // default encode string

3
src/common/inc/tname.h
View File

@ -23,5 +23,8 @@ void extractTableName(const char *tableId, char *name);
char* extractDBName(const char *tableId, char *name);
SSchema tGetTableNameColumnSchema();
bool tscValidateTableNameLength(size_t len);
#endif // TDENGINE_NAME_H

15
src/common/src/tdataformat.c
View File

@ -100,6 +100,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
pBuilder->nCols = 0;
pBuilder->tlen = 0;
pBuilder->flen = 0;
pBuilder->vlen = 0;
pBuilder->version = version;
}
@ -124,10 +125,12 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int3
if (IS_VAR_DATA_TYPE(type)) {
colSetBytes(pCol, bytes);
pBuilder->tlen += (TYPE_BYTES[type] + sizeof(VarDataLenT) + bytes);
pBuilder->tlen += (TYPE_BYTES[type] + bytes);
pBuilder->vlen += bytes - sizeof(VarDataLenT);
} else {
colSetBytes(pCol, TYPE_BYTES[type]);
pBuilder->tlen += TYPE_BYTES[type];
pBuilder->vlen += TYPE_BYTES[type];
}
pBuilder->nCols++;
@ -150,6 +153,7 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
schemaNCols(pSchema) = pBuilder->nCols;
schemaTLen(pSchema) = pBuilder->tlen;
schemaFLen(pSchema) = pBuilder->flen;
schemaVLen(pSchema) = pBuilder->vlen;
memcpy(schemaColAt(pSchema, 0), pBuilder->columns, sizeof(STColumn) * pBuilder->nCols);
@ -197,10 +201,10 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
pDataCol->len = 0;
if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
pDataCol->spaceSize = (sizeof(VarDataLenT) + pDataCol->bytes) * maxPoints;
pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
pDataCol->pData = POINTER_SHIFT(*pBuf, TYPE_BYTES[pDataCol->type] * maxPoints);
*pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize + TYPE_BYTES[pDataCol->type] * maxPoints);
pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints);
pDataCol->spaceSize = pDataCol->bytes * maxPoints;
*pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize + sizeof(VarDataOffsetT) * maxPoints);
} else {
pDataCol->spaceSize = pDataCol->bytes * maxPoints;
pDataCol->dataOff = NULL;
@ -269,8 +273,7 @@ void dataColSetNullAt(SDataCol *pCol, int index) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
pCol->dataOff[index] = pCol->len;
char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
varDataLen(ptr) = (pCol->type == TSDB_DATA_TYPE_BINARY) ? sizeof(char) : TSDB_NCHAR_SIZE;
setNull(varDataVal(ptr), pCol->type, pCol->bytes);
setVardataNull(ptr, pCol->type);
pCol->len += varDataTLen(ptr);
} else {
setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes);

31
src/common/src/tglobal.c
View File

@ -65,6 +65,7 @@ int64_t tsMsPerDay[] = {86400000L, 86400000000L, 86400000000000L};
char tsFirst[TSDB_EP_LEN] = {0};
char tsSecond[TSDB_EP_LEN] = {0};
char tsArbitrator[TSDB_EP_LEN] = {0};
char tsLocalFqdn[TSDB_FQDN_LEN] = {0};
char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port
uint16_t tsServerPort = 6030;
uint16_t tsDnodeShellPort = 6030; // udp[6035-6039] tcp[6035]
@ -197,6 +198,7 @@ char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
char tsInternalPass[] = "secretkey";
int32_t tsMonitorInterval = 30; // seconds
int8_t tsDaylight = 0;
char tsTimezone[64] = {0};
char tsLocale[TSDB_LOCALE_LEN] = {0};
char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string
@ -305,6 +307,16 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "fqdn";
cfg.ptr = tsLocalFqdn;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_FQDN_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// port
cfg.option = "serverPort";
cfg.ptr = &tsServerPort;
@ -708,7 +720,7 @@ static void doInitGlobalConfig() {
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_DB_NAME_LEN;
cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
@ -718,7 +730,7 @@ static void doInitGlobalConfig() {
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_USER_LEN;
cfg.ptrLength = TSDB_USER_LEN - 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
@ -728,7 +740,7 @@ static void doInitGlobalConfig() {
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_NOT_PRINT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_PASSWORD_LEN;
cfg.ptrLength = TSDB_PASSWORD_LEN - 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
@ -923,7 +935,7 @@ static void doInitGlobalConfig() {
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_DB_NAME_LEN;
cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
@ -1251,9 +1263,14 @@ bool taosCheckGlobalCfg() {
taosSetAllDebugFlag();
}
taosGetFqdn(tsLocalEp);
sprintf(tsLocalEp + strlen(tsLocalEp), ":%d", tsServerPort);
uPrint("localEp is %s", tsLocalEp);
if (tsLocalFqdn[0] == 0) {
taosGetFqdn(tsLocalFqdn);
}
strcpy(tsLocalEp, tsLocalFqdn);
snprintf(tsLocalEp + strlen(tsLocalEp), sizeof(tsLocalEp), ":%d", tsServerPort);
uPrint("localEp is: %s", tsLocalEp);
if (tsFirst[0] == 0) {
strcpy(tsFirst, tsLocalEp);

16
src/common/src/tname.c
View File

@ -28,8 +28,7 @@ void extractTableName(const char* tableId, char* name) {
size_t s1 = strcspn(tableId, &TS_PATH_DELIMITER[0]);
size_t s2 = strcspn(&tableId[s1 + 1], &TS_PATH_DELIMITER[0]);
strncpy(name, &tableId[s1 + s2 + 2], TSDB_TABLE_NAME_LEN);
name[TSDB_TABLE_NAME_LEN] = 0;
tstrncpy(name, &tableId[s1 + s2 + 2], TSDB_TABLE_NAME_LEN);
}
char* extractDBName(const char* tableId, char* name) {
@ -38,3 +37,16 @@ char* extractDBName(const char* tableId, char* name) {
return strncpy(name, &tableId[offset1 + 1], len);
}
SSchema tGetTableNameColumnSchema() {
SSchema s = {0};
s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE;
s.type = TSDB_DATA_TYPE_BINARY;
s.colId = TSDB_TBNAME_COLUMN_INDEX;
strncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN);
return s;
}
bool tscValidateTableNameLength(size_t len) {
return len < TSDB_TABLE_NAME_LEN;
}

1
src/common/src/ttimezone.c
View File

@ -58,6 +58,7 @@ void tsSetTimeZone() {
* (BST, +0100)
*/
sprintf(tsTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
tsDaylight = daylight;
uPrint("timezone format changed to %s", tsTimezone);
}

6
src/connector/python/linux/python2/taos/cinterface.py
View File

@ -130,9 +130,9 @@ _CONVERT_FUNC = {
# Corresponding TAOS_FIELD structure in C
class TaosField(ctypes.Structure):
_fields_ = [('name', ctypes.c_char * 64),
('bytes', ctypes.c_short),
('type', ctypes.c_char)]
_fields_ = [('name', ctypes.c_char * 65),
('type', ctypes.c_char),
('bytes', ctypes.c_short)]
# C interface class
class CTaosInterface(object):

12
src/connector/python/linux/python3/taos/cinterface.py
View File

@ -130,9 +130,9 @@ _CONVERT_FUNC = {
# Corresponding TAOS_FIELD structure in C
class TaosField(ctypes.Structure):
_fields_ = [('name', ctypes.c_char * 64),
('bytes', ctypes.c_short),
('type', ctypes.c_char)]
_fields_ = [('name', ctypes.c_char * 65),
('type', ctypes.c_char),
('bytes', ctypes.c_short)]
# C interface class
class CTaosInterface(object):
@ -226,8 +226,8 @@ class CTaosInterface(object):
if connection.value == None:
print('connect to TDengine failed')
# sys.exit(1)
else:
print('connect to TDengine success')
#else:
# print('connect to TDengine success')
return connection
@ -236,7 +236,7 @@ class CTaosInterface(object):
'''Close the TDengine handle
'''
CTaosInterface.libtaos.taos_close(connection)
print('connection is closed')
#print('connection is closed')
@staticmethod
def query(connection, sql):

6
src/connector/python/windows/python2/taos/cinterface.py
View File

@ -130,9 +130,9 @@ _CONVERT_FUNC = {
# Corresponding TAOS_FIELD structure in C
class TaosField(ctypes.Structure):
_fields_ = [('name', ctypes.c_char * 64),
('bytes', ctypes.c_short),
('type', ctypes.c_char)]
_fields_ = [('name', ctypes.c_char * 65),
('type', ctypes.c_char),
('bytes', ctypes.c_short)]
# C interface class
class CTaosInterface(object):

6
src/connector/python/windows/python3/taos/cinterface.py
View File

@ -130,9 +130,9 @@ _CONVERT_FUNC = {
# Corresponding TAOS_FIELD structure in C
class TaosField(ctypes.Structure):
_fields_ = [('name', ctypes.c_char * 64),
('bytes', ctypes.c_short),
('type', ctypes.c_char)]
_fields_ = [('name', ctypes.c_char * 65),
('type', ctypes.c_char),
('bytes', ctypes.c_short)]
# C interface class
class CTaosInterface(object):

8
src/cq/src/cqMain.c
View File

@ -73,8 +73,8 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
return NULL;
}
strcpy(pContext->user, pCfg->user);
strcpy(pContext->pass, pCfg->pass);
tstrncpy(pContext->user, pCfg->user, sizeof(pContext->user));
tstrncpy(pContext->pass, pCfg->pass, sizeof(pContext->pass));
const char* db = pCfg->db;
for (const char* p = db; *p != 0; p++) {
if (*p == '.') {
@ -82,7 +82,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
break;
}
}
strcpy(pContext->db, db);
tstrncpy(pContext->db, db, sizeof(pContext->db));
pContext->vgId = pCfg->vgId;
pContext->cqWrite = pCfg->cqWrite;
pContext->ahandle = ahandle;
@ -215,7 +215,7 @@ void cqDrop(void *handle) {
cTrace("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr);
free(pObj);
pthread_mutex_lock(&pContext->mutex);
pthread_mutex_unlock(&pContext->mutex);
}
static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {

8
src/dnode/src/dnodeMain.c
View File

@ -61,12 +61,10 @@ static const SDnodeComponent tsDnodeComponents[] = {
};
static int dnodeCreateDir(const char *dir) {
struct stat dirstat;
if (stat(dir, &dirstat) < 0) {
if (mkdir(dir, 0755) != 0 && errno != EEXIST) {
return -1;
}
if (mkdir(dir, 0755) != 0 && errno != EEXIST) {
return -1;
}
return 0;
}

40
src/dnode/src/dnodeMgmt.c
View File

@ -177,10 +177,12 @@ void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
memcpy(item, pMsg, sizeof(SRpcMsg));
taosWriteQitem(tsMgmtQueue, 1, item);
} else {
SRpcMsg rsp;
rsp.handle = pMsg->handle;
rsp.pCont = NULL;
rsp.code = TSDB_CODE_DND_OUT_OF_MEMORY;
SRpcMsg rsp = {
.handle = pMsg->handle,
.pCont = NULL,
.code = TSDB_CODE_DND_OUT_OF_MEMORY
};
rpcSendResponse(&rsp);
rpcFreeCont(pMsg->pCont);
}
@ -188,9 +190,9 @@ void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
static void *dnodeProcessMgmtQueue(void *param) {
SRpcMsg *pMsg;
SRpcMsg rsp;
SRpcMsg rsp = {0};
int type;
void *handle;
void * handle;
while (1) {
if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) {
@ -251,6 +253,7 @@ static int32_t dnodeOpenVnodes() {
if (status != TSDB_CODE_SUCCESS) {
dPrint("Get dnode list failed");
free(vnodeList);
return status;
}
@ -290,6 +293,7 @@ static void dnodeCloseVnodes() {
if (status != TSDB_CODE_SUCCESS) {
dPrint("Get dnode list failed");
free(vnodeList);
return;
}
@ -410,15 +414,35 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
dnodeProcessModuleStatus(pCfg->moduleStatus);
dnodeUpdateDnodeCfg(pCfg);
dnodeUpdateMnodeInfos(pMnodes);
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
}
static bool dnodeCheckMnodeInfos(SDMMnodeInfos *pMnodes) {
if (pMnodes->nodeNum <= 0 || pMnodes->nodeNum > 3) {
dError("invalid mnode infos, num:%d", pMnodes->nodeNum);
return false;
}
for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
if (pMnodeInfo->nodeId <= 0 || strlen(pMnodeInfo->nodeEp) <= 5) {
dError("invalid mnode info:%d, nodeId:%d nodeEp:%s", i, pMnodeInfo->nodeId, pMnodeInfo->nodeEp);
return false;
}
}
return true;
}
static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
bool mnodesChanged = (memcmp(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)) != 0);
bool mnodesNotInit = (tsDMnodeInfos.nodeNum == 0);
if (!(mnodesChanged || mnodesNotInit)) return;
if (!dnodeCheckMnodeInfos(pMnodes)) return;
memcpy(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos));
dPrint("mnode infos is changed, nodeNum:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
@ -456,6 +480,7 @@ static bool dnodeReadMnodeInfos() {
return false;
}
content[len] = 0;
cJSON* root = cJSON_Parse(content);
if (root == NULL) {
dError("failed to read mnodeIpList.json, invalid json format");
@ -547,6 +572,7 @@ static void dnodeSaveMnodeInfos() {
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
fflush(fp);
fclose(fp);
free(content);
@ -628,6 +654,7 @@ static bool dnodeReadDnodeCfg() {
return false;
}
content[len] = 0;
cJSON* root = cJSON_Parse(content);
if (root == NULL) {
dError("failed to read dnodeCfg.json, invalid json format");
@ -668,6 +695,7 @@ static void dnodeSaveDnodeCfg() {
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
fflush(fp);
fclose(fp);
free(content);

11
src/dnode/src/dnodePeer.c
View File

@ -83,10 +83,13 @@ void dnodeCleanupServer() {
}
static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
SRpcMsg rspMsg;
rspMsg.handle = pMsg->handle;
rspMsg.pCont = NULL;
rspMsg.contLen = 0;
SRpcMsg rspMsg = {
.handle = pMsg->handle,
.pCont = NULL,
.contLen = 0
};
if (pMsg->pCont == NULL) return;
if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
rspMsg.code = TSDB_CODE_RPC_NOT_READY;

16
src/dnode/src/dnodeShell.c
View File

@ -19,6 +19,7 @@
#include "taosdef.h"
#include "taosmsg.h"
#include "tglobal.h"
#include "tutil.h"
#include "http.h"
#include "mnode.h"
#include "dnode.h"
@ -108,10 +109,13 @@ void dnodeCleanupShell() {
}
void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
SRpcMsg rpcMsg;
rpcMsg.handle = pMsg->handle;
rpcMsg.pCont = NULL;
rpcMsg.contLen = 0;
SRpcMsg rpcMsg = {
.handle = pMsg->handle,
.pCont = NULL,
.contLen = 0
};
if (pMsg->pCont == NULL) return;
if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
@ -143,7 +147,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
if (code != TSDB_CODE_RPC_NOT_READY) return code;
SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg));
strcpy(pMsg->user, user);
tstrncpy(pMsg->user, user, sizeof(pMsg->user));
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pMsg;
@ -201,7 +205,7 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid) {
int16_t numOfTags = htons(pTable->numOfTags);
int32_t sid = htonl(pTable->sid);
uint64_t uid = htobe64(pTable->uid);
dPrint("table:%s, numOfColumns:%d numOfTags:%d sid:%d uid:%d", pTable->tableId, numOfColumns, numOfTags, sid, uid);
dPrint("table:%s, numOfColumns:%d numOfTags:%d sid:%d uid:%" PRIu64, pTable->tableId, numOfColumns, numOfTags, sid, uid);
return rpcRsp.pCont;
}

4
src/dnode/src/dnodeSystem.c
View File

@ -29,11 +29,11 @@ int32_t main(int32_t argc, char *argv[]) {
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) > TSDB_FILENAME_LEN - 1) {
if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) {
printf("config file path overflow");
exit(EXIT_FAILURE);
}
strcpy(configDir, argv[i]);
tstrncpy(configDir, argv[i], TSDB_FILENAME_LEN);
} else {
printf("'-c' requires a parameter, default:%s\n", configDir);
exit(EXIT_FAILURE);

9
src/dnode/src/dnodeVRead.c
View File

@ -26,13 +26,6 @@
#include "dnodeVRead.h"
#include "vnode.h"
typedef struct {
SRspRet rspRet;
void *pCont;
int32_t contLen;
SRpcMsg rpcMsg;
} SReadMsg;
typedef struct {
pthread_t thread; // thread
int32_t workerId; // worker ID
@ -218,7 +211,7 @@ static void *dnodeProcessReadQueue(void *param) {
}
dTrace("%p, msg:%s will be processed in vread queue", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]);
int32_t code = vnodeProcessRead(pVnode, pReadMsg->rpcMsg.msgType, pReadMsg->pCont, pReadMsg->contLen, &pReadMsg->rspRet);
int32_t code = vnodeProcessRead(pVnode, pReadMsg);
dnodeSendRpcReadRsp(pVnode, pReadMsg, code);
taosFreeQitem(pReadMsg);
}

9
src/inc/query.h
View File

@ -70,13 +70,20 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo);
int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen);
/**
* Decide if more results will be produced or not
* Decide if more results will be produced or not, NOTE: this function will increase the ref count of QInfo,
* so it can be only called once for each retrieve
*
* @param qinfo
* @return
*/
bool qHasMoreResultsToRetrieve(qinfo_t qinfo);
/**
* kill current ongoing query and free query handle automatically
* @param qinfo
*/
int32_t qKillQuery(qinfo_t qinfo);
#ifdef __cplusplus
}
#endif

4
src/inc/taos.h
View File

@ -53,9 +53,9 @@ typedef enum {
} TSDB_OPTION;
typedef struct taosField {
char name[64];
short bytes;
char name[65];
uint8_t type;
short bytes;
} TAOS_FIELD;
#ifdef _TD_GO_DLL_

21
src/inc/taosdef.h
View File

@ -186,30 +186,31 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_BINARY_OP_MULTIPLY 14
#define TSDB_BINARY_OP_DIVIDE 15
#define TSDB_BINARY_OP_REMAINDER 16
#define TSDB_USERID_LEN 9
#define TS_PATH_DELIMITER_LEN 1
#define TSDB_METER_ID_LEN_MARGIN 10
#define TSDB_TABLE_ID_LEN (TSDB_DB_NAME_LEN+TSDB_TABLE_NAME_LEN+2*TS_PATH_DELIMITER_LEN+TSDB_USERID_LEN+TSDB_METER_ID_LEN_MARGIN) //TSDB_DB_NAME_LEN+TSDB_TABLE_NAME_LEN+2*strlen(TS_PATH_DELIMITER)+strlen(USERID)
#define TSDB_UNI_LEN 24
#define TSDB_USER_LEN TSDB_UNI_LEN
#define TSDB_ACCT_LEN TSDB_UNI_LEN
// ACCOUNT is a 32 bit positive integer
// this is the length of its string representation
// including the terminator zero
#define TSDB_ACCT_LEN 11
#define TSDB_PASSWORD_LEN TSDB_UNI_LEN
#define TSDB_MAX_COLUMNS 1024
#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns
#define TSDB_NODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 192
#define TSDB_DB_NAME_LEN 32
#define TSDB_COL_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string
#define TSDB_DB_NAME_LEN 33
#define TSDB_TABLE_ID_LEN (TSDB_ACCT_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN)
#define TSDB_COL_NAME_LEN 65
#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64
#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE
#define TSDB_MAX_SQL_SHOW_LEN 256
#define TSDB_MAX_ALLOWED_SQL_LEN (8*1024*1024U) // sql length should be less than 6mb
#define TSDB_MAX_ALLOWED_SQL_LEN (8*1024*1024U) // sql length should be less than 8mb
#define TSDB_MAX_BYTES_PER_ROW TSDB_MAX_COLUMNS * 64
#define TSDB_MAX_TAGS_LEN 65536
#define TSDB_MAX_BYTES_PER_ROW 65535
#define TSDB_MAX_TAGS_LEN 65535
#define TSDB_MAX_TAGS 128
#define TSDB_AUTH_LEN 16

1
src/inc/taoserror.h
View File

@ -71,6 +71,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT, 0, 0x0100, "operations
TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED, 0, 0x0101, "memory corrupted")
TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY, 0, 0x0102, "out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG, 0, 0x0103, "invalid config message")
TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED, 0, 0x0104, "file is corrupted")
//client
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL, 0, 0x0200, "invalid sql")

57
src/inc/taosmsg.h
View File

@ -224,7 +224,7 @@ typedef struct {
typedef struct SSchema {
uint8_t type;
char name[TSDB_COL_NAME_LEN + 1];
char name[TSDB_COL_NAME_LEN];
int16_t colId;
int16_t bytes;
} SSchema;
@ -243,14 +243,14 @@ typedef struct {
uint64_t uid;
uint64_t superTableUid;
uint64_t createdTime;
char tableId[TSDB_TABLE_ID_LEN + 1];
char superTableId[TSDB_TABLE_ID_LEN + 1];
char tableId[TSDB_TABLE_ID_LEN];
char superTableId[TSDB_TABLE_ID_LEN];
char data[];
} SMDCreateTableMsg;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1];
char tableId[TSDB_TABLE_ID_LEN];
char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
int8_t igExists;
int8_t getMeta;
int16_t numOfTags;
@ -262,13 +262,13 @@ typedef struct {
} SCMCreateTableMsg;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN + 1];
char tableId[TSDB_TABLE_ID_LEN];
int8_t igNotExists;
} SCMDropTableMsg;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1];
char tableId[TSDB_TABLE_ID_LEN];
char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
int16_t type; /* operation type */
int16_t numOfCols; /* number of schema */
int32_t tagValLen;
@ -292,11 +292,11 @@ typedef struct {
typedef struct {
char clientVersion[TSDB_VERSION_LEN];
char msgVersion[TSDB_VERSION_LEN];
char db[TSDB_TABLE_ID_LEN + 1];
char db[TSDB_TABLE_ID_LEN];
} SCMConnectMsg;
typedef struct {
char acctId[TSDB_ACCT_LEN + 1];
char acctId[TSDB_ACCT_LEN];
char serverVersion[TSDB_VERSION_LEN];
int8_t writeAuth;
int8_t superAuth;
@ -321,18 +321,18 @@ typedef struct {
} SAcctCfg;
typedef struct {
char user[TSDB_USER_LEN + 1];
char pass[TSDB_KEY_LEN + 1];
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
SAcctCfg cfg;
} SCMCreateAcctMsg, SCMAlterAcctMsg;
typedef struct {
char user[TSDB_USER_LEN + 1];
char user[TSDB_USER_LEN];
} SCMDropUserMsg, SCMDropAcctMsg;
typedef struct {
char user[TSDB_USER_LEN + 1];
char pass[TSDB_KEY_LEN + 1];
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
int8_t privilege;
int8_t flag;
} SCMCreateUserMsg, SCMAlterUserMsg;
@ -342,14 +342,14 @@ typedef struct {
int32_t vgId;
int32_t sid;
uint64_t uid;
char tableId[TSDB_TABLE_ID_LEN + 1];
char tableId[TSDB_TABLE_ID_LEN];
} SMDDropTableMsg;
typedef struct {
int32_t contLen;
int32_t vgId;
uint64_t uid;
char tableId[TSDB_TABLE_ID_LEN + 1];
char tableId[TSDB_TABLE_ID_LEN];
} SMDDropSTableMsg;
typedef struct {
@ -501,8 +501,7 @@ typedef struct {
} SVnodeLoad;
typedef struct {
char acct[TSDB_USER_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1];
char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
int32_t cacheBlockSize; //MB
int32_t totalBlocks;
int32_t maxTables;
@ -521,7 +520,7 @@ typedef struct {
} SCMCreateDbMsg, SCMAlterDbMsg;
typedef struct {
char db[TSDB_TABLE_ID_LEN + 1];
char db[TSDB_TABLE_ID_LEN];
uint8_t ignoreNotExists;
} SCMDropDbMsg, SCMUseDbMsg;
@ -606,13 +605,13 @@ typedef struct {
} SMDVnodeDesc;
typedef struct {
char db[TSDB_DB_NAME_LEN + 1];
char db[TSDB_DB_NAME_LEN];
SMDVnodeCfg cfg;
SMDVnodeDesc nodes[TSDB_MAX_REPLICA];
} SMDCreateVnodeMsg;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN + 1];
char tableId[TSDB_TABLE_ID_LEN];
int16_t createFlag;
char tags[];
} SCMTableInfoMsg;
@ -639,7 +638,7 @@ typedef struct {
typedef struct STableMetaMsg {
int32_t contLen;
char tableId[TSDB_TABLE_ID_LEN + 1]; // table id
char tableId[TSDB_TABLE_ID_LEN]; // table id
uint8_t numOfTags;
uint8_t precision;
uint8_t tableType;
@ -660,7 +659,7 @@ typedef struct SMultiTableMeta {
typedef struct {
int32_t dataLen;
char name[TSDB_TABLE_ID_LEN + 1];
char name[TSDB_TABLE_ID_LEN];
char data[TSDB_MAX_TAGS_LEN];
} STagData;
@ -671,7 +670,7 @@ typedef struct {
*/
typedef struct {
int8_t type;
char db[TSDB_DB_NAME_LEN + 1];
char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
uint16_t payloadLen;
char payload[];
} SCMShowMsg;
@ -746,15 +745,15 @@ typedef struct {
uint64_t uid;
uint64_t stime; // stream starting time
int32_t status;
char tableId[TSDB_TABLE_ID_LEN + 1];
char tableId[TSDB_TABLE_ID_LEN];
} SMDAlterStreamMsg;
typedef struct {
char user[TSDB_USER_LEN + 1];
char user[TSDB_USER_LEN];
char spi;
char encrypt;
char secret[TSDB_KEY_LEN + 1];
char ckey[TSDB_KEY_LEN + 1];
char secret[TSDB_KEY_LEN];
char ckey[TSDB_KEY_LEN];
} SDMAuthMsg, SDMAuthRsp;
#pragma pack(pop)

2
src/inc/tcq.h
View File

@ -27,7 +27,7 @@ typedef struct {
int vgId;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_NAME_LEN + 1];
char db[TSDB_DB_NAME_LEN];
FCqWrite cqWrite;
} SCqCfg;

4
src/inc/trpc.h
View File

@ -78,11 +78,13 @@ void rpcClose(void *);
void *rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont);
void *rpcReallocCont(void *ptr, int contLen);
void rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg);
void *rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg);
void rpcSendResponse(const SRpcMsg *pMsg);
void rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen);
void rpcCanelRequest(void *pContext);
#ifdef __cplusplus
}

16
src/inc/tsdb.h
View File

@ -65,6 +65,13 @@ typedef struct {
int8_t compression;
} STsdbCfg;
// --------- TSDB REPOSITORY USAGE STATISTICS
typedef struct {
int64_t totalStorage; // total bytes occupie
int64_t compStorage;
int64_t pointsWritten; // total data points written
} STsdbStat;
typedef void TsdbRepoT; // use void to hide implementation details from outside
void tsdbSetDefaultCfg(STsdbCfg *pCfg);
@ -306,6 +313,15 @@ int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, uint64_t uid, STableGroupInfo *pGr
*/
void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
/**
* get the statistics of repo usage
* @param repo. point to the tsdbrepo
* @param totalPoints. total data point written
* @param totalStorage. total bytes took by the tsdb
* @param compStorage. total bytes took by the tsdb after compressed
*/
void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage);
#ifdef __cplusplus
}
#endif

9
src/inc/vnode.h
View File

@ -34,6 +34,13 @@ typedef struct {
void *qhandle; //used by query and retrieve msg
} SRspRet;
typedef struct {
SRspRet rspRet;
void *pCont;
int32_t contLen;
SRpcMsg rpcMsg;
} SReadMsg;
int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg);
int32_t vnodeDrop(int32_t vgId);
int32_t vnodeOpen(int32_t vgId, char *rootDir);
@ -52,7 +59,7 @@ void* vnodeGetWal(void *pVnode);
int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item);
void vnodeBuildStatusMsg(void * param);
int32_t vnodeProcessRead(void *pVnode, int msgType, void *pCont, int32_t contLen, SRspRet *ret);
int32_t vnodeProcessRead(void *pVnode, SReadMsg *pReadMsg);
#ifdef __cplusplus
}

2
src/kit/shell/src/shellDarwin.c
View File

@ -97,7 +97,7 @@ void shellParseArgument(int argc, char *argv[], struct arguments *arguments) {
}
} else if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) > TSDB_FILENAME_LEN - 1) {
if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) {
fprintf(stderr, "config file path: %s overflow max len %d\n", argv[i], TSDB_FILENAME_LEN - 1);
exit(EXIT_FAILURE);
}

1
src/kit/shell/src/shellEngine.c
View File

@ -469,7 +469,6 @@ static int dumpResultToFile(const char* fname, TAOS_RES* result) {
} while( row != NULL);
fclose(fp);
taos_free_result(result);
return numOfRows;
}

2
src/kit/shell/src/shellLinux.c
View File

@ -81,7 +81,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
fprintf(stderr, "Invalid path %s\n", arg);
return -1;
}
if (strlen(full_path.we_wordv[0]) > TSDB_FILENAME_LEN - 1) {
if (strlen(full_path.we_wordv[0]) >= TSDB_FILENAME_LEN) {
fprintf(stderr, "config file path: %s overflow max len %d\n", full_path.we_wordv[0], TSDB_FILENAME_LEN - 1);
wordfree(&full_path);
return -1;

2
src/kit/shell/src/shellWindows.c
View File

@ -77,7 +77,7 @@ void shellParseArgument(int argc, char *argv[], struct arguments *arguments) {
}
} else if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) > TSDB_FILENAME_LEN - 1) {
if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) {
fprintf(stderr, "config file path: %s overflow max len %d\n", argv[i], TSDB_FILENAME_LEN - 1);
exit(EXIT_FAILURE);
}

42
src/kit/taosdemo/taosdemo.c
View File

@ -297,7 +297,7 @@ void *deleteTable();
void *asyncWrite(void *sarg);
void generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary);
int generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary);
void rand_string(char *str, int size);
@ -710,7 +710,7 @@ void *readTable(void *sarg) {
int32_t code = taos_errno(pSql);
if (code != 0) {
fprintf(stderr, "Failed to query:%s\n", taos_errstr(taos));
fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql));
taos_free_result(pSql);
taos_close(taos);
exit(EXIT_FAILURE);
@ -756,7 +756,7 @@ void *readMetric(void *sarg) {
for (int j = 0; j < n; j++) {
char condition[BUFFER_SIZE - 30] = "\0";
char tempS[BUFFER_SIZE] = "\0";
char tempS[64] = "\0";
int m = 10 < num_of_tables ? 10 : num_of_tables;
@ -779,7 +779,7 @@ void *readMetric(void *sarg) {
int32_t code = taos_errno(pSql);
if (code != 0) {
fprintf(stderr, "Failed to query:%s\n", taos_errstr(taos));
fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql));
taos_free_result(pSql);
taos_close(taos);
exit(1);
@ -817,8 +817,10 @@ void queryDB(TAOS *taos, char *command) {
i--;
}
if (i == 0) {
fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(taos));
if (code != 0) {
fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
taos_free_result(pSql);
taos_close(taos);
exit(EXIT_FAILURE);
}
@ -844,14 +846,19 @@ void *syncWrite(void *sarg) {
int k;
for (k = 0; k < winfo->nrecords_per_request;) {
int rand_num = rand() % 100;
if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate)
{
int len = -1;
if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) {
long d = tmp_time - rand() % 1000000 + rand_num;
generateData(data, data_type, ncols_per_record, d, len_of_binary);
} else
{
generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary);
len = generateData(data, data_type, ncols_per_record, d, len_of_binary);
} else {
len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary);
}
//assert(len + pstr - buffer < BUFFER_SIZE);
if (len + pstr - buffer >= BUFFER_SIZE) { // too long
break;
}
pstr += sprintf(pstr, " %s", data);
inserted++;
k++;
@ -914,7 +921,7 @@ void callBack(void *param, TAOS_RES *res, int code) {
int64_t tmp_time = tb_info->timestamp;
if (code < 0) {
fprintf(stderr, "failed to insert data %d:reason; %s\n", code, taos_errstr(tb_info->taos));
fprintf(stderr, "failed to insert data %d:reason; %s\n", code, taos_errstr(res));
exit(EXIT_FAILURE);
}
@ -966,7 +973,7 @@ double getCurrentTime() {
return tv.tv_sec + tv.tv_usec / 1E6;
}
void generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary) {
int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary) {
memset(res, 0, MAX_DATA_SIZE);
char *pstr = res;
pstr += sprintf(pstr, "(%" PRId64, timestamp);
@ -1000,9 +1007,16 @@ void generateData(char *res, char **data_type, int num_of_cols, int64_t timestam
rand_string(s, len_of_binary);
pstr += sprintf(pstr, ", \"%s\"", s);
}
if (pstr - res > MAX_DATA_SIZE) {
perror("column length too long, abort");
exit(-1);
}
}
pstr += sprintf(pstr, ")");
return pstr - res;
}
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890";

14
src/kit/taosdump/taosdump.c
View File

@ -88,21 +88,21 @@ enum _describe_table_index {
};
typedef struct {
char field[TSDB_COL_NAME_LEN + 1];
char field[TSDB_COL_NAME_LEN];
char type[16];
int length;
char note[128];
} SColDes;
typedef struct {
char name[TSDB_COL_NAME_LEN + 1];
char name[TSDB_COL_NAME_LEN];
SColDes cols[];
} STableDef;
extern char version[];
typedef struct {
char name[TSDB_DB_NAME_LEN + 1];
char name[TSDB_DB_NAME_LEN];
int32_t replica;
int32_t days;
int32_t keep;
@ -177,8 +177,8 @@ typedef struct SDumpArguments {
char *password;
uint16_t port;
// output file
char output[TSDB_FILENAME_LEN + 1];
char input[TSDB_FILENAME_LEN + 1];
char output[TSDB_FILENAME_LEN];
char input[TSDB_FILENAME_LEN];
char *encode;
// dump unit option
bool all_databases;
@ -643,6 +643,8 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) {
lseek(fd, 0, SEEK_SET);
while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) {
tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
}
@ -902,6 +904,8 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
lseek(fd, 0, SEEK_SET);
while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) {
tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
}

22
src/mnode/inc/mnodeDef.h
View File

@ -32,8 +32,8 @@ struct SMnodeObj;
typedef struct SDnodeObj {
int32_t dnodeId;
uint16_t dnodePort;
char dnodeFqdn[TSDB_FQDN_LEN + 1];
char dnodeEp[TSDB_EP_LEN + 1];
char dnodeFqdn[TSDB_FQDN_LEN];
char dnodeEp[TSDB_EP_LEN];
int64_t createdTime;
uint32_t lastAccess;
int32_t openVnodes;
@ -115,7 +115,7 @@ typedef struct {
typedef struct SVgObj {
uint32_t vgId;
char dbName[TSDB_DB_NAME_LEN + 1];
char dbName[TSDB_DB_NAME_LEN];
int64_t createdTime;
SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
int32_t numOfVnodes;
@ -154,8 +154,8 @@ typedef struct {
} SDbCfg;
typedef struct SDbObj {
char name[TSDB_DB_NAME_LEN + 1];
char acct[TSDB_USER_LEN + 1];
char name[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
char acct[TSDB_USER_LEN];
int64_t createdTime;
int32_t cfgVersion;
SDbCfg cfg;
@ -172,9 +172,9 @@ typedef struct SDbObj {
} SDbObj;
typedef struct SUserObj {
char user[TSDB_USER_LEN + 1];
char pass[TSDB_KEY_LEN + 1];
char acct[TSDB_USER_LEN + 1];
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
char acct[TSDB_USER_LEN];
int64_t createdTime;
int8_t superAuth;
int8_t writeAuth;
@ -203,8 +203,8 @@ typedef struct {
} SAcctInfo;
typedef struct SAcctObj {
char user[TSDB_USER_LEN + 1];
char pass[TSDB_KEY_LEN + 1];
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
SAcctCfg cfg;
int32_t acctId;
int64_t createdTime;
@ -219,7 +219,7 @@ typedef struct SAcctObj {
typedef struct {
int8_t type;
int32_t index;
char db[TSDB_DB_NAME_LEN + 1];
char db[TSDB_DB_NAME_LEN];
void * pIter;
int16_t numOfColumns;
int32_t rowSize;

2
src/mnode/inc/mnodeProfile.h
View File

@ -22,7 +22,7 @@ extern "C" {
#include "mnodeDef.h"
typedef struct {
char user[TSDB_USER_LEN + 1];
char user[TSDB_USER_LEN];
int8_t killed;
uint16_t port;
uint32_t ip;

21
src/mnode/src/mnodeDb.c
View File

@ -84,9 +84,12 @@ static int32_t mnodeDbActionDelete(SSdbOper *pOper) {
mnodeDropAllChildTables(pDb);
mnodeDropAllSuperTables(pDb);
mnodeDropAllDbVgroups(pDb);
mnodeDropDbFromAcct(pAcct, pDb);
mnodeDecAcctRef(pAcct);
if (pAcct) {
mnodeDropDbFromAcct(pAcct, pDb);
mnodeDecAcctRef(pAcct);
}
return TSDB_CODE_SUCCESS;
}
@ -328,8 +331,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
if (code != 0) return code;
pDb = calloc(1, sizeof(SDbObj));
strncpy(pDb->name, pCreate->db, TSDB_DB_NAME_LEN);
strncpy(pDb->acct, pAcct->user, TSDB_USER_LEN);
tstrncpy(pDb->name, pCreate->db, sizeof(pDb->name));
tstrncpy(pDb->acct, pAcct->user, sizeof(pDb->acct));
pDb->createdTime = taosGetTimestampMs();
pDb->cfg = (SDbCfg) {
.cacheBlockSize = pCreate->cacheBlockSize,
@ -373,7 +376,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
}
bool mnodeCheckIsMonitorDB(char *db, char *monitordb) {
char dbName[TSDB_DB_NAME_LEN + 1] = {0};
char dbName[TSDB_DB_NAME_LEN] = {0};
extractDBName(db, dbName);
size_t len = strlen(dbName);
@ -453,7 +456,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;
pShow->bytes[cols] = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE;
pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
@ -610,7 +613,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
char* name = mnodeGetDbStr(pDb->name);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, TSDB_DB_NAME_LEN);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, TSDB_DB_NAME_LEN - 1);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -882,7 +885,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
}
if (pDb->cfg.replications - replications >= 2) {
mError("db:%s, replica number can't change from 3 to 1", pDb->name, replications);
mError("db:%s, replica number can't change from %d to %d", pDb->name, pDb->cfg.replications, replications);
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
}
}

8
src/mnode/src/mnodeDnode.c
View File

@ -90,7 +90,7 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
SDnodeObj *pSaved = mnodeGetDnode(pDnode->dnodeId);
if (pDnode != pSaved) {
if (pDnode != pSaved && pDnode != NULL && pSaved != NULL) {
memcpy(pSaved, pDnode, pOper->rowSize);
free(pDnode);
}
@ -237,7 +237,9 @@ void mnodeUpdateDnode(SDnodeObj *pDnode) {
.pObj = pDnode
};
sdbUpdateRow(&oper);
if (sdbUpdateRow(&oper) != 0) {
mError("dnodeId:%d, failed update", pDnode->dnodeId);
}
}
static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
@ -383,7 +385,7 @@ static int32_t mnodeCreateDnode(char *ep) {
pDnode->createdTime = taosGetTimestampMs();
pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->totalVnodes = TSDB_INVALID_VNODE_NUM;
strcpy(pDnode->dnodeEp, ep);
tstrncpy(pDnode->dnodeEp, ep, TSDB_EP_LEN);
taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort);
SSdbOper oper = {

8
src/mnode/src/mnodeMain.c
View File

@ -134,8 +134,12 @@ void mnodeStopSystem() {
}
mnodeCleanupSystem();
mPrint("mnode file is removed");
remove(tsMnodeDir);
if (remove(tsMnodeDir) != 0) {
mPrint("failed to remove mnode file, reason:%s", strerror(errno));
} else {
mPrint("mnode file is removed");
}
}
static void mnodeInitTimer() {

34
src/mnode/src/mnodeProfile.c
View File

@ -67,7 +67,7 @@ int32_t mnodeInitProfile() {
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg);
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_CONN, mnodeProcessKillConnectionMsg);
tsMnodeConnCache = taosCacheInitWithCb(tsMnodeTmr, CONN_CHECK_TIME, mnodeFreeConn);
tsMnodeConnCache = taosCacheInitWithCb(CONN_CHECK_TIME, mnodeFreeConn);
return 0;
}
@ -97,7 +97,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) {
.connId = connId,
.stime = taosGetTimestampMs()
};
strcpy(connObj.user, user);
tstrncpy(connObj.user, user, sizeof(connObj.user));
char key[10];
sprintf(key, "%u", connId);
@ -222,7 +222,7 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi
SConnObj *pConnObj = NULL;
int32_t cols = 0;
char * pWrite;
char ipStr[TSDB_IPv4ADDR_LEN + 7];
char ipStr[TSDB_IPv4ADDR_LEN + 6];
while (numOfRows < rows) {
pShow->pIter = mnodeGetNextConn(pShow->pIter, &pConnObj);
@ -235,12 +235,14 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, TSDB_USER_LEN);
size_t size = sizeof(pConnObj->user);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, size);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
snprintf(ipStr, TSDB_IPv4ADDR_LEN + 6, "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, TSDB_IPv4ADDR_LEN + 6);
snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
size = sizeof(ipStr);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, size);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -342,7 +344,7 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
SConnObj *pConnObj = NULL;
int32_t cols = 0;
char * pWrite;
char ipStr[TSDB_IPv4ADDR_LEN + 7];
char ipStr[TSDB_IPv4ADDR_LEN + 6];
while (numOfRows < rows) {
pShow->pIter = mnodeGetNextConn(pShow->pIter, &pConnObj);
@ -358,12 +360,14 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, TSDB_USER_LEN);
size_t size = sizeof(pConnObj->user);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, size);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
snprintf(ipStr, TSDB_IPv4ADDR_LEN + 6, "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, TSDB_IPv4ADDR_LEN + 6);
snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
size = sizeof(ipStr);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, size);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -463,7 +467,7 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v
SConnObj *pConnObj = NULL;
int32_t cols = 0;
char * pWrite;
char ipStr[TSDB_IPv4ADDR_LEN + 7];
char ipStr[TSDB_IPv4ADDR_LEN + 6];
while (numOfRows < rows) {
pShow->pIter = mnodeGetNextConn(pShow->pIter, &pConnObj);
@ -479,12 +483,14 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, TSDB_USER_LEN);
size_t size = sizeof(pConnObj->user);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, size);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
snprintf(ipStr, TSDB_IPv4ADDR_LEN + 6, "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, TSDB_IPv4ADDR_LEN + 6);
snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
size = sizeof(ipStr);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, size);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

16
src/mnode/src/mnodeSdb.c
View File

@ -30,6 +30,8 @@
#include "mnodeDnode.h"
#include "mnodeSdb.h"
#define SDB_TABLE_LEN 12
typedef enum {
SDB_ACTION_INSERT,
SDB_ACTION_DELETE,
@ -43,7 +45,7 @@ typedef enum {
} ESdbStatus;
typedef struct _SSdbTable {
char tableName[TSDB_DB_NAME_LEN + 1];
char tableName[SDB_TABLE_LEN];
ESdbTable tableId;
ESdbKey keyType;
int32_t hashSessions;
@ -174,10 +176,10 @@ static void sdbRestoreTables() {
totalRows += pTable->numOfRows;
numOfTables++;
sdbTrace("table:%s, is restored, numOfRows:%d", pTable->tableName, pTable->numOfRows);
sdbTrace("table:%s, is restored, numOfRows:%" PRId64, pTable->tableName, pTable->numOfRows);
}
sdbTrace("sdb is restored, version:%d totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables);
sdbTrace("sdb is restored, version:%" PRId64 " totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables);
}
void sdbUpdateMnodeRoles() {
@ -449,7 +451,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
pthread_mutex_unlock(&pTable->mutex);
sdbTrace("table:%s, insert record:%s to hash, rowSize:%d vnumOfRows:%d version:%" PRIu64, pTable->tableName,
sdbTrace("table:%s, insert record:%s to hash, rowSize:%d numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion());
(*pTable->insertFp)(pOper);
@ -473,7 +475,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
pTable->numOfRows--;
pthread_mutex_unlock(&pTable->mutex);
sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d version:%" PRIu64, pTable->tableName,
sdbTrace("table:%s, delete record:%s from hash, numOfRows:%" PRId64 "version:%" PRIu64, pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1;
@ -484,7 +486,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
}
static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
sdbTrace("table:%s, update record:%s in hash, numOfRows:%d version:%" PRIu64, pTable->tableName,
sdbTrace("table:%s, update record:%s in hash, numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
(*pTable->updateFp)(pOper);
@ -717,7 +719,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
if (pTable == NULL) return NULL;
strcpy(pTable->tableName, pDesc->tableName);
tstrncpy(pTable->tableName, pDesc->tableName, SDB_TABLE_LEN);
pTable->keyType = pDesc->keyType;
pTable->tableId = pDesc->tableId;
pTable->hashSessions = pDesc->hashSessions;

23
src/mnode/src/mnodeShow.c
View File

@ -65,7 +65,7 @@ int32_t mnodeInitShow() {
mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mnodeProcessConnectMsg);
mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mnodeProcessUseMsg);
tsMnodeShowCache = taosCacheInitWithCb(tsMnodeTmr, 10, mnodeFreeShowObj);
tsMnodeShowCache = taosCacheInitWithCb(10, mnodeFreeShowObj);
return 0;
}
@ -138,15 +138,16 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
}
pShowRsp->qhandle = htobe64((uint64_t) pShow);
mTrace("%p, show type:%s, start to get meta", pShow, mnodeGetShowType(pShowMsg->type));
int32_t code = (*tsMnodeShowMetaFp[pShowMsg->type])(&pShowRsp->tableMeta, pShow, pMsg->rpcMsg.handle);
if (code == 0) {
mTrace("%p, show type:%s index:%d, get meta finished, rows:%d cols:%d result:%s", pShow,
mnodeGetShowType(pShowMsg->type), pShow->index, pShow->numOfRows, pShow->numOfColumns, tstrerror(code));
if (code == TSDB_CODE_SUCCESS) {
pMsg->rpcRsp.rsp = pShowRsp;
pMsg->rpcRsp.len = sizeof(SCMShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
mnodeReleaseShowObj(pShow, false);
return TSDB_CODE_SUCCESS;
} else {
mError("%p, show type:%s, failed to get meta, reason:%s", pShow, mnodeGetShowType(pShowMsg->type), tstrerror(code));
rpcFreeCont(pShowRsp);
mnodeReleaseShowObj(pShow, true);
return code;
@ -161,8 +162,7 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
SShowObj *pShow = (SShowObj *)pRetrieve->qhandle;
mTrace("%p, show type:%s, retrieve data", pShow, mnodeGetShowType(pShow->type));
/*
* in case of server restart, apps may hold qhandle created by server before
* restart, which is actually invalid, therefore, signature check is required.
@ -171,7 +171,10 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
mError("%p, show is invalid", pShow);
return TSDB_CODE_MND_INVALID_SHOWOBJ;
}
mTrace("%p, show type:%s index:%d, start retrieve data, numOfReads:%d numOfRows:%d", pShow,
mnodeGetShowType(pShow->type), pShow->index, pShow->numOfReads, pShow->numOfRows);
if (mnodeCheckShowFinished(pShow)) {
mTrace("%p, show is already read finished, numOfReads:%d numOfRows:%d", pShow, pShow->numOfReads, pShow->numOfRows);
pShow->numOfReads = pShow->numOfRows;
@ -198,6 +201,9 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE)
rowsRead = (*tsMnodeShowRetrieveFp[pShow->type])(pShow, pRsp->data, rowsToRead, pMsg->rpcMsg.handle);
mTrace("%p, show type:%s index:%d, stop retrieve data, rowsRead:%d rowsToRead:%d", pShow,
mnodeGetShowType(pShow->type), pShow->index, rowsRead, rowsToRead);
if (rowsRead < 0) {
rpcFreeCont(pRsp);
mnodeReleaseShowObj(pShow, false);
@ -211,7 +217,7 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
pMsg->rpcRsp.rsp = pRsp;
pMsg->rpcRsp.len = size;
if (rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows - pShow->numOfReads == rowsToRead)) {
if (rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) {
pRsp->completed = 1;
mnodeReleaseShowObj(pShow, true);
} else {
@ -324,6 +330,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
connect_over:
if (code != TSDB_CODE_SUCCESS) {
rpcFreeCont(pConnectRsp);
mLError("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code));
} else {
mLPrint("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code));

77
src/mnode/src/mnodeTable.c
View File

@ -157,10 +157,12 @@ static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) {
if (pDb != NULL) pAcct = mnodeGetAcct(pDb->acct);
if (pTable->info.type == TSDB_CHILD_TABLE) {
grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->superTable->numOfColumns - 1);
mnodeRemoveTableFromStable(pTable->superTable, pTable);
mnodeDecTableRef(pTable->superTable);
if (pTable->superTable) {
grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->superTable->numOfColumns - 1);
mnodeRemoveTableFromStable(pTable->superTable, pTable);
mnodeDecTableRef(pTable->superTable);
}
} else {
grantRestore(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->numOfColumns - 1);
@ -201,7 +203,7 @@ static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) {
assert(pTable != NULL && pOper->rowData != NULL);
int32_t len = strlen(pTable->info.tableId);
if (len > TSDB_TABLE_ID_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID;
if (len >= TSDB_TABLE_ID_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID;
memcpy(pOper->rowData, pTable->info.tableId, len);
memset(pOper->rowData + len, 0, 1);
@ -232,7 +234,7 @@ static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) {
if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
int32_t len = strlen(pOper->rowData);
if (len > TSDB_TABLE_ID_LEN) {
if (len >= TSDB_TABLE_ID_LEN) {
free(pTable);
return TSDB_CODE_MND_INVALID_TABLE_ID;
}
@ -453,7 +455,7 @@ static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) {
assert(pOper->pObj != NULL && pOper->rowData != NULL);
int32_t len = strlen(pStable->info.tableId);
if (len > TSDB_TABLE_ID_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID;
if (len >= TSDB_TABLE_ID_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID;
memcpy(pOper->rowData, pStable->info.tableId, len);
memset(pOper->rowData + len, 0, 1);
@ -477,7 +479,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbOper *pOper) {
if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
int32_t len = strlen(pOper->rowData);
if (len > TSDB_TABLE_ID_LEN){
if (len >= TSDB_TABLE_ID_LEN){
free(pStable);
return TSDB_CODE_MND_INVALID_TABLE_ID;
}
@ -951,7 +953,7 @@ static int32_t mnodeModifySuperTableTagName(SSuperTableObj *pStable, char *oldTa
// update
SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col);
strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN);
tstrncpy(schema->name, newTagName, sizeof(schema->name));
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
@ -1004,7 +1006,7 @@ static int32_t mnodeAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SS
sizeof(SSchema) * pStable->numOfTags);
memcpy(pStable->schema + pStable->numOfColumns, schema, sizeof(SSchema) * ncols);
SSchema *tschema = (SSchema *) (pStable->schema + sizeof(SSchema) * pStable->numOfColumns);
SSchema *tschema = (SSchema *) (pStable->schema + pStable->numOfColumns);
for (int32_t i = 0; i < ncols; i++) {
tschema[i].colId = pStable->nextColId++;
}
@ -1078,8 +1080,9 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow,
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
SSchema tbnameSchema = tGetTableNameColumnSchema();
pShow->bytes[cols] = tbnameSchema.bytes;
pSchema[cols].type = tbnameSchema.type;
strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -1138,7 +1141,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
prefixLen = strlen(prefix);
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
char stableName[TSDB_TABLE_NAME_LEN + 1] = {0};
char stableName[TSDB_TABLE_NAME_LEN] = {0};
while (numOfRows < rows) {
pShow->pIter = mnodeGetNextSuperTable(pShow->pIter, &pTable);
@ -1151,7 +1154,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
memset(stableName, 0, tListLen(stableName));
mnodeExtractTableName(pTable->info.tableId, stableName);
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, stableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) {
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, stableName, sizeof(stableName) - 1, &info) != TSDB_PATTERN_MATCH) {
mnodeDecTableRef(pTable);
continue;
}
@ -1160,7 +1163,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
int16_t len = strnlen(stableName, TSDB_DB_NAME_LEN);
int16_t len = strnlen(stableName, TSDB_DB_NAME_LEN - 1);
*(int16_t*) pWrite = len;
pWrite += sizeof(int16_t); // todo refactor
@ -1228,7 +1231,7 @@ static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pT
assert(numOfCols <= TSDB_MAX_COLUMNS);
for (int32_t i = 0; i < numOfCols; ++i) {
strncpy(pSchema->name, pTable->schema[i].name, TSDB_COL_NAME_LEN);
tstrncpy(pSchema->name, pTable->schema[i].name, sizeof(pSchema->name));
pSchema->type = pTable->schema[i].type;
pSchema->bytes = htons(pTable->schema[i].bytes);
pSchema->colId = htons(pTable->schema[i].colId);
@ -1249,12 +1252,12 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) {
pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns);
pMeta->tableType = pTable->info.type;
pMeta->contLen = sizeof(STableMetaMsg) + mnodeSetSchemaFromSuperTable(pMeta->schema, pTable);
strncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN);
tstrncpy(pMeta->tableId, pTable->info.tableId, sizeof(pMeta->tableId));
pMsg->rpcRsp.len = pMeta->contLen;
pMeta->contLen = htons(pMeta->contLen);
pMsg->rpcRsp.rsp = pMeta;
pMsg->rpcRsp.len = pMeta->contLen;
mTrace("stable:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid);
return TSDB_CODE_SUCCESS;
@ -1769,8 +1772,9 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
strncpy(pCreateMsg->tableId, pInfo->tableId, tListLen(pInfo->tableId));
strcpy(pCreateMsg->db, pMsg->pDb->name);
size_t size = sizeof(pInfo->tableId);
tstrncpy(pCreateMsg->tableId, pInfo->tableId, size);
tstrncpy(pCreateMsg->db, pMsg->pDb->name, sizeof(pCreateMsg->db));
pCreateMsg->igExists = 1;
pCreateMsg->getMeta = 1;
pCreateMsg->contLen = htonl(contLen);
@ -2032,7 +2036,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
pMultiMeta->numOfTables = 0;
for (int32_t t = 0; t < pInfo->numOfTables; ++t) {
char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN + 1);
char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN);
SChildTableObj *pTable = mnodeGetChildTable(tableId);
if (pTable == NULL) continue;
@ -2079,8 +2083,9 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
SSchema s = tGetTableNameColumnSchema();
pShow->bytes[cols] = s.bytes;
pSchema[cols].type = s.type;
strcpy(pSchema[cols].name, "table_name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -2097,8 +2102,9 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
SSchema tbCol = tGetTableNameColumnSchema();
pShow->bytes[cols] = tbCol.bytes;
pSchema[cols].type = tbCol.type;
strcpy(pSchema[cols].name, "stable_name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -2141,12 +2147,12 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
continue;
}
char tableName[TSDB_TABLE_NAME_LEN + 1] = {0};
char tableName[TSDB_TABLE_NAME_LEN] = {0};
// pattern compare for table name
mnodeExtractTableName(pTable->info.tableId, tableName);
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) {
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, sizeof(tableName) - 1, &info) != TSDB_PATTERN_MATCH) {
mnodeDecTableRef(pTable);
continue;
}
@ -2155,7 +2161,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, TSDB_TABLE_NAME_LEN);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, sizeof(tableName) - 1);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -2173,10 +2179,10 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
memset(tableName, 0, tListLen(tableName));
memset(tableName, 0, sizeof(tableName));
if (pTable->info.type == TSDB_CHILD_TABLE) {
mnodeExtractTableName(pTable->superTable->info.tableId, tableName);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, TSDB_TABLE_NAME_LEN);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, sizeof(tableName) - 1);
}
cols++;
@ -2268,8 +2274,9 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
SSchema tbnameColSchema = tGetTableNameColumnSchema();
pShow->bytes[cols] = tbnameColSchema.bytes;
pSchema[cols].type = tbnameColSchema.type;
strcpy(pSchema[cols].name, "table_name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -2331,12 +2338,12 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
continue;
}
char tableName[TSDB_TABLE_NAME_LEN + 1] = {0};
char tableName[TSDB_TABLE_NAME_LEN] = {0};
// pattern compare for table name
mnodeExtractTableName(pTable->info.tableId, tableName);
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) {
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, sizeof(tableName) - 1, &info) != TSDB_PATTERN_MATCH) {
mnodeDecTableRef(pTable);
continue;
}
@ -2345,7 +2352,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, TSDB_TABLE_NAME_LEN);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, sizeof(tableName) - 1);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

5
src/mnode/src/mnodeUser.c
View File

@ -315,7 +315,8 @@ static int32_t mnodeRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, voi
cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->user, TSDB_USER_LEN);
size_t size = sizeof(pUser->user);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->user, size);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -336,7 +337,7 @@ static int32_t mnodeRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, voi
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->acct, TSDB_USER_LEN);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->acct, sizeof(pUser->user));
cols++;
numOfRows++;

2
src/os/darwin/src/darwinPlatform.c
View File

@ -189,7 +189,7 @@ void taosGetSystemLocale() {
if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
char *locale = setlocale(LC_CTYPE, "chs");
if (locale != NULL) {
strncpy(tsLocale, locale, sizeof(tsLocale) / sizeof(tsLocale[0]));
tstrncpy(tsLocale, locale, sizeof(tsLocale));
cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
uPrint("locale not configured, set to default:%s", tsLocale);
}

39
src/os/linux/src/linuxSysPara.c
View File

@ -160,7 +160,7 @@ static void taosGetSystemTimezone() {
/* load time zone string from /etc/timezone */
FILE *f = fopen("/etc/timezone", "r");
char buf[64] = {0};
char buf[65] = {0};
if (f != NULL) {
fread(buf, 64, 1, f);
fclose(f);
@ -229,7 +229,7 @@ static void taosGetSystemLocale() { // get and set default locale
uError("can't get locale from system, set it to en_US.UTF-8");
strcpy(tsLocale, "en_US.UTF-8");
} else {
tstrncpy(tsLocale, locale, tListLen(tsLocale));
tstrncpy(tsLocale, locale, sizeof(tsLocale));
uError("locale not configured, set to system default:%s", tsLocale);
}
}
@ -242,7 +242,7 @@ static void taosGetSystemLocale() { // get and set default locale
str++;
char *revisedCharset = taosCharsetReplace(str);
strncpy(tsCharset, revisedCharset, tListLen(tsCharset));
tstrncpy(tsCharset, revisedCharset, sizeof(tsCharset));
free(revisedCharset);
uWarn("charset not configured, set to system default:%s", tsCharset);
@ -332,33 +332,42 @@ bool taosGetDisk() {
}
static bool taosGetCardInfo(int64_t *bytes) {
*bytes = 0;
FILE *fp = fopen(tsSysNetFile, "r");
if (fp == NULL) {
uError("open file:%s failed", tsSysNetFile);
return false;
}
int64_t rbytes, rpackts, tbytes, tpackets;
int64_t nouse1, nouse2, nouse3, nouse4, nouse5, nouse6;
char nouse0[200] = {0};
size_t len;
char * line = NULL;
*bytes = 0;
size_t len = 2048;
char * line = calloc(1, len);
while (!feof(fp)) {
tfree(line);
len = 0;
memset(line, 0, len);
int64_t rbytes = 0;
int64_t rpackts = 0;
int64_t tbytes = 0;
int64_t tpackets = 0;
int64_t nouse1 = 0;
int64_t nouse2 = 0;
int64_t nouse3 = 0;
int64_t nouse4 = 0;
int64_t nouse5 = 0;
int64_t nouse6 = 0;
char nouse0[200] = {0};
getline(&line, &len, fp);
if (line == NULL) {
break;
}
line[len - 1] = 0;
if (strstr(line, "lo:") != NULL) {
continue;
}
sscanf(line,
"%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64,
"%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64
" %" PRId64,
nouse0, &rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &tbytes, &tpackets);
*bytes += (rbytes + tbytes);
}

2
src/os/windows/src/twindows.c
View File

@ -226,7 +226,7 @@ void taosGetSystemLocale() {
if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
char *locale = setlocale(LC_CTYPE, "chs");
if (locale != NULL) {
strncpy(tsLocale, locale, sizeof(tsLocale) / sizeof(tsLocale[0]));
tstrncpy(tsLocale, locale, sizeof(tsLocale));
cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
uPrint("locale not configured, set to default:%s", tsLocale);
}

4
src/plugins/http/inc/httpHandle.h
View File

@ -67,7 +67,7 @@
#define HTTP_COMPRESS_IDENTITY 0
#define HTTP_COMPRESS_GZIP 2
#define HTTP_SESSION_ID_LEN (TSDB_USER_LEN * 2 + 1)
#define HTTP_SESSION_ID_LEN (TSDB_USER_LEN + TSDB_PASSWORD_LEN)
typedef enum {
HTTP_CONTEXT_STATE_READY,
@ -84,7 +84,7 @@ typedef struct {
int expire;
int access;
void *taos;
char id[HTTP_SESSION_ID_LEN + 1];
char id[HTTP_SESSION_ID_LEN];
} HttpSession;
typedef enum {

4
src/plugins/http/src/gcHandle.c
View File

@ -48,7 +48,7 @@ void gcInitHandle(HttpServer* pServer) { httpAddMethod(pServer, &gcDecodeMethod)
bool gcGetUserFromUrl(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
if (pParser->path[GC_USER_URL_POS].len > TSDB_USER_LEN - 1 || pParser->path[GC_USER_URL_POS].len <= 0) {
if (pParser->path[GC_USER_URL_POS].len >= TSDB_USER_LEN || pParser->path[GC_USER_URL_POS].len <= 0) {
return false;
}
@ -58,7 +58,7 @@ bool gcGetUserFromUrl(HttpContext* pContext) {
bool gcGetPassFromUrl(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
if (pParser->path[GC_PASS_URL_POS].len > TSDB_PASSWORD_LEN - 1 || pParser->path[GC_PASS_URL_POS].len <= 0) {
if (pParser->path[GC_PASS_URL_POS].len >= TSDB_PASSWORD_LEN || pParser->path[GC_PASS_URL_POS].len <= 0) {
return false;
}

15
src/plugins/http/src/httpAuth.c
View File

@ -88,8 +88,8 @@ bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len) {
free(base64);
return false;
} else {
strncpy(pContext->user, descrypt, TSDB_USER_LEN);
strncpy(pContext->pass, descrypt + TSDB_USER_LEN, TSDB_PASSWORD_LEN);
tstrncpy(pContext->user, descrypt, sizeof(pContext->user));
tstrncpy(pContext->pass, descrypt + TSDB_USER_LEN, sizeof(pContext->pass));
httpTrace("context:%p, fd:%d, ip:%s, taosd token:%s parsed success, user:%s", pContext, pContext->fd,
pContext->ipstr, token, pContext->user);
@ -100,14 +100,17 @@ bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len) {
}
bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen) {
char buffer[TSDB_USER_LEN + TSDB_PASSWORD_LEN] = {0};
strncpy(buffer, pContext->user, TSDB_USER_LEN);
strncpy(buffer + TSDB_USER_LEN, pContext->pass, TSDB_PASSWORD_LEN);
char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0};
size_t size = sizeof(pContext->user);
tstrncpy(buffer, pContext->user, size);
size = sizeof(pContext->pass);
tstrncpy(buffer + sizeof(pContext->user), pContext->pass, size);
char *encrypt = taosDesEncode(KEY_DES_4, buffer, TSDB_USER_LEN + TSDB_PASSWORD_LEN);
char *base64 = base64_encode((const unsigned char *)encrypt, TSDB_USER_LEN + TSDB_PASSWORD_LEN);
strncpy(token, base64, (size_t)strlen(base64));
size_t len = strlen(base64);
tstrncpy(token, base64, len + 1);
free(encrypt);
free(base64);

2
src/plugins/http/src/httpSql.c
View File

@ -87,7 +87,7 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) {
}
if (code < 0) {
if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, -code)) {
if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code)) {
singleCmd->code = code;
httpTrace("context:%p, fd:%d, ip:%s, user:%s, process pos jump to:%d, last code:%s, last sql:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos + 1, tstrerror(code), sql);

4
src/plugins/http/src/httpUtil.c
View File

@ -307,7 +307,7 @@ void httpTrimTableName(char *name) {
for (int i = 0; name[i] != 0; i++) {
if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'')
name[i] = '_';
if (i == TSDB_TABLE_NAME_LEN + 1) {
if (i == TSDB_TABLE_NAME_LEN) {
name[i] = 0;
break;
}
@ -323,7 +323,7 @@ int httpShrinkTableName(HttpContext *pContext, int pos, char *name) {
len++;
}
if (len < TSDB_TABLE_NAME_LEN) {
if (len < TSDB_TABLE_NAME_LEN - 1) {
return pos;
}

4
src/plugins/http/src/restHandle.c
View File

@ -61,7 +61,7 @@ void restInitHandle(HttpServer* pServer) {
bool restGetUserFromUrl(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
if (pParser->path[REST_USER_URL_POS].len > TSDB_USER_LEN - 1 || pParser->path[REST_USER_URL_POS].len <= 0) {
if (pParser->path[REST_USER_URL_POS].len >= TSDB_USER_LEN || pParser->path[REST_USER_URL_POS].len <= 0) {
return false;
}
@ -71,7 +71,7 @@ bool restGetUserFromUrl(HttpContext* pContext) {
bool restGetPassFromUrl(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
if (pParser->path[REST_PASS_URL_POS].len > TSDB_PASSWORD_LEN - 1 || pParser->path[REST_PASS_URL_POS].len <= 0) {
if (pParser->path[REST_PASS_URL_POS].len >= TSDB_PASSWORD_LEN || pParser->path[REST_PASS_URL_POS].len <= 0) {
return false;
}

14
src/plugins/http/src/tgHandle.c
View File

@ -209,7 +209,7 @@ void tgParseSchemaMetric(cJSON *metric) {
goto ParseEnd;
}
int nameLen = (int)strlen(field->valuestring);
if (nameLen == 0 || nameLen > TSDB_TABLE_NAME_LEN) {
if (nameLen == 0 || nameLen >= TSDB_TABLE_NAME_LEN) {
parsedOk = false;
goto ParseEnd;
}
@ -306,21 +306,21 @@ void tgCleanupHandle() {
bool tgGetUserFromUrl(HttpContext *pContext) {
HttpParser *pParser = &pContext->parser;
if (pParser->path[TG_USER_URL_POS].len > TSDB_USER_LEN - 1 || pParser->path[TG_USER_URL_POS].len <= 0) {
if (pParser->path[TG_USER_URL_POS].len >= TSDB_USER_LEN || pParser->path[TG_USER_URL_POS].len <= 0) {
return false;
}
tstrncpy(pContext->user, pParser->path[TG_USER_URL_POS].pos, TSDB_USER_LEN);
tstrncpy(pContext->user, pParser->path[TG_USER_URL_POS].pos, sizeof(pContext->user));
return true;
}
bool tgGetPassFromUrl(HttpContext *pContext) {
HttpParser *pParser = &pContext->parser;
if (pParser->path[TG_PASS_URL_POS].len > TSDB_PASSWORD_LEN - 1 || pParser->path[TG_PASS_URL_POS].len <= 0) {
if (pParser->path[TG_PASS_URL_POS].len >= TSDB_PASSWORD_LEN || pParser->path[TG_PASS_URL_POS].len <= 0) {
return false;
}
tstrncpy(pContext->pass, pParser->path[TG_PASS_URL_POS].pos, TSDB_PASSWORD_LEN);
tstrncpy(pContext->pass, pParser->path[TG_PASS_URL_POS].pos, sizeof(pContext->pass));
return true;
}
@ -409,7 +409,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_NULL);
return false;
}
if (nameLen >= TSDB_TABLE_NAME_LEN - 7) {
if (nameLen >= TSDB_TABLE_NAME_LEN - 8) {
httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_LONG);
return false;
}
@ -498,7 +498,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
return false;
}
if (strlen(host->valuestring) >= TSDB_TABLE_NAME_LEN) {
if (strlen(host->valuestring) >= TSDB_TABLE_NAME_LEN - 1) {
httpSendErrorResp(pContext, HTTP_TG_TABLE_SIZE);
return false;
}

6
src/plugins/monitor/src/monitorMain.c
View File

@ -156,7 +156,7 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
", io_read float, io_write float"
", req_http int, req_select int, req_insert int"
") tags (dnodeid int, fqdn binary(%d))",
tsMonitorDbName, TSDB_FQDN_LEN + 1);
tsMonitorDbName, TSDB_FQDN_LEN);
} else if (cmd == MONITOR_CMD_CREATE_TB_DN) {
snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn%d using %s.dn tags(%d, '%s')", tsMonitorDbName,
dnodeGetDnodeId(), tsMonitorDbName, dnodeGetDnodeId(), tsLocalEp);
@ -175,7 +175,7 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
", totalConns smallint, maxConns smallint"
", accessState smallint"
") tags (acctId binary(%d))",
tsMonitorDbName, TSDB_USER_LEN + 1);
tsMonitorDbName, TSDB_USER_LEN);
} else if (cmd == MONITOR_CMD_CREATE_TB_ACCT_ROOT) {
snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct_%s using %s.acct tags('%s')", tsMonitorDbName, "root",
tsMonitorDbName, "root");
@ -183,7 +183,7 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
snprintf(sql, SQL_LENGTH,
"create table if not exists %s.slowquery(ts timestamp, username "
"binary(%d), created_time timestamp, time bigint, sql binary(%d))",
tsMonitorDbName, TSDB_TABLE_ID_LEN, TSDB_SLOW_QUERY_SQL_LEN);
tsMonitorDbName, TSDB_TABLE_ID_LEN - 1, TSDB_SLOW_QUERY_SQL_LEN);
} else if (cmd == MONITOR_CMD_CREATE_TB_LOG) {
snprintf(sql, SQL_LENGTH,
"create table if not exists %s.log(ts timestamp, level tinyint, "

14
src/query/inc/qfill.h
View File

@ -50,7 +50,8 @@ typedef struct SFillInfo {
char * nextValues; // next row of data
char** pData; // original result data block involved in filling data
int32_t capacityInRows; // data buffer size in rows
int8_t slidingUnit; // sliding time unit
int8_t precision; // time resoluation
SFillColInfo* pFillCol; // column info for fill operations
} SFillInfo;
@ -61,12 +62,13 @@ typedef struct SPoint {
int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, char timeUnit, int16_t precision);
SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity,
int32_t numOfCols, int64_t slidingTime, int32_t fillType, SFillColInfo* pFillCol);
SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
SFillColInfo* pFillCol);
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
void taosDestoryFillInfo(SFillInfo *pFillInfo);
void* taosDestoryFillInfo(SFillInfo *pFillInfo);
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
@ -74,9 +76,7 @@ void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput)
void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput);
TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision);
int64_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows);
int64_t getFilledNumOfRes(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
int32_t taosNumOfRemainRows(SFillInfo *pFillInfo);

2
src/query/inc/tsqlfunction.h
View File

@ -112,7 +112,7 @@ enum {
#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0)
#define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY))
#define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0)
#define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0)
#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0)
typedef struct SArithmeticSupport {

410
src/query/src/qExecutor.c
View File

@ -12,8 +12,8 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "qfill.h"
#include "os.h"
#include "qfill.h"
#include "hash.h"
#include "hashfunc.h"
@ -354,7 +354,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin
int16_t bytes) {
SQuery *pQuery = pRuntimeEnv->pQuery;
int32_t *p1 = (int32_t *)taosHashGet(pWindowResInfo->hashList, pData, bytes);
int32_t *p1 = (int32_t *) taosHashGet(pWindowResInfo->hashList, pData, bytes);
if (p1 != NULL) {
pWindowResInfo->curIndex = *p1;
} else { // more than the capacity, reallocate the resources
@ -484,7 +484,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes
// set time window for current result
pWindowRes->window = *win;
setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes);
return TSDB_CODE_SUCCESS;
}
@ -685,14 +685,14 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow
SDataBlockInfo *pDataBlockInfo, TSKEY *primaryKeys,
__block_search_fn_t searchFn) {
SQuery *pQuery = pRuntimeEnv->pQuery;
// tumbling time window query, a special case of sliding time window query
if (pQuery->slidingTime == pQuery->intervalTime) {
// todo opt
}
getNextTimeWindow(pQuery, pNextWin);
// next time window is not in current block
if ((pNextWin->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
(pNextWin->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
@ -720,7 +720,7 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow
*/
if (QUERY_IS_ASC_QUERY(pQuery) && primaryKeys[startPos] > pNextWin->ekey) {
TSKEY next = primaryKeys[startPos];
pNextWin->ekey += ((next - pNextWin->ekey + pQuery->slidingTime - 1)/pQuery->slidingTime) * pQuery->slidingTime;
pNextWin->skey = pNextWin->ekey - pQuery->intervalTime + 1;
} else if ((!QUERY_IS_ASC_QUERY(pQuery)) && primaryKeys[startPos] < pNextWin->skey) {
@ -729,7 +729,7 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow
pNextWin->skey -= ((pNextWin->skey - next + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime;
pNextWin->ekey = pNextWin->skey + pQuery->intervalTime - 1;
}
return startPos;
}
@ -919,12 +919,25 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
int64_t v = -1;
// not assign result buffer yet, add new result buffer
switch(type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: v = GET_INT8_VAL(pData); break;
case TSDB_DATA_TYPE_SMALLINT: v = GET_INT16_VAL(pData); break;
case TSDB_DATA_TYPE_INT: v = GET_INT32_VAL(pData); break;
case TSDB_DATA_TYPE_BIGINT: v = GET_INT64_VAL(pData); break;
}
// assert(pRuntimeEnv->windowResInfo.hashList->size <= 2);
SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes);
if (pWindowRes == NULL) {
return -1;
}
// not assign result buffer yet, add new result buffer
pWindowRes->window.skey = v;
pWindowRes->window.ekey = v;
if (pWindowRes->pos.pageId == -1) {
int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage);
if (ret != 0) {
@ -1022,12 +1035,16 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
return false;
}
if (functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_LAST) {
return !QUERY_IS_ASC_QUERY(pQuery);
} else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST) {
if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST) {
return QUERY_IS_ASC_QUERY(pQuery);
}
// todo add comments
if ((functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_LAST)) {
return pCtx->param[0].i64Key == pQuery->order.order;
// return !QUERY_IS_ASC_QUERY(pQuery);
}
// in the supplementary scan, only the following functions need to be executed
if (IS_REVERSE_SCAN(pRuntimeEnv)) {
return false;
@ -1079,7 +1096,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
int32_t j = 0;
int32_t offset = -1;
for (j = 0; j < pDataBlockInfo->rows; ++j) {
offset = GET_COL_DATA_POS(pQuery, j, step);
@ -1368,8 +1385,10 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
int32_t index = pSqlFuncMsg->colInfo.colIndex;
if (TSDB_COL_IS_TAG(pIndex->flag)) {
if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor
pCtx->inputBytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
pCtx->inputType = TSDB_DATA_TYPE_BINARY;
SSchema s = tGetTableNameColumnSchema();
pCtx->inputBytes = s.bytes;
pCtx->inputType = s.type;
} else {
pCtx->inputBytes = pQuery->tagColList[index].bytes;
pCtx->inputType = pQuery->tagColList[index].type;
@ -1466,7 +1485,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
tfree(pRuntimeEnv->pCtx);
}
taosDestoryFillInfo(pRuntimeEnv->pFillInfo);
pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo);
destroyResultBuf(pRuntimeEnv->pResultBuf, pQInfo);
tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
@ -1477,19 +1496,6 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
static bool isQueryKilled(SQInfo *pQInfo) {
return (pQInfo->code == TSDB_CODE_TSC_QUERY_CANCELLED);
#if 0
/*
* check if the queried meter is going to be deleted.
* if it will be deleted soon, stop current query ASAP.
*/
SMeterObj *pMeterObj = pQInfo->pObj;
if (vnodeIsMeterState(pMeterObj, TSDB_METER_STATE_DROPPING)) {
pQInfo->killed = 1;
return true;
}
return (pQInfo->killed == 1);
#endif
}
static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED; }
@ -1572,10 +1578,14 @@ static bool needReverseScan(SQuery *pQuery) {
continue;
}
if (((functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) && QUERY_IS_ASC_QUERY(pQuery)) ||
((functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_FIRST_DST) && !QUERY_IS_ASC_QUERY(pQuery))) {
if ((functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_FIRST_DST) && !QUERY_IS_ASC_QUERY(pQuery)) {
return true;
}
if (functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) {
int32_t order = pQuery->pSelectExpr[i].base.arg->argValue.i64;
return order != pQuery->order.order;
}
}
return false;
@ -2028,6 +2038,34 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) {
return midPos;
}
static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capacity) {
SQuery* pQuery = pRuntimeEnv->pQuery;
if (capacity < pQuery->rec.capacity) {
return;
}
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
int32_t bytes = pQuery->pSelectExpr[i].bytes;
assert(bytes > 0 && capacity > 0);
char *tmp = realloc(pQuery->sdata[i], bytes * capacity + sizeof(tFilePage));
if (tmp == NULL) { // todo handle the oom
assert(0);
} else {
pQuery->sdata[i] = (tFilePage *)tmp;
}
// set the pCtx output buffer position
pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data;
}
qTrace("QInfo:%p realloc output buffer to inc output buffer from: %d rows to:%d rows", GET_QINFO_ADDR(pRuntimeEnv),
pQuery->rec.capacity, capacity);
pQuery->rec.capacity = capacity;
}
static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) {
// in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block
SQuery* pQuery = pRuntimeEnv->pQuery;
@ -2070,7 +2108,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
SQuery *pQuery = pRuntimeEnv->pQuery;
STableQueryInfo* pTableQueryInfo = pQuery->current;
SQueryCostInfo* summary = &pRuntimeEnv->summary;
qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d",
GET_QINFO_ADDR(pRuntimeEnv), pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, pTableQueryInfo->lastKey,
pQuery->order.order);
@ -2111,7 +2149,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
SDataStatis *pStatis = NULL;
SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
// query start position can not move into tableApplyFunctionsOnBlock due to limit/offset condition
pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : blockInfo.rows - 1;
int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock);
@ -2500,7 +2538,7 @@ int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) {
SResultInfo *pResultInfo = &pWindowRes->resultInfo[j];
assert(pResultInfo != NULL);
if (pResultInfo->numOfRes > 0) {
return pResultInfo->numOfRes;
}
@ -2549,7 +2587,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
assert(pQInfo->numOfGroupResultPages == 0);
return 0;
} else if (numOfTables == 1) { // no need to merge results since only one table in each group
}
SCompSupporter cs = {pTableList, posList, pQInfo};
@ -2638,7 +2676,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
#endif
qTrace("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt);
tfree(pTableList);
tfree(posList);
tfree(pTree);
@ -2868,12 +2906,12 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) {
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
int32_t functionId = pQuery->pSelectExpr[j].base.functionId;
pRuntimeEnv->pCtx[j].currentStage = 0;
SResultInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
if (pResInfo->initialized) {
continue;
}
aAggs[functionId].init(&pRuntimeEnv->pCtx[j]);
}
}
@ -2914,8 +2952,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) {
pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
}
}
updateNumOfResult(pRuntimeEnv, pQuery->rec.rows);
}
}
@ -3052,7 +3089,7 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus
pQuery->window = pTableQueryInfo->win;
}
void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv);
SQuery *pQuery = pRuntimeEnv->pQuery;
STableQueryInfo *pTableQueryInfo = pQuery->current;
@ -3246,7 +3283,7 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult);
int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
@ -3266,7 +3303,7 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult
void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) {
SQuery *pQuery = pRuntimeEnv->pQuery;
// Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
@ -3275,21 +3312,21 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
if (pCtx->resultInfo->complete) {
continue;
}
pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult);
pCtx->currentStage = 0;
int32_t functionId = pCtx->functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
}
/*
* set the output buffer information and intermediate buffer
* not all queries require the interResultBuf, such as COUNT
*/
pCtx->resultInfo->superTableQ = pRuntimeEnv->stableQuery; // set super table query flag
if (!pCtx->resultInfo->initialized) {
aAggs[functionId].init(pCtx);
}
@ -3494,18 +3531,32 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) {
assert(pQuery->rec.rows <= pQuery->rec.capacity);
}
static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) {
static UNUSED_FUNC void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) {
SQuery *pQuery = pRuntimeEnv->pQuery;
// update the number of result for each, only update the number of rows for the corresponding window result.
if (pQuery->intervalTime == 0) {
int32_t g = pTableQueryInfo->groupIndex;
assert(pRuntimeEnv->windowResInfo.size > 0);
SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&g, sizeof(g));
if (pWindowRes->numOfRows == 0) {
pWindowRes->numOfRows = getNumOfResult(pRuntimeEnv);
for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) {
SWindowResult *pResult = &pRuntimeEnv->windowResInfo.pResult[i];
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
int32_t functionId = pRuntimeEnv->pCtx[j].functionId;
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) {
continue;
}
pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes);
}
}
// int32_t g = pTableQueryInfo->groupIndex;
// assert(pRuntimeEnv->windowResInfo.size > 0);
//
// SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&g, sizeof(g));
// if (pWindowRes->numOfRows == 0) {
// pWindowRes->numOfRows = getNumOfResult(pRuntimeEnv);
// }
}
}
@ -3517,7 +3568,7 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *
SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo;
pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1;
if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) {
if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock);
} else {
blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock);
@ -3557,9 +3608,7 @@ bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) {
* first result row in the actual result set will fill nothing.
*/
if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime,
pQuery->slidingTimeUnit, pQuery->precision);
int32_t numOfTotal = taosGetNumOfResultWithFill(pFillInfo, remain, ekey, pQuery->rec.capacity);
int32_t numOfTotal = getFilledNumOfRes(pFillInfo, pQuery->window.ekey, pQuery->rec.capacity);
return numOfTotal > 0;
}
@ -3601,7 +3650,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
}
}
int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t numOfRows, int32_t *numOfInterpo) {
int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t *numOfInterpo) {
SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
SQuery *pQuery = pRuntimeEnv->pQuery;
SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo;
@ -4013,7 +4062,8 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool
if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) {
SFillColInfo* pColInfo = taosCreateFillColInfo(pQuery);
pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, 0, 0, pQuery->rec.capacity, pQuery->numOfOutput,
pQuery->slidingTime, pQuery->fillType, pColInfo);
pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision,
pQuery->fillType, pColInfo);
}
// todo refactor
@ -4080,21 +4130,22 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
SDataStatis *pStatis = NULL;
SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
if (!isIntervalQuery(pQuery)) {
int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1;
setExecutionContext(pQInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIndex, blockInfo.window.ekey + step);
} else { // interval query
TSKEY nextKey = blockInfo.window.skey;
setIntervalQueryRange(pQInfo, nextKey);
/*int32_t ret = */setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo);
if (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
if (!isIntervalQuery(pQuery)) {
int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1;
setExecutionContext(pQInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIndex, blockInfo.window.ekey + step);
} else { // interval query
TSKEY nextKey = blockInfo.window.skey;
setIntervalQueryRange(pQInfo, nextKey);
/*int32_t ret = */setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo);
}
}
summary->totalRows += blockInfo.rows;
stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey);
qTrace("QInfo:%p check data block, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, lastKey:%" PRId64,
GET_QINFO_ADDR(pRuntimeEnv), blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey,
blockInfo.rows, pQuery->current->lastKey);
pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, pQuery->current->lastKey);
}
int64_t et = taosGetTimestampMs();
@ -4219,7 +4270,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
// here we simply set the first table as current table
pQuery->current = ((SGroupItem*) taosArrayGet(group, 0))->info;
scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
int64_t numOfRes = getNumOfResult(pRuntimeEnv);
if (numOfRes > 0) {
@ -4232,10 +4283,84 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
// enable execution for next table, when handling the projection query
enableExecutionForNextTable(pRuntimeEnv);
if (pQuery->rec.rows >= pQuery->rec.capacity) {
setQueryStatus(pQuery, QUERY_RESBUF_FULL);
break;
}
}
} else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group-by on normal columns query
while (pQInfo->groupIndex < numOfGroups) {
SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
qTrace("QInfo:%p group by normal columns group:%d, total group:%d", pQInfo, pQInfo->groupIndex, numOfGroups);
STsdbQueryCond cond = {
.twindow = pQuery->window,
.colList = pQuery->colList,
.order = pQuery->order.order,
.numOfCols = pQuery->numOfCols,
};
SArray *g1 = taosArrayInit(1, POINTER_BYTES);
SArray *tx = taosArrayClone(group);
taosArrayPush(g1, &tx);
STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1};
// include only current table
if (pRuntimeEnv->pQueryHandle != NULL) {
tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
pRuntimeEnv->pQueryHandle = NULL;
}
pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
SArray* s = tsdbGetQueriedTableIdList(pRuntimeEnv->pQueryHandle);
assert(taosArrayGetSize(s) >= 1);
setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(s, 0), pQInfo->tsdb);
// here we simply set the first table as current table
scanMultiTableDataBlocks(pQInfo);
pQInfo->groupIndex += 1;
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
// no results generated for current group, continue to try the next group
if (pWindowResInfo->size <= 0) {
continue;
}
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status;
pStatus->closed = true; // enable return all results for group by normal columns
SWindowResult *pResult = &pWindowResInfo->pResult[i];
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes);
}
}
qTrace("QInfo:%p generated groupby columns results %d rows for group %d completed", pQInfo, pWindowResInfo->size,
pQInfo->groupIndex);
int32_t currentGroupIndex = pQInfo->groupIndex;
pQuery->rec.rows = 0;
pQInfo->groupIndex = 0;
ensureOutputBufferSimple(pRuntimeEnv, pWindowResInfo->size);
copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult);
pQInfo->groupIndex = currentGroupIndex; //restore the group index
assert(pQuery->rec.rows == pWindowResInfo->size);
clearClosedTimeWindow(pRuntimeEnv);
break;
}
} else {
/*
* 1. super table projection query, 2. group-by on normal columns query, 3. ts-comp query
* 1. super table projection query, 2. ts-comp query
* if the subgroup index is larger than 0, results generated by group by tbname,k is existed.
* we need to return it to client in the first place.
*/
@ -4282,7 +4407,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
}
}
scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
skipResults(pRuntimeEnv);
// the limitation of output result is reached, set the query completed
@ -4348,25 +4473,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur;
}
// todo refactor
if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status;
pStatus->closed = true; // enable return all results for group by normal columns
SWindowResult *pResult = &pWindowResInfo->pResult[i];
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes);
}
}
pQInfo->groupIndex = 0;
pQuery->rec.rows = 0;
copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult);
}
qTrace(
"QInfo %p numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%"PRId64", offset:%" PRId64,
pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total,
@ -4448,7 +4554,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
*/
if (isIntervalQuery(pQuery)) {
copyResToQueryResultBuf(pQInfo, pQuery);
#ifdef _DEBUG_VIEW
displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num);
#endif
@ -4469,7 +4574,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
// query error occurred or query is killed, abort current execution
if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) {
qTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code);
qTrace("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code));
return;
}
@ -4490,7 +4595,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
setQueryStatus(pQuery, QUERY_COMPLETED);
if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) {
qTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code);
qTrace("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code));
return;
}
@ -4526,7 +4631,7 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
pQuery->current = pTableInfo; // set current query table info
scanAllDataBlocks(pRuntimeEnv, pTableInfo->lastKey);
scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey);
finalizeQueryResult(pRuntimeEnv);
if (isQueryKilled(pQInfo)) {
@ -4559,7 +4664,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
}
while (1) {
scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
finalizeQueryResult(pRuntimeEnv);
if (isQueryKilled(pQInfo)) {
@ -4606,7 +4711,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start)
SQuery *pQuery = pRuntimeEnv->pQuery;
while (1) {
scanAllDataBlocks(pRuntimeEnv, start);
scanOneTableDataBlocks(pRuntimeEnv, start);
if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) {
return;
@ -4666,13 +4771,11 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
limitResults(pRuntimeEnv);
break;
} else {
TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime,
pQuery->slidingTimeUnit, pQuery->precision);
taosFillSetStartInfo(pRuntimeEnv->pFillInfo, pQuery->rec.rows, ekey);
taosFillSetStartInfo(pRuntimeEnv->pFillInfo, pQuery->rec.rows, pQuery->window.ekey);
taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata);
numOfInterpo = 0;
pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, pQuery->rec.rows, &numOfInterpo);
pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfInterpo);
if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
limitResults(pRuntimeEnv);
break;
@ -4704,8 +4807,7 @@ static void tableQueryImpl(SQInfo *pQInfo) {
* So, we do keep in this procedure instead of launching retrieve procedure for next results.
*/
int32_t numOfInterpo = 0;
int32_t remain = taosNumOfRemainRows(pRuntimeEnv->pFillInfo);
pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, remain, &numOfInterpo);
pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfInterpo);
if (pQuery->rec.rows > 0) {
limitResults(pRuntimeEnv);
@ -4869,7 +4971,7 @@ static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pEx
(pFuncMsg->functionId == TSDB_FUNC_COUNT && pFuncMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) {
continue;
}
return false;
}
}
@ -5147,9 +5249,10 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
type = TSDB_DATA_TYPE_DOUBLE;
bytes = tDataTypeDesc[type].nSize;
} else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column
type = TSDB_DATA_TYPE_BINARY;
bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
} else {
SSchema s = tGetTableNameColumnSchema();
type = s.type;
bytes = s.bytes;
} else{
int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols);
assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags || j == TSDB_TBNAME_COLUMN_INDEX);
@ -5158,10 +5261,11 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
type = pCol->type;
bytes = pCol->bytes;
} else {
type = TSDB_DATA_TYPE_BINARY;
bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
}
SSchema s = tGetTableNameColumnSchema();
type = s.type;
bytes = s.bytes;
}
}
int32_t param = pExprs[i].base.arg[0].argValue.i64;
@ -5242,7 +5346,7 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) {
if (pQuery->colList[i].numOfFilters > 0) {
SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[j];
memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoData));
memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfo));
pFilterInfo->info = pQuery->colList[i];
pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters;
@ -5826,20 +5930,38 @@ _over:
//pQInfo already freed in initQInfo, but *pQInfo may not pointer to null;
if (code != TSDB_CODE_SUCCESS) {
*pQInfo = NULL;
} else {
SQInfo* pq = (SQInfo*) (*pQInfo);
T_REF_INC(pq);
T_REF_INC(pq);
}
// if failed to add ref for all meters in this query, abort current query
return code;
}
void qDestroyQueryInfo(qinfo_t pQInfo) {
static void doDestoryQueryInfo(SQInfo* pQInfo) {
assert(pQInfo != NULL);
qTrace("QInfo:%p query completed", pQInfo);
// print the query cost summary
queryCostStatis(pQInfo);
queryCostStatis(pQInfo); // print the query cost summary
freeQInfo(pQInfo);
}
void qDestroyQueryInfo(qinfo_t qHandle) {
SQInfo* pQInfo = (SQInfo*) qHandle;
if (!isValidQInfo(pQInfo)) {
return;
}
int16_t ref = T_REF_DEC(pQInfo);
qTrace("QInfo:%p dec refCount, value:%d", pQInfo, ref);
if (ref == 0) {
doDestoryQueryInfo(pQInfo);
}
}
void qTableQuery(qinfo_t qinfo) {
SQInfo *pQInfo = (SQInfo *)qinfo;
@ -5850,6 +5972,7 @@ void qTableQuery(qinfo_t qinfo) {
if (isQueryKilled(pQInfo)) {
qTrace("QInfo:%p it is already killed, abort", pQInfo);
qDestroyQueryInfo(pQInfo);
return;
}
@ -5865,7 +5988,7 @@ void qTableQuery(qinfo_t qinfo) {
}
sem_post(&pQInfo->dataReady);
// vnodeDecRefCount(pQInfo);
qDestroyQueryInfo(pQInfo);
}
int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
@ -5891,20 +6014,29 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
bool qHasMoreResultsToRetrieve(qinfo_t qinfo) {
SQInfo *pQInfo = (SQInfo *)qinfo;
if (pQInfo == NULL || pQInfo->signature != pQInfo || pQInfo->code != TSDB_CODE_SUCCESS) {
if (!isValidQInfo(pQInfo) || pQInfo->code != TSDB_CODE_SUCCESS) {
qTrace("QInfo:%p invalid qhandle or error occurs, abort query, code:%x", pQInfo, pQInfo->code);
return false;
}
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
bool ret = false;
if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
return false;
ret = false;
} else if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) {
return true;
ret = true;
} else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
return true;
ret = true;
} else {
assert(0);
}
if (ret) {
T_REF_INC(pQInfo);
qTrace("QInfo:%p has more results waits for client retrieve", pQInfo);
}
return ret;
}
int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen) {
@ -5949,31 +6081,44 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
return code;
}
int32_t qKillQuery(qinfo_t qinfo) {
SQInfo *pQInfo = (SQInfo *)qinfo;
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
setQueryKilled(pQInfo);
qDestroyQueryInfo(pQInfo);
return TSDB_CODE_SUCCESS;
}
static void buildTagQueryResult(SQInfo* pQInfo) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
SQuery * pQuery = pRuntimeEnv->pQuery;
size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
assert(numOfGroup == 0 || numOfGroup == 1);
if (numOfGroup == 0) {
return;
}
SArray* pa = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
size_t num = taosArrayGetSize(pa);
assert(num == pQInfo->groupInfo.numOfTables);
int32_t count = 0;
int32_t functionId = pQuery->pSelectExpr[0].base.functionId;
if (functionId == TSDB_FUNC_TID_TAG) { // return the tags & table Id
assert(pQuery->numOfOutput == 1);
SExprInfo* pExprInfo = &pQuery->pSelectExpr[0];
int32_t rsize = pExprInfo->bytes;
count = 0;
while(pQInfo->tableIndex < num && count < pQuery->rec.capacity) {
int32_t i = pQInfo->tableIndex++;
SGroupItem *item = taosArrayGet(pa, i);
@ -6015,12 +6160,12 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
}
}
}
count += 1;
}
qTrace("QInfo:%p create (tableId, tag) info completed, rows:%d", pQInfo, count);
} else if (functionId == TSDB_FUNC_COUNT) {// handle the "count(tbname)" query
*(int64_t*) pQuery->sdata[0]->data = num;
@ -6029,16 +6174,17 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
qTrace("QInfo:%p create count(tbname) query, res:%d rows:1", pQInfo, count);
} else { // return only the tags|table name etc.
count = 0;
SSchema tbnameSchema = tGetTableNameColumnSchema();
while(pQInfo->tableIndex < num && count < pQuery->rec.capacity) {
int32_t i = pQInfo->tableIndex++;
SExprInfo* pExprInfo = pQuery->pSelectExpr;
SGroupItem* item = taosArrayGet(pa, i);
for(int32_t j = 0; j < pQuery->numOfOutput; ++j) {
if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
char* data = tsdbGetTableName(pQInfo->tsdb, &item->id);
char* dst = pQuery->sdata[j]->data + count * (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE);
char* dst = pQuery->sdata[j]->data + count * tbnameSchema.bytes;
memcpy(dst, data, varDataTLen(data));
} else {// todo refactor
int16_t type = pExprInfo[j].type;
@ -6046,7 +6192,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
char* data = tsdbGetTableTagVal(pQInfo->tsdb, &item->id, pExprInfo[j].base.colInfo.colId, type, bytes);
char* dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
if (data == NULL) {
setVardataNull(dst, type);
@ -6064,7 +6210,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
}
count += 1;
}
qTrace("QInfo:%p create tag values results completed, rows:%d", pQInfo, count);
}

12
src/query/src/qUtil.c
View File

@ -113,7 +113,9 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
for (int32_t i = 0; i < num; ++i) {
SWindowResult *pResult = &pWindowResInfo->pResult[i];
if (pResult->status.closed) { // remove the window slot from hash table
taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE);
taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, pWindowResInfo->type);
printf("remove ============>%ld, remain size:%ld\n", pResult->window.skey, pWindowResInfo->hashList->size);
} else {
break;
}
@ -133,14 +135,16 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
}
pWindowResInfo->size = remain;
printf("---------------size:%ld\n", taosHashGetSize(pWindowResInfo->hashList));
for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
SWindowResult *pResult = &pWindowResInfo->pResult[k];
int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE);
int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey,
tDataTypeDesc[pWindowResInfo->type].nSize);
int32_t v = (*p - num);
assert(v >= 0 && v <= pWindowResInfo->size);
taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, TSDB_KEYSIZE, (char *)&v, sizeof(int32_t));
taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, tDataTypeDesc[pWindowResInfo->type].nSize,
(char *)&v, sizeof(int32_t));
}
pWindowResInfo->curIndex = -1;

4
src/query/src/qast.c
View File

@ -138,7 +138,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken
memcpy(pNode->pSchema, &pSchema[i], sizeof(SSchema));
} else {
pNode->pSchema->type = TSDB_DATA_TYPE_BINARY;
pNode->pSchema->bytes = TSDB_TABLE_NAME_LEN;
pNode->pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
strcpy(pNode->pSchema->name, TSQL_TBNAME_L);
pNode->pSchema->colId = -1;
}
@ -1127,7 +1127,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
left->pSchema = pSchema;
pSchema->type = TSDB_DATA_TYPE_BINARY;
pSchema->bytes = TSDB_TABLE_NAME_LEN;
pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
strcpy(pSchema->name, TSQL_TBNAME_L);
pSchema->colId = -1;

66
src/query/src/qfill.c
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "qfill.h"
#include "os.h"
#include "qfill.h"
#include "qextbuffer.h"
#include "taosdef.h"
#include "taosmsg.h"
@ -58,7 +58,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, ch
}
SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int32_t fillType, SFillColInfo* pFillCol) {
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, SFillColInfo* pFillCol) {
if (fillType == TSDB_FILL_NONE) {
return NULL;
}
@ -72,8 +72,10 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_
pFillInfo->pFillCol = pFillCol;
pFillInfo->numOfTags = numOfTags;
pFillInfo->numOfCols = numOfCols;
pFillInfo->precision = precision;
pFillInfo->slidingTime = slidingTime;
pFillInfo->slidingUnit = slidingUnit;
pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
int32_t rowsize = 0;
@ -102,9 +104,9 @@ void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
pFillInfo->numOfTotal = 0;
}
void taosDestoryFillInfo(SFillInfo* pFillInfo) {
void* taosDestoryFillInfo(SFillInfo* pFillInfo) {
if (pFillInfo == NULL) {
return;
return NULL;
}
tfree(pFillInfo->prevValues);
@ -119,6 +121,15 @@ void taosDestoryFillInfo(SFillInfo* pFillInfo) {
tfree(pFillInfo->pFillCol);
tfree(pFillInfo);
return NULL;
}
static TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision) {
if (order == TSDB_ORDER_ASC) {
return ekey;
} else {
return taosGetIntervalStartTimestamp(ekey, timeInterval, slidingTimeUnit, precision);
}
}
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
@ -126,8 +137,10 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey)
return;
}
pFillInfo->endKey = taosGetRevisedEndKey(endKey, pFillInfo->order, pFillInfo->slidingTime, pFillInfo->slidingUnit,
pFillInfo->precision);
pFillInfo->rowIdx = 0;
pFillInfo->endKey = endKey;
pFillInfo->numOfRows = numOfRows;
// ensure the space
@ -165,36 +178,29 @@ void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInpu
}
}
TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision) {
if (order == TSDB_ORDER_ASC) {
return ekey;
} else {
return taosGetIntervalStartTimestamp(ekey, timeInterval, slidingTimeUnit, precision);
}
}
int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
int64_t* tsList = (int64_t*) pFillInfo->pData[0];
static int32_t taosGetTotalNumOfFilledRes(SFillInfo* pFillInfo, const TSKEY* tsArray, int32_t remain,
int64_t nInterval, int64_t ekey) {
if (remain > 0) { // still fill gap within current data block, not generating data after the result set.
TSKEY lastKey = tsArray[pFillInfo->numOfRows - 1];
int32_t total = (int32_t)(labs(lastKey - pFillInfo->start) / nInterval) + 1;
int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
assert(total >= remain);
return total;
TSKEY ekey1 = taosGetRevisedEndKey(ekey, pFillInfo->order, pFillInfo->slidingTime, pFillInfo->slidingUnit,
pFillInfo->precision);
int64_t numOfRes = -1;
if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set.
TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];
numOfRes = (int64_t)(labs(lastKey - pFillInfo->start) / pFillInfo->slidingTime) + 1;
assert(numOfRes >= numOfRows);
} else { // reach the end of data
if ((ekey < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) ||
(ekey > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) {
if ((ekey1 < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) ||
(ekey1 > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) {
return 0;
} else {
return (int32_t)(labs(ekey - pFillInfo->start) / nInterval) + 1;
} else { // the numOfRes rows are all filled with specified policy
numOfRes = (labs(ekey1 - pFillInfo->start) / pFillInfo->slidingTime) + 1;
}
}
}
int64_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows) {
int32_t numOfRes = taosGetTotalNumOfFilledRes(pFillInfo, (int64_t*) pFillInfo->pData[0], numOfRows,
pFillInfo->slidingTime, ekey);
return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
}
@ -496,8 +502,8 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu
int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) {
int32_t remain = taosNumOfRemainRows(pFillInfo); // todo use iterator?
int32_t rows = taosGetNumOfResultWithFill(pFillInfo, remain, pFillInfo->endKey, capacity);
int32_t rows = getFilledNumOfRes(pFillInfo, pFillInfo->endKey, capacity);
int32_t numOfRes = generateDataBlockImpl(pFillInfo, output, remain, rows, pFillInfo->pData);
assert(numOfRes == rows);

2
src/query/src/qtsbuf.c
View File

@ -51,7 +51,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
return NULL;
}
strncpy(pTSBuf->path, path, PATH_MAX);
tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path));
pTSBuf->f = fopen(pTSBuf->path, "r+");
if (pTSBuf->f == NULL) {

98
src/rpc/src/rpcMain.c
View File

@ -73,6 +73,7 @@ typedef struct {
SRpcInfo *pRpc; // associated SRpcInfo
SRpcIpSet ipSet; // ip list provided by app
void *ahandle; // handle provided by app
struct SRpcConn *pConn; // pConn allocated
char msgType; // message type
uint8_t *pCont; // content provided by app
int32_t contLen; // content length
@ -339,7 +340,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
}
void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) {
void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) {
SRpcInfo *pRpc = (SRpcInfo *)shandle;
SRpcReqContext *pContext;
@ -367,7 +368,7 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg)
rpcSendReqToServer(pRpc, pContext);
return;
return pContext;
}
void rpcSendResponse(const SRpcMsg *pRsp) {
@ -393,7 +394,6 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
if ( pConn->inType == 0 || pConn->user[0] == 0 ) {
tTrace("%s, connection is already released, rsp wont be sent", pConn->info);
rpcUnlockConn(pConn);
rpcDecRef(pRpc);
return;
}
@ -426,6 +426,10 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
rpcSendMsgToPeer(pConn, msg, msgLen);
pConn->secured = 1; // connection shall be secured
if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg);
pConn->pReqMsg = NULL;
pConn->reqMsgLen = 0;
rpcUnlockConn(pConn);
rpcDecRef(pRpc); // decrease the referene count
@ -458,7 +462,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
pInfo->clientPort = pConn->peerPort;
// pInfo->serverIp = pConn->destIp;
strncpy(pInfo->user, pConn->user, sizeof(pInfo->user));
tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user));
return 0;
}
@ -482,6 +486,35 @@ void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pMsg, SRpcMsg
return;
}
// this API is used by server app to keep an APP context in case connection is broken
int rpcReportProgress(void *handle, char *pCont, int contLen) {
SRpcConn *pConn = (SRpcConn *)handle;
if (pConn->user[0]) {
// pReqMsg and reqMsgLen is re-used to store the context from app server
pConn->pReqMsg = pCont;
pConn->reqMsgLen = contLen;
return 0;
}
tTrace("%s, rpc connection is already released", pConn->info);
rpcFreeCont(pCont);
return -1;
}
/* todo: cancel process may have race condition, pContext may have been released
just before app calls the rpcCancelRequest */
void rpcCancelRequest(void *handle) {
SRpcReqContext *pContext = handle;
if (pContext->pConn) {
tTrace("%s, app trys to cancel request", pContext->pConn->info);
rpcCloseConn(pContext->pConn);
pContext->pConn = NULL;
rpcFreeCont(pContext->pCont);
}
}
static void rpcFreeMsg(void *msg) {
if ( msg ) {
char *temp = (char *)msg - sizeof(SRpcReqContext);
@ -493,7 +526,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort,
SRpcConn *pConn;
uint32_t peerIp = taosGetIpFromFqdn(peerFqdn);
if (peerIp == -1) {
if (peerIp == 0xFFFFFFFF) {
tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn);
terrno = TSDB_CODE_RPC_APP_ERROR;
return NULL;
@ -542,7 +575,7 @@ static void rpcCloseConn(void *thandle) {
if ( pRpc->connType == TAOS_CONN_SERVER) {
char hashstr[40] = {0};
size_t size = sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType);
size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType);
taosHashRemove(pRpc->hash, hashstr, size);
rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg
@ -592,7 +625,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
char hashstr[40] = {0};
SRpcHead *pHead = (SRpcHead *)pRecv->msg;
size_t size = sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType);
size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType);
// check if it is already allocated
SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size));
@ -682,7 +715,7 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) {
if (pConn) {
pConn->tretry = 0;
pConn->ahandle = pContext->ahandle;
sprintf(pConn->info, "%s %p %p", pRpc->label, pConn, pConn->ahandle);
snprintf(pConn->info, sizeof(pConn->info), "%s %p %p", pRpc->label, pConn, pConn->ahandle);
pConn->tretry = 0;
} else {
tError("%s %p, failed to set up connection(%s)", pRpc->label, pContext->ahandle, tstrerror(terrno));
@ -811,7 +844,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
if (rpcIsReq(pHead->msgType)) {
pConn->ahandle = (void *)pHead->ahandle;
sprintf(pConn->info, "%s %p %p", pRpc->label, pConn, pConn->ahandle);
snprintf(pConn->info, sizeof(pConn->info), "%s %p %p", pRpc->label, pConn, pConn->ahandle);
}
sid = pConn->sid;
@ -846,6 +879,22 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
return pConn;
}
static void rpcReportBrokenLinkToServer(SRpcConn *pConn) {
SRpcInfo *pRpc = pConn->pRpc;
// if there are pending request, notify the app
tTrace("%s, notify the server app, connection is gone", pConn->info);
SRpcMsg rpcMsg;
rpcMsg.pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server
rpcMsg.contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length
rpcMsg.ahandle = pConn->ahandle;
rpcMsg.handle = pConn;
rpcMsg.msgType = pConn->inType;
rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
if (pRpc->cfp) (*(pRpc->cfp))(&rpcMsg, NULL);
}
static void rpcProcessBrokenLink(SRpcConn *pConn) {
if (pConn == NULL) return;
SRpcInfo *pRpc = pConn->pRpc;
@ -859,19 +908,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) {
taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl);
}
if (pConn->inType) {
// if there are pending request, notify the app
tTrace("%s, connection is gone, notify the app", pConn->info);
/*
SRpcMsg rpcMsg;
rpcMsg.pCont = NULL;
rpcMsg.contLen = 0;
rpcMsg.handle = pConn;
rpcMsg.msgType = pConn->inType;
rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
(*(pRpc->cfp))(&rpcMsg);
*/
}
if (pConn->inType) rpcReportBrokenLinkToServer(pConn);
rpcUnlockConn(pConn);
rpcCloseConn(pConn);
@ -920,6 +957,7 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
SRpcInfo *pRpc = pContext->pRpc;
pContext->pConn = NULL;
if (pContext->pRsp) {
// for synchronous API
memcpy(pContext->pSet, &pContext->ipSet, sizeof(SRpcIpSet));
@ -1088,6 +1126,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
return;
}
pContext->pConn = pConn;
pConn->ahandle = pContext->ahandle;
rpcLockConn(pConn);
@ -1210,23 +1249,10 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) {
static void rpcProcessIdleTimer(void *param, void *tmrId) {
SRpcConn *pConn = (SRpcConn *)param;
SRpcInfo *pRpc = pConn->pRpc;
if (pConn->user[0]) {
tTrace("%s, close the connection since no activity", pConn->info);
if (pConn->inType && pRpc->cfp) {
// if there are pending request, notify the app
tTrace("%s, notify the app, connection is gone", pConn->info);
/*
SRpcMsg rpcMsg;
rpcMsg.pCont = NULL;
rpcMsg.contLen = 0;
rpcMsg.handle = pConn;
rpcMsg.msgType = pConn->inType;
rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
(*(pRpc->cfp))(&rpcMsg);
*/
}
if (pConn->inType) rpcReportBrokenLinkToServer(pConn);
rpcCloseConn(pConn);
} else {
tTrace("%s, idle timer:%p not processed", pConn->info, tmrId);

6
src/tsdb/inc/tsdbMain.h
View File

@ -161,7 +161,7 @@ typedef struct {
int64_t index;
int numOfCacheBlocks;
SList * memPool;
} STsdbCachePool;
} STsdbBufferPool;
typedef struct {
TSKEY keyFirst;
@ -173,7 +173,7 @@ typedef struct {
typedef struct {
int cacheBlockSize;
int totalCacheBlocks;
STsdbCachePool pool;
STsdbBufferPool pool;
STsdbCacheBlock *curBlock;
SCacheMem * mem;
SCacheMem * imem;
@ -357,6 +357,8 @@ typedef struct STsdbRepo {
STsdbAppH appH;
STsdbStat stat;
// The meter meta handle of this TSDB repository
STsdbMeta *tsdbMeta;

8
src/tsdb/src/tsdbCache.c
View File

@ -35,7 +35,7 @@ STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TsdbRepoT *pRepo)
pCache->totalCacheBlocks = totalBlocks;
pCache->pRepo = pRepo;
STsdbCachePool *pPool = &(pCache->pool);
STsdbBufferPool *pPool = &(pCache->pool);
pPool->index = 0;
pPool->memPool = tdListNew(sizeof(STsdbCacheBlock *));
if (pPool->memPool == NULL) goto _err;
@ -106,7 +106,7 @@ static void tsdbFreeCacheMem(SCacheMem *mem) {
}
static int tsdbAllocBlockFromPool(STsdbCache *pCache) {
STsdbCachePool *pPool = &(pCache->pool);
STsdbBufferPool *pPool = &(pCache->pool);
tsdbLockRepo(pCache->pRepo);
if (listNEles(pPool->memPool) == 0) {
@ -170,7 +170,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
}
static int tsdbAddCacheBlockToPool(STsdbCache *pCache) {
STsdbCachePool *pPool = &pCache->pool;
STsdbBufferPool *pPool = &pCache->pool;
STsdbCacheBlock *pBlock = malloc(sizeof(STsdbCacheBlock) + pCache->cacheBlockSize);
if (pBlock == NULL) return -1;
@ -184,7 +184,7 @@ static int tsdbAddCacheBlockToPool(STsdbCache *pCache) {
}
static int tsdbRemoveCacheBlockFromPool(STsdbCache *pCache) {
STsdbCachePool *pPool = &pCache->pool;
STsdbBufferPool *pPool = &pCache->pool;
STsdbCacheBlock *pBlock = NULL;
ASSERT(pCache->totalCacheBlocks >= 0);

16
src/tsdb/src/tsdbMain.c
View File

@ -953,6 +953,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
STsdbMeta *pMeta = pRepo->tsdbMeta;
int64_t points = 0;
STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid};
STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId);
@ -964,7 +965,9 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY
// Check schema version
int32_t tversion = pBlock->sversion;
int16_t nversion = schemaVersion(tsdbGetTableSchema(pMeta, pTable));
STSchema * pSchema = tsdbGetTableSchema(pMeta, pTable);
ASSERT(pSchema != NULL);
int16_t nversion = schemaVersion(pSchema);
if (tversion > nversion) {
tsdbTrace("vgId:%d table:%s tid:%d server schema version %d is older than clien version %d, try to config.",
pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, nversion, tversion);
@ -1014,7 +1017,10 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY
return -1;
}
(*affectedrows)++;
points++;
}
atomic_fetch_add_64(&(pRepo->stat.pointsWritten), points * (pSchema->numOfCols));
atomic_fetch_add_64(&(pRepo->stat.totalStorage), points * pSchema->vlen);
return TSDB_CODE_SUCCESS;
}
@ -1381,3 +1387,11 @@ uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t
return magic;
}
void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage){
ASSERT(repo != NULL);
STsdbRepo * pRepo = repo;
*totalPoints = pRepo->stat.pointsWritten;
*totalStorage = pRepo->stat.totalStorage;
*compStorage = pRepo->stat.compStorage;
}

4
src/tsdb/src/tsdbMeta.c
View File

@ -334,7 +334,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
pTable->schema[0] = tdDupSchema(pCfg->schema);
pTable->tagSchema = tdDupSchema(pCfg->tagSchema);
tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN);
tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN - 1);
pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1);
if (pTable->name == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@ -355,7 +355,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
pTable->tableId.tid = pCfg->tableId.tid;
pTable->lastKey = TSKEY_INITIAL_VAL;
tsize = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN);
tsize = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN - 1);
pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1);
if (pTable->name == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;

6
src/tsdb/src/tsdbMetaFile.c
View File

@ -105,7 +105,7 @@ int32_t tsdbInsertMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t c
return -1;
}
fsync(mfh->fd);
// fsync(mfh->fd);
mfh->tombSize++;
@ -132,7 +132,7 @@ int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, uint64_t uid) {
return -1;
}
fsync(mfh->fd);
// fsync(mfh->fd);
mfh->nDel++;
@ -167,7 +167,7 @@ int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t c
return -1;
}
fsync(mfh->fd);
// fsync(mfh->fd);
return 0;
}

291
src/tsdb/src/tsdbRead.c
View File

@ -72,7 +72,7 @@ typedef struct STableCheckInfo {
int32_t compSize;
int32_t numOfBlocks; // number of qualified data blocks not the original blocks
SDataCols* pDataCols;
int32_t chosen; // indicate which iterator should move forward
bool initBuf; // whether to initialize the in-memory skip list iterator or not
SSkipListIterator* iter; // mem buffer skip list iterator
@ -233,8 +233,6 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond*
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
// pQueryHandle->outputCapacity = 2; // only allowed two rows to be loaded
changeQueryHandleForInterpQuery(pQueryHandle);
return pQueryHandle;
}
@ -313,14 +311,14 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) {
rmem = SL_GET_NODE_DATA(node);
}
}
if (pCheckInfo->iiter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
if (node != NULL) {
rimem = SL_GET_NODE_DATA(node);
}
}
if (rmem != NULL && rimem != NULL) {
if (dataRowKey(rmem) < dataRowKey(rimem)) {
pCheckInfo->chosen = 0;
@ -335,17 +333,17 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) {
return rimem;
}
}
if (rmem != NULL) {
pCheckInfo->chosen = 0;
return rmem;
}
if (rimem != NULL) {
pCheckInfo->chosen = 1;
return rimem;
}
return NULL;
}
@ -355,11 +353,11 @@ bool moveToNextRow(STableCheckInfo* pCheckInfo) {
if (pCheckInfo->iter != NULL) {
hasNext = tSkipListIterNext(pCheckInfo->iter);
}
if (hasNext) {
return hasNext;
}
if (pCheckInfo->iiter != NULL) {
return tSkipListIterGet(pCheckInfo->iiter) != NULL;
}
@ -368,17 +366,17 @@ bool moveToNextRow(STableCheckInfo* pCheckInfo) {
if (pCheckInfo->iiter != NULL) {
hasNext = tSkipListIterNext(pCheckInfo->iiter);
}
if (hasNext) {
return hasNext;
}
if (pCheckInfo->iter != NULL) {
return tSkipListIterGet(pCheckInfo->iter) != NULL;
}
}
}
return hasNext;
}
@ -397,7 +395,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
if (row == NULL) {
return false;
}
pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer
tsdbTrace("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, %p", pHandle,
pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qinfo);
@ -583,9 +581,9 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
bool blockLoaded = false;
SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
int64_t st = taosGetTimestampUs();
if (pCheckInfo->pDataCols == NULL) {
STsdbMeta* pMeta = tsdbGetMeta(pRepo);
pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
@ -605,67 +603,32 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
assert(pCols->numOfRows != 0);
taosArrayDestroy(sa);
tfree(data);
int64_t et = taosGetTimestampUs() - st;
tsdbTrace("%p load file block into buffer, elapsed time:%"PRId64 " us", pQueryHandle, et);
return blockLoaded;
}
static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
SQueryFilePos* cur = &pQueryHandle->cur;
SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlock);
/*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
TSKEY k1 = TSKEY_INITIAL_VAL, k2 = TSKEY_INITIAL_VAL;
if (pCheckInfo->iter != NULL && tSkipListIterGet(pCheckInfo->iter) != NULL) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
SDataRow row = SL_GET_NODE_DATA(node);
k1 = dataRowKey(row);
if (k1 == binfo.window.skey) {
if (tSkipListIterNext(pCheckInfo->iter)) {
node = tSkipListIterGet(pCheckInfo->iter);
row = SL_GET_NODE_DATA(node);
k1 = dataRowKey(row);
} else {
k1 = TSKEY_INITIAL_VAL;
}
}
}
if (pCheckInfo->iiter != NULL && tSkipListIterGet(pCheckInfo->iiter) != NULL) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
SDataRow row = SL_GET_NODE_DATA(node);
k2 = dataRowKey(row);
if (k2 == binfo.window.skey) {
if (tSkipListIterNext(pCheckInfo->iiter)) {
node = tSkipListIterGet(pCheckInfo->iiter);
row = SL_GET_NODE_DATA(node);
k2 = dataRowKey(row);
} else {
k2 = TSKEY_INITIAL_VAL;
}
}
}
SDataRow row = getSDataRowInTableMem(pCheckInfo);
TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL;
cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1);
if ((ASCENDING_TRAVERSE(pQueryHandle->order) &&
((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey) || (k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey))) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) &&
((k1 != TSKEY_INITIAL_VAL && k1 >= binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 >= binfo.window.skey)))) {
if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
if ((ASCENDING_TRAVERSE(pQueryHandle->order) &&
((k1 != TSKEY_INITIAL_VAL && k1 < binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 < binfo.window.skey))) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) &&
(((k1 != TSKEY_INITIAL_VAL && k1 > binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 > binfo.window.skey))))) {
if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
// do not load file block into buffer
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
@ -718,7 +681,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) {
return false;
}
SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows);
@ -756,7 +719,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
return pQueryHandle->realNumOfRows > 0;
}
static int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
int firstPos, lastPos, midPos = -1;
int numOfRows;
TSKEY* keyList;
@ -868,37 +831,63 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap
return numOfRows + num;
}
static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t capacity,
int32_t numOfRows, SDataRow row, STSchema* pSchema) {
int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
int32_t numOfTableCols = schemaNCols(pSchema);
static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row,
STsdbMeta *pMeta, int32_t numOfCols, STable* pTable) {
char* pData = NULL;
for (int32_t i = 0; i < numOfCols; ++i) {
// the schema version info is embeded in SDataRow
STSchema* pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, dataRowVersion(row));
int32_t numOfRowCols = schemaNCols(pSchema);
int32_t i = 0, j = 0;
while(i < numOfCols && j < numOfRowCols) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
if (pSchema->columns[j].colId < pColInfo->info.colId) {
j++;
continue;
}
if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else {
pData = pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
}
int32_t offset = 0;
for (int32_t j = 0; j < numOfTableCols; ++j) {
if (pColInfo->info.colId == pSchema->columns[j].colId) {
offset = pSchema->columns[j].offset;
break;
if (pSchema->columns[j].colId == pColInfo->info.colId) {
void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset);
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
memcpy(pData, value, varDataTLen(value));
} else {
memcpy(pData, value, pColInfo->info.bytes);
}
j++;
i++;
} else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pData, pColInfo->info.type);
} else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
}
i++;
}
assert(offset != -1); // todo handle error
void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset);
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
memcpy(pData, value, varDataTLen(value));
}
while (i < numOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else {
memcpy(pData, value, pColInfo->info.bytes);
pData = pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
}
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pData, pColInfo->info.type);
} else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
}
i++;
}
}
@ -911,7 +900,16 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
initTableMemIterator(pQueryHandle, pCheckInfo);
SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
// for search the endPos, so the order needs to reverse
int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
STable* pTable = pCheckInfo->pTableObj;
int32_t endPos = cur->pos;
if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) {
endPos = blockInfo.rows - 1;
@ -920,8 +918,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
endPos = 0;
cur->mixBlock = (cur->pos != blockInfo.rows - 1);
} else {
int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
assert(pCols->numOfRows > 0);
endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
cur->mixBlock = true;
}
@ -933,8 +931,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
int32_t numOfRows = 0;
pQueryHandle->cur.win = TSWINDOW_INITIALIZER;
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
// no data in buffer, load data from file directly
if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) {
int32_t start = cur->pos;
@ -950,12 +947,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
// todo opt in case of no data in buffer
numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
// if the buffer is not full in case of descending order query, move the data in the front of the buffer
// if the buffer is not full in case of descending order query, move the data in the front of the buffer
if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < pQueryHandle->outputCapacity) {
int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
int32_t reqNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
for(int32_t i = 0; i < reqNumOfCols; ++i) {
for(int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
}
@ -969,20 +965,15 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
pQueryHandle->realNumOfRows = numOfRows;
cur->rows = numOfRows;
return;
} else if (pCheckInfo->iter != NULL && pCheckInfo->iiter == NULL) {
// } else if (pCheckInfo->iter == NULL && pCheckInfo->iiter != NULL) {
// } else { // iter and iiter are all not NULL, three-way merge data block
STSchema* pSchema = tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj);
} else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
SSkipListNode* node = NULL;
do {
node = tSkipListIterGet(pCheckInfo->iter);
if (node == NULL) {
SDataRow row = getSDataRowInTableMem(pCheckInfo);
if (row == NULL) {
break;
}
SDataRow row = SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row);
TSKEY key = dataRowKey(row);
if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
break;
@ -995,7 +986,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
copyOneRowFromMem(pQueryHandle, pCheckInfo, pQueryHandle->outputCapacity, numOfRows, row, pSchema);
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, pMeta, numOfCols, pTable);
numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key;
@ -1005,17 +996,16 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
cur->lastKey = key + step;
cur->mixBlock = true;
tSkipListIterNext(pCheckInfo->iter);
moveToNextRow(pCheckInfo);
} else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it
tSkipListIterNext(pCheckInfo->iter);
moveToNextRow(pCheckInfo);
} else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = tsArray[pos];
}
int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
int32_t end = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order);
int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order);
if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
tSkipListIterNext(pCheckInfo->iter);
}
@ -1093,9 +1083,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
// if the buffer is not full in case of descending order query, move the data in the front of the buffer
if (numOfRows < pQueryHandle->outputCapacity) {
int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
int32_t requiredNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
for(int32_t i = 0; i < requiredNumOfCols; ++i) {
for(int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
}
@ -1224,7 +1212,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
*numOfAllocBlocks = numOfBlocks;
int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
SBlockOrderSupporter sup = {0};
sup.numOfTables = numOfTables;
sup.numOfBlocksPerTable = calloc(1, sizeof(int32_t) * numOfTables);
@ -1268,17 +1256,17 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
}
assert(numOfBlocks == cnt);
// since there is only one table qualified, blocks are not sorted
if (numOfQualTables == 1) {
memcpy(pQueryHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks);
cleanBlockOrderSupporter(&sup, numOfQualTables);
tsdbTrace("%p create data blocks info struct completed for 1 table, %d blocks not sorted %p ", pQueryHandle, cnt,
pQueryHandle->qinfo);
return TSDB_CODE_SUCCESS;
}
tsdbTrace("%p create data blocks info struct completed, %d blocks in %d tables %p", pQueryHandle, cnt,
numOfQualTables, pQueryHandle->qinfo);
@ -1567,9 +1555,6 @@ void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
for(int32_t i = 0; i < numOfTables; ++i) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
if (pCheckInfo->pTableObj->tableId.uid == 12094628167747) {
printf("abc\n");
}
if (pCheckInfo->pTableObj->lastKey > key) {
key = pCheckInfo->pTableObj->lastKey;
index = i;
@ -1652,9 +1637,9 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
*skey = TSKEY_INITIAL_VAL;
int64_t st = taosGetTimestampUs();
STSchema* pSchema = tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj);
int32_t numOfTableCols = schemaNCols(pSchema);
STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
STable* pTable = pCheckInfo->pTableObj;
do {
SDataRow row = getSDataRowInTableMem(pCheckInfo);
if (row == NULL) {
@ -1662,10 +1647,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
}
TSKEY key = dataRowKey(row);
if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
tsdbTrace("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey,
pQueryHandle->window.ekey);
@ -1677,59 +1659,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
}
*ekey = key;
char* pData = NULL;
int32_t i = 0, j = 0;
while(i < numOfCols && j < numOfTableCols) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
if (pSchema->columns[j].colId < pColInfo->info.colId) {
j++;
continue;
}
if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else {
pData = pColInfo->pData + (maxRowsToRead - numOfRows - 1) * pColInfo->info.bytes;
}
if (pSchema->columns[j].colId == pColInfo->info.colId) {
void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset);
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
memcpy(pData, value, varDataTLen(value));
} else {
memcpy(pData, value, pColInfo->info.bytes);
}
j++;
i++;
} else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pData, pColInfo->info.type);
} else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
}
i++;
}
}
while (i < numOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else {
pData = pColInfo->pData + (maxRowsToRead - numOfRows - 1) * pColInfo->info.bytes;
}
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pData, pColInfo->info.type);
} else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
}
i++;
}
copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, pMeta, numOfCols, pTable);
if (++numOfRows >= maxRowsToRead) {
moveToNextRow(pCheckInfo);
break;
@ -1752,7 +1683,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
int64_t elapsedTime = taosGetTimestampUs() - st;
tsdbTrace("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d", pQueryHandle,
elapsedTime, numOfRows, numOfCols);
return numOfRows;
}
@ -2011,7 +1942,7 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
f1 = (char*) pTable1->name;
f2 = (char*) pTable2->name;
type = TSDB_DATA_TYPE_BINARY;
bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
bytes = tGetTableNameColumnSchema().bytes;
} else {
STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex);
bytes = pCol->bytes;

57
src/util/inc/tcache.h
View File

@ -37,8 +37,8 @@ typedef struct SCacheDataNode {
uint64_t expiredTime; // expiredTime expiredTime when this element should be remove from cache
uint64_t signature;
uint32_t size; // allocated size for current SCacheDataNode
uint16_t keySize : 15;
bool inTrash : 1; // denote if it is in trash or not
uint16_t keySize: 15;
bool inTrashCan: 1;// denote if it is in trash or not
T_REF_DECLARE()
char *key;
char data[];
@ -50,46 +50,49 @@ typedef struct STrashElem {
SCacheDataNode *pData;
} STrashElem;
/*
* to accommodate the old data which has the same key value of new one in hashList
* when an new node is put into cache, if an existed one with the same key:
* 1. if the old one does not be referenced, update it.
* 2. otherwise, move the old one to pTrash, addedTime the new one.
*
* when the node in pTrash does not be referenced, it will be release at the expired expiredTime
*/
typedef struct {
int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included.
int64_t refreshTime;
/*
* to accommodate the old datanode which has the same key value of new one in hashList
* when an new node is put into cache, if an existed one with the same key:
* 1. if the old one does not be referenced, update it.
* 2. otherwise, move the old one to pTrash, addedTime the new one.
*
* when the node in pTrash does not be referenced, it will be release at the expired expiredTime
*/
STrashElem * pTrash;
void * tmrCtrl;
void * pTimer;
SCacheStatis statistics;
SHashObj * pHashTable;
int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included.
int64_t refreshTime;
STrashElem * pTrash;
void * tmrCtrl;
void * pTimer;
SCacheStatis statistics;
SHashObj * pHashTable;
_hash_free_fn_t freeFp;
int numOfElemsInTrash; // number of element in trash
int16_t deleting; // set the deleting flag to stop refreshing ASAP.
T_REF_DECLARE()
uint32_t numOfElemsInTrash; // number of element in trash
uint8_t deleting; // set the deleting flag to stop refreshing ASAP.
pthread_t refreshWorker;
#if defined(LINUX)
pthread_rwlock_t lock;
#else
pthread_mutex_t lock;
#endif
} SCacheObj;
/**
*
* @param maxSessions maximum slots available for hash elements
* @param tmrCtrl timer ctrl
* initialize the cache object
* @param refreshTime refresh operation interval time, the maximum survival time when one element is expired and
* not referenced by other objects
* @return
*/
SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTimeInSeconds);
SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTimeInSeconds, void (*freeCb)(void *data));
SCacheObj *taosCacheInit(int64_t refreshTimeInSeconds);
/**
* initialize the cache object and set the free object callback function
* @param refreshTimeInSeconds
* @param freeCb
* @return
*/
SCacheObj *taosCacheInitWithCb(int64_t refreshTimeInSeconds, void (*freeCb)(void *data));
/**
* add data into cache

5
src/util/inc/tkvstore.h
View File

@ -46,11 +46,12 @@ typedef struct {
} SKVStore;
int tdCreateKVStore(char *fname);
int tdDestroyKVStore();
int tdDestroyKVStore(char *fname);
SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH);
void tdCloseKVStore(SKVStore *pStore);
int tdKVStoreStartCommit(SKVStore *pStore);
int tdUpdateRecordInKVStore(SKVStore *pStore, uint64_t uid, void *cont, int contLen);
int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen);
int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid);
int tdKVStoreEndCommit(SKVStore *pStore);
#ifdef __cplusplus

2
src/util/inc/ttime.h
View File

@ -56,7 +56,7 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) {
int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts);
int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec);
int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
void deltaToUtcInitOnce();
#ifdef __cplusplus

403
src/util/src/tcache.c
View File

@ -77,31 +77,7 @@ static FORCE_INLINE void taosFreeNode(void *data) {
* @param lifespan total survial expiredTime from now
* @return SCacheDataNode
*/
static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size,
uint64_t duration) {
size_t totalSize = size + sizeof(SCacheDataNode) + keyLen + 1;
SCacheDataNode *pNewNode = calloc(1, totalSize);
if (pNewNode == NULL) {
uError("failed to allocate memory, reason:%s", strerror(errno));
return NULL;
}
memcpy(pNewNode->data, pData, size);
pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + size;
pNewNode->keySize = keyLen;
memcpy(pNewNode->key, key, keyLen);
pNewNode->addedTime = (uint64_t)taosGetTimestampMs();
pNewNode->expiredTime = pNewNode->addedTime + duration;
pNewNode->signature = (uint64_t)pNewNode;
pNewNode->size = (uint32_t)totalSize;
return pNewNode;
}
static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration);
/**
* addedTime object node into trash, and this object is closed for referencing if it is addedTime to trash
@ -109,50 +85,15 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const
* @param pCacheObj Cache object
* @param pNode Cache slot object
*/
static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
if (pNode->inTrash) { /* node is already in trash */
return;
}
STrashElem *pElem = calloc(1, sizeof(STrashElem));
pElem->pData = pNode;
pElem->next = pCacheObj->pTrash;
if (pCacheObj->pTrash) {
pCacheObj->pTrash->prev = pElem;
}
pElem->prev = NULL;
pCacheObj->pTrash = pElem;
pNode->inTrash = true;
pCacheObj->numOfElemsInTrash++;
uTrace("key:%s %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash);
}
static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode);
/**
* remove node in trash can
* @param pCacheObj
* @param pElem
*/
static void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem);
static void taosRemoveFromTrash(SCacheObj *pCacheObj, STrashElem *pElem) {
if (pElem->pData->signature != (uint64_t)pElem->pData) {
uError("key:sig:%d %p data has been released, ignore", pElem->pData->signature, pElem->pData);
return;
}
pCacheObj->numOfElemsInTrash--;
if (pElem->prev) {
pElem->prev->next = pElem->next;
} else { /* pnode is the header, update header */
pCacheObj->pTrash = pElem->next;
}
if (pElem->next) {
pElem->next->prev = pElem->prev;
}
pElem->pData->signature = 0;
if (pCacheObj->freeFp) pCacheObj->freeFp(pElem->pData->data);
free(pElem->pData);
free(pElem);
}
/**
* remove nodes in trash with refCount == 0 in cache
* @param pNode
@ -160,42 +101,7 @@ static void taosRemoveFromTrash(SCacheObj *pCacheObj, STrashElem *pElem) {
* @param force force model, if true, remove data in trash without check refcount.
* may cause corruption. So, forece model only applys before cache is closed
*/
static void taosTrashEmpty(SCacheObj *pCacheObj, bool force) {
__cache_wr_lock(pCacheObj);
if (pCacheObj->numOfElemsInTrash == 0) {
if (pCacheObj->pTrash != NULL) {
uError("key:inconsistency data in cache, numOfElem in trash:%d", pCacheObj->numOfElemsInTrash);
}
pCacheObj->pTrash = NULL;
__cache_unlock(pCacheObj);
return;
}
STrashElem *pElem = pCacheObj->pTrash;
while (pElem) {
T_REF_VAL_CHECK(pElem->pData);
if (pElem->next == pElem) {
pElem->next = NULL;
}
if (force || (T_REF_VAL_GET(pElem->pData) == 0)) {
uTrace("key:%s %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData,
pCacheObj->numOfElemsInTrash - 1);
STrashElem *p = pElem;
pElem = pElem->next;
taosRemoveFromTrash(pCacheObj, p);
} else {
pElem = pElem->next;
}
}
assert(pCacheObj->numOfElemsInTrash >= 0);
__cache_unlock(pCacheObj);
}
static void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force);
/**
* release node
@ -304,87 +210,20 @@ static FORCE_INLINE SCacheDataNode *taosAddToCacheImpl(SCacheObj *pCacheObj, con
return pNode;
}
static void doCleanupDataCache(SCacheObj *pCacheObj) {
__cache_wr_lock(pCacheObj);
//if (taosHashGetSize(pCacheObj->pHashTable) > 0) {
taosHashCleanup(pCacheObj->pHashTable);
//}
__cache_unlock(pCacheObj);
taosTrashEmpty(pCacheObj, true);
__cache_lock_destroy(pCacheObj);
memset(pCacheObj, 0, sizeof(SCacheObj));
free(pCacheObj);
}
/**
* do cleanup the taos cache
* @param pCacheObj
*/
static void doCleanupDataCache(SCacheObj *pCacheObj);
/**
* refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pCacheObj->refreshTime
* @param handle Cache object handle
*/
static void taosCacheRefresh(void *handle, void *tmrId) {
SCacheObj *pCacheObj = (SCacheObj *)handle;
if (pCacheObj == NULL || T_REF_VAL_GET(pCacheObj) == 0) {
uTrace("object is destroyed. no refresh retry");
return;
}
int16_t ref = T_REF_INC(pCacheObj);
if (ref == 1) {
T_REF_DEC(pCacheObj);
return;
}
// todo add the ref before start the timer
int32_t num = taosHashGetSize(pCacheObj->pHashTable);
if (num == 0) {
ref = T_REF_DEC(pCacheObj);
if (ref == 0) {
doCleanupDataCache(pCacheObj);
} else {
taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
}
return;
}
uint64_t expiredTime = taosGetTimestampMs();
pCacheObj->statistics.refreshCount++;
SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable);
__cache_wr_lock(pCacheObj);
while (taosHashIterNext(pIter)) {
if (pCacheObj->deleting == 1) {
taosHashDestroyIter(pIter);
break;
}
SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) {
taosCacheReleaseNode(pCacheObj, pNode);
}
}
__cache_unlock(pCacheObj);
taosHashDestroyIter(pIter);
static void* taosCacheRefresh(void *handle);
taosTrashEmpty(pCacheObj, false);
ref = T_REF_DEC(pCacheObj);
if (ref == 0) {
doCleanupDataCache(pCacheObj);
return;
} else {
taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
}
}
SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb)(void *data)) {
if (tmrCtrl == NULL || refreshTime <= 0) {
SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) {
if (refreshTime <= 0) {
return NULL;
}
@ -394,7 +233,7 @@ SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb
return NULL;
}
pCacheObj->pHashTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false);
pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false);
if (pCacheObj->pHashTable == NULL) {
free(pCacheObj);
uError("failed to allocate memory, reason:%s", strerror(errno));
@ -406,25 +245,27 @@ SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb
pCacheObj->freeFp = freeCb;
pCacheObj->refreshTime = refreshTime * 1000;
pCacheObj->tmrCtrl = tmrCtrl;
taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
if (__cache_lock_init(pCacheObj) != 0) {
taosTmrStopA(&pCacheObj->pTimer);
taosHashCleanup(pCacheObj->pHashTable);
free(pCacheObj);
uError("failed to init lock, reason:%s", strerror(errno));
return NULL;
}
T_REF_INC(pCacheObj);
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
pthread_create(&pCacheObj->refreshWorker, &thattr, taosCacheRefresh, pCacheObj);
pthread_attr_destroy(&thattr);
return pCacheObj;
}
SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) {
return taosCacheInitWithCb(tmrCtrl, refreshTime, NULL);
SCacheObj *taosCacheInit(int64_t refreshTime) {
return taosCacheInitWithCb(refreshTime, NULL);
}
void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, size_t dataSize, int duration) {
@ -600,16 +441,188 @@ void taosCacheEmpty(SCacheObj *pCacheObj) {
__cache_unlock(pCacheObj);
taosHashDestroyIter(pIter);
taosTrashEmpty(pCacheObj, false);
taosTrashCanEmpty(pCacheObj, false);
}
void taosCacheCleanup(SCacheObj *pCacheObj) {
if (pCacheObj == NULL) {
return;
}
int32_t ref = T_REF_DEC(pCacheObj);
if (ref == 0) {
doCleanupDataCache(pCacheObj);
}
pCacheObj->deleting = 1;
pthread_join(pCacheObj->refreshWorker, NULL);
doCleanupDataCache(pCacheObj);
}
SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size,
uint64_t duration) {
size_t totalSize = size + sizeof(SCacheDataNode) + keyLen + 1;
SCacheDataNode *pNewNode = calloc(1, totalSize);
if (pNewNode == NULL) {
uError("failed to allocate memory, reason:%s", strerror(errno));
return NULL;
}
memcpy(pNewNode->data, pData, size);
pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + size;
pNewNode->keySize = keyLen;
memcpy(pNewNode->key, key, keyLen);
pNewNode->addedTime = (uint64_t)taosGetTimestampMs();
pNewNode->expiredTime = pNewNode->addedTime + duration;
pNewNode->signature = (uint64_t)pNewNode;
pNewNode->size = (uint32_t)totalSize;
return pNewNode;
}
void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
if (pNode->inTrashCan) { /* node is already in trash */
return;
}
STrashElem *pElem = calloc(1, sizeof(STrashElem));
pElem->pData = pNode;
pElem->next = pCacheObj->pTrash;
if (pCacheObj->pTrash) {
pCacheObj->pTrash->prev = pElem;
}
pElem->prev = NULL;
pCacheObj->pTrash = pElem;
pNode->inTrashCan = true;
pCacheObj->numOfElemsInTrash++;
uTrace("key:%s %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash);
}
void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) {
if (pElem->pData->signature != (uint64_t)pElem->pData) {
uError("key:sig:%d %p data has been released, ignore", pElem->pData->signature, pElem->pData);
return;
}
pCacheObj->numOfElemsInTrash--;
if (pElem->prev) {
pElem->prev->next = pElem->next;
} else { /* pnode is the header, update header */
pCacheObj->pTrash = pElem->next;
}
if (pElem->next) {
pElem->next->prev = pElem->prev;
}
pElem->pData->signature = 0;
if (pCacheObj->freeFp) pCacheObj->freeFp(pElem->pData->data);
free(pElem->pData);
free(pElem);
}
void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) {
__cache_wr_lock(pCacheObj);
if (pCacheObj->numOfElemsInTrash == 0) {
if (pCacheObj->pTrash != NULL) {
uError("key:inconsistency data in cache, numOfElem in trash:%d", pCacheObj->numOfElemsInTrash);
}
pCacheObj->pTrash = NULL;
__cache_unlock(pCacheObj);
return;
}
STrashElem *pElem = pCacheObj->pTrash;
while (pElem) {
T_REF_VAL_CHECK(pElem->pData);
if (pElem->next == pElem) {
pElem->next = NULL;
}
if (force || (T_REF_VAL_GET(pElem->pData) == 0)) {
uTrace("key:%s %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData,
pCacheObj->numOfElemsInTrash - 1);
STrashElem *p = pElem;
pElem = pElem->next;
taosRemoveFromTrashCan(pCacheObj, p);
} else {
pElem = pElem->next;
}
}
assert(pCacheObj->numOfElemsInTrash >= 0);
__cache_unlock(pCacheObj);
}
void doCleanupDataCache(SCacheObj *pCacheObj) {
__cache_wr_lock(pCacheObj);
taosHashCleanup(pCacheObj->pHashTable);
__cache_unlock(pCacheObj);
taosTrashCanEmpty(pCacheObj, true);
__cache_lock_destroy(pCacheObj);
memset(pCacheObj, 0, sizeof(SCacheObj));
free(pCacheObj);
}
void* taosCacheRefresh(void *handle) {
SCacheObj *pCacheObj = (SCacheObj *)handle;
if (pCacheObj == NULL) {
uTrace("object is destroyed. no refresh retry");
return NULL;
}
const int32_t SLEEP_DURATION = 500; //500 ms
int64_t totalTick = pCacheObj->refreshTime / SLEEP_DURATION;
int64_t count = 0;
while(1) {
taosMsleep(500);
// check if current cache object will be deleted every 500ms.
if (pCacheObj->deleting) {
break;
}
if (++count < totalTick) {
continue;
}
// reset the count value
count = 0;
size_t num = taosHashGetSize(pCacheObj->pHashTable);
if (num == 0) {
continue;
}
uint64_t expiredTime = taosGetTimestampMs();
pCacheObj->statistics.refreshCount++;
SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable);
__cache_wr_lock(pCacheObj);
while (taosHashIterNext(pIter)) {
SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) {
taosCacheReleaseNode(pCacheObj, pNode);
}
}
__cache_unlock(pCacheObj);
taosHashDestroyIter(pIter);
taosTrashCanEmpty(pCacheObj, false);
}
return NULL;
}

2
src/util/src/tconfig.c
View File

@ -242,7 +242,7 @@ void taosReadGlobalLogCfg() {
wordexp_t full_path;
wordexp(configDir, &full_path, 0);
if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
if (strlen(full_path.we_wordv[0]) > TSDB_FILENAME_LEN - 1) {
if (strlen(full_path.we_wordv[0]) >= TSDB_FILENAME_LEN) {
printf("\nconfig file: %s path overflow max len %d, all variables are set to default\n", full_path.we_wordv[0], TSDB_FILENAME_LEN - 1);
wordfree(&full_path);
return;

303
src/util/src/tkvstore.c
View File

@ -34,42 +34,67 @@
#define TD_KVSTORE_SNAP_SUFFIX ".snap"
#define TD_KVSTORE_NEW_SUFFIX ".new"
typedef struct {
uint64_t uid;
int64_t offset;
int64_t size;
} SKVRecord;
static int tdInitKVStoreHeader(int fd, char *fname);
static void * tdEncodeStoreInfo(void *buf, SStoreInfo *pInfo);
// static void * tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo);
static void * tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo);
static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH);
static char * tdGetKVStoreSnapshotFname(char *fdata);
static char * tdGetKVStoreNewFname(char *fdata);
static void tdFreeKVStore(SKVStore *pStore);
static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo);
static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo);
static void * tdEncodeKVRecord(void *buf, SKVRecord *pRecord);
static void * tdDecodeKVRecord(void *buf, SKVRecord *pRecord);
static int tdRestoreKVStore(SKVStore *pStore);
int tdCreateKVStore(char *fname) {
char *tname = strdup(fname);
if (tname == NULL) return TSDB_CODE_COM_OUT_OF_MEMORY;
int fd = open(fname, O_RDWR | O_CREAT, 0755);
if (fd < 0) {
uError("failed to open file %s since %s", fname, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
int code = tdInitKVStoreHeader(fd, fname);
if (code != TSDB_CODE_SUCCESS) return code;
if (tdInitKVStoreHeader(fd, fname) < 0) {
close(fd);
return -1;
}
if (fsync(fd) < 0) {
uError("failed to fsync file %s since %s", fname, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
close(fd);
return -1;
}
if (close(fd) < 0) {
uError("failed to close file %s since %s", fname, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
return TSDB_CODE_SUCCESS;
return 0;
}
int tdDestroyKVStore(char *fname) {
if (remove(fname) < 0) {
uError("failed to remove file %s since %s", fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
return 0;
}
SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) {
SStoreInfo info = {0};
SKVStore *pStore = tdNewKVStore(fname, iFunc, aFunc, appH);
if (pStore == NULL) return NULL;
@ -89,14 +114,44 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH
goto _err;
}
// TODO: rewind the file
if (tdLoadKVStoreHeader(pStore->sfd, pStore->fsnap, &info) < 0) goto _err;
if (ftruncate(pStore->fd, info.size) < 0) {
uError("failed to truncate %s to " PRId64 " size since %s", pStore->fname, info.size, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err;
close(pStore->sfd);
pStore->sfd = -1;
remove(pStore->fsnap);
}
// TODO: Recover from the file
if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err;
struct stat tfstat;
if (fstat(pStore->fd, &tfstat) < 0) {
uError("failed to fstat file %s since %s", pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
ASSERT(info.size == tfstat.st_size);
if (lseek(pStore->fd, TD_KVSTORE_HEADER_SIZE, SEEK_SET) < 0) {
uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
pStore->info.size += TD_KVSTORE_HEADER_SIZE;
if (tdRestoreKVStore(pStore) < 0) goto _err;
close(pStore->fd);
pStore->fd = -1;
return pStore;
@ -113,7 +168,11 @@ _err:
return NULL;
}
void tdCloseKVStore(SKVStore *pStore) { tdFreeKVStore(pStore); }
int tdKVStoreStartCommit(SKVStore *pStore) {
ASSERT(pStore->fd < 0);
pStore->fd = open(pStore->fname, O_RDWR);
if (pStore->fd < 0) {
uError("failed to open file %s since %s", pStore->fname, strerror(errno));
@ -147,6 +206,14 @@ int tdKVStoreStartCommit(SKVStore *pStore) {
}
pStore->sfd = -1;
if (lseek(pStore->fd, 0, SEEK_END) < 0) {
uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
ASSERT(pStore->info.size == lseek(pStore->fd, 0, SEEK_CUR));
return 0;
_err:
@ -162,11 +229,50 @@ _err:
return -1;
}
int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen) {
SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid));
if (pRecord != NULL) {
pStore->info.tombSize += (pRecord->size + sizeof(SKVRecord));
}
// TODO
return 0;
}
int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) {
SKVRecord rInfo = {0};
char buf[128] = "\0";
SKVRecord *pRecord = taosHashGet(pStore->map, &uid, sizeof(uid));
if (pRecord == NULL) {
uError("failed to drop KV store record with key " PRIu64 " since not find", uid);
return -1;
}
rInfo.offset = -pRecord->offset;
rInfo.uid = pRecord->uid;
rInfo.size = pRecord->size;
void *pBuf = tdEncodeKVRecord(buf, &rInfo);
if (twrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
uError("failed to write %d bytes to file %s since %s", POINTER_DISTANCE(pBuf, buf), pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
pStore->info.size += POINTER_DISTANCE(pBuf, buf);
pStore->info.nDels++;
pStore->info.nRecords--;
pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
return 0;
}
int tdKVStoreEndCommit(SKVStore *pStore) {
ASSERT(pStore->fd > 0);
terrno = tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &(pStore->info));
if (terrno != TSDB_CODE_SUCCESS) return -1;
if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &(pStore->info)) < 0) return -1;
if (fsync(pStore->fd) < 0) {
uError("failed to fsync file %s since %s", pStore->fname, strerror(errno));
@ -179,27 +285,56 @@ int tdKVStoreEndCommit(SKVStore *pStore) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
pStore->fd = -1;
remove(pStore->fsnap);
return 0;
}
static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
char buf[TD_KVSTORE_HEADER_SIZE] = "\0";
if (lseek(fd, 0, SEEK_SET) < 0) {
uError("failed to lseek file %s since %s", fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (tread(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) {
uError("file %s is broken", fname);
terrno = TSDB_CODE_COM_FILE_CORRUPTED;
return -1;
}
tdDecodeStoreInfo(buf, pInfo);
return 0;
}
static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
char buf[TD_KVSTORE_HEADER_SIZE] = "\0";
if (lseek(fd, 0, SEEK_SET) < 0) {
uError("failed to lseek file %s since %s", fname, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
tdEncodeStoreInfo(buf, pInfo);
taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE);
if (twrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
uError("failed to write file %s %d bytes since %s", fname, TD_KVSTORE_HEADER_SIZE, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
return TSDB_CODE_SUCCESS;
return 0;
}
static int tdInitKVStoreHeader(int fd, char *fname) {
@ -217,21 +352,24 @@ static void *tdEncodeStoreInfo(void *buf, SStoreInfo *pInfo) {
return buf;
}
// static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) {
// buf = taosDecodeVariantI64(buf, &(pInfo->size));
// buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
// buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
// buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) {
buf = taosDecodeVariantI64(buf, &(pInfo->size));
buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
// return buf;
// }
return buf;
}
static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) {
SKVStore *pStore = (SKVStore *)malloc(sizeof(SKVStore));
if (pStore == NULL) goto _err;
pStore->fname = strdup(fname);
if (pStore->map == NULL) goto _err;
if (pStore->map == NULL) {
terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
goto _err;
}
pStore->fsnap = tdGetKVStoreSnapshotFname(fname);
if (pStore->fsnap == NULL) goto _err;
@ -254,7 +392,6 @@ static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void
return pStore;
_err:
terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
tdFreeKVStore(pStore);
return NULL;
}
@ -289,4 +426,116 @@ static char *tdGetKVStoreNewFname(char *fdata) {
}
sprintf(fname, "%s%s", fdata, TD_KVSTORE_NEW_SUFFIX);
return fname;
}
static void *tdEncodeKVRecord(void *buf, SKVRecord *pRecord) {
buf = taosEncodeFixedU64(buf, pRecord->uid);
buf = taosEncodeFixedI64(buf, pRecord->offset);
buf = taosEncodeFixedI64(buf, pRecord->size);
return buf;
}
static void *tdDecodeKVRecord(void *buf, SKVRecord *pRecord) {
buf = taosDecodeFixedU64(buf, &(pRecord->uid));
buf = taosDecodeFixedI64(buf, &(pRecord->offset));
buf = taosDecodeFixedI64(buf, &(pRecord->size));
return buf;
}
static int tdRestoreKVStore(SKVStore *pStore) {
char tbuf[128] = "\0";
void * buf = NULL;
int maxBufSize = 0;
SKVRecord rInfo = {0};
ASSERT(TD_KVSTORE_HEADER_SIZE == lseek(pStore->fd, 0, SEEK_CUR));
while (true) {
ssize_t tsize = tread(pStore->fd, tbuf, sizeof(SKVRecord));
if (tsize == 0) break;
if (tsize < sizeof(SKVRecord)) {
uError("failed to read %d bytes from file %s since %s", sizeof(SKVRecord), pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
char *pBuf = tdDecodeKVRecord(tbuf, &rInfo);
ASSERT(POINTER_DISTANCE(pBuf, tbuf) == sizeof(SKVRecord));
ASSERT(rInfo.offset > 0 ? pStore->info.size == rInfo.offset : true);
if (rInfo.offset < 0) {
taosHashRemove(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid));
pStore->info.size += sizeof(SKVRecord);
pStore->info.nRecords--;
pStore->info.nDels++;
pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) + sizeof(SKVRecord));
} else {
// TODO: add statistics
ASSERT(rInfo.offset > 0 && rInfo.size > 0);
if (taosHashPut(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) {
uError("failed to put record in KV store %s", pStore->fname);
terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
goto _err;
}
maxBufSize = MAX(maxBufSize, rInfo.size);
if (lseek(pStore->fd, rInfo.size, SEEK_CUR) < 0) {
uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
}
}
buf = malloc(maxBufSize);
if (buf == NULL) {
uError("failed to allocate %d bytes in KV store %s", maxBufSize, pStore->fname);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
SHashMutableIterator *pIter = taosHashCreateIter(pStore->map);
if (pIter == NULL) {
uError("failed to create hash iter while opening KV store %s", pStore->fname);
terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
goto _err;
}
while (taosHashIterNext(pIter)) {
SKVRecord *pRecord = taosHashIterGet(pIter);
if (lseek(pStore->fd, pRecord->offset, SEEK_SET) < 0) {
uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (tread(pStore->fd, buf, pRecord->size) < pRecord->size) {
uError("failed to read %d bytes from file %s since %s", pRecord->size, pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (!taosCheckChecksumWhole((uint8_t *)buf, pRecord->size)) {
uError("file %s has checksum error, offset " PRId64 " size %d", pStore->fname, pRecord->offset, pRecord->size);
terrno = TSDB_CODE_COM_FILE_CORRUPTED;
goto _err;
}
if (pStore->iFunc) (*pStore->iFunc)(pStore->appH, buf, pRecord->size);
}
taosHashDestroyIter(pIter);
if (pStore->aFunc) (*pStore->aFunc)(pStore->appH);
tfree(buf);
return 0;
_err:
tfree(buf);
return -1;
}

4
src/util/src/tsched.c
View File

@ -15,6 +15,7 @@
#include "os.h"
#include "taosdef.h"
#include "tutil.h"
#include "tulog.h"
#include "tsched.h"
#include "ttimer.h"
@ -62,8 +63,7 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) {
}
pSched->queueSize = queueSize;
strncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow
pSched->label[sizeof(pSched->label)-1] = '\0';
tstrncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow
pSched->fullSlot = 0;
pSched->emptySlot = 0;

11
src/util/src/tsocket.c
View File

@ -29,11 +29,12 @@ int taosGetFqdn(char *fqdn) {
hints.ai_flags = AI_CANONNAME;
getaddrinfo(hostname, NULL, &hints, &result);
int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
if (result) {
strcpy(fqdn, result->ai_canonname);
freeaddrinfo(result);
} else {
uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
code = -1;
}
@ -42,9 +43,12 @@ int taosGetFqdn(char *fqdn) {
uint32_t taosGetIpFromFqdn(const char *fqdn) {
struct addrinfo hints = {0};
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
struct addrinfo *result = NULL;
getaddrinfo(fqdn, NULL, &hints, &result);
int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result);
if (result) {
struct sockaddr *sa = result->ai_addr;
struct sockaddr_in *si = (struct sockaddr_in*)sa;
@ -53,7 +57,8 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) {
freeaddrinfo(result);
return ip;
} else {
return -1;
uError("failed get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret));
return 0xFFFFFFFF;
}
}

40
src/util/src/ttime.c
View File

@ -24,7 +24,6 @@
#include "taosdef.h"
#include "ttime.h"
#include "tutil.h"
/*
* mktime64 - Converts date to seconds.
* Converts Gregorian date to seconds since 1970-01-01 00:00:00.
@ -119,15 +118,21 @@ static int month[12] = {
static int64_t parseFraction(char* str, char** end, int32_t timePrec);
static int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec);
static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec);
static int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec);
static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t timePrec) = {
parseLocaltime,
parseLocaltimeWithDst
};
int32_t taosGetTimestampSec() { return (int32_t)time(NULL); }
int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec) {
int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t daylight) {
/* parse datatime string in with tz */
if (strnchr(timestr, 'T', len, false) != NULL) {
return parseTimeWithTz(timestr, time, timePrec);
} else {
return parseLocaltime(timestr, time, timePrec);
return (*parseLocaltimeFp[daylight])(timestr, time, timePrec);
}
}
@ -304,9 +309,6 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) {
return -1;
}
/* mktime will be affected by TZ, set by using taos_options */
//int64_t seconds = mktime(&tm);
//int64_t seconds = (int64_t)user_mktime(&tm);
int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
int64_t fraction = 0;
@ -324,6 +326,32 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) {
return 0;
}
int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec) {
*time = 0;
struct tm tm = {0};
tm.tm_isdst = -1;
char* str = strptime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
if (str == NULL) {
return -1;
}
/* mktime will be affected by TZ, set by using taos_options */
int64_t seconds = mktime(&tm);
int64_t fraction = 0;
if (*str == '.') {
/* parse the second fraction part */
if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) {
return -1;
}
}
int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : 1000000;
*time = factor * seconds + fraction;
return 0;
}
static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* result) {
*result = val;

3
src/util/src/ttimer.c
View File

@ -552,8 +552,7 @@ void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* lab
return NULL;
}
strncpy(ctrl->label, label, sizeof(ctrl->label));
ctrl->label[sizeof(ctrl->label) - 1] = 0;
tstrncpy(ctrl->label, label, sizeof(ctrl->label));
tmrTrace("%s timer controller is initialized, number of timer controllers: %d.", label, numOfTmrCtrl);
return ctrl;
}

8
src/util/src/tutil.c
View File

@ -582,13 +582,13 @@ bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
}
int taosCheckVersion(char *input_client_version, char *input_server_version, int comparedSegments) {
char client_version[64] = {0};
char server_version[64] = {0};
char client_version[TSDB_VERSION_LEN] = {0};
char server_version[TSDB_VERSION_LEN] = {0};
int clientVersionNumber[4] = {0};
int serverVersionNumber[4] = {0};
strcpy(client_version, input_client_version);
strcpy(server_version, input_server_version);
tstrncpy(client_version, input_client_version, sizeof(client_version));
tstrncpy(server_version, input_server_version, sizeof(server_version));
if (!taosGetVersionNumber(client_version, clientVersionNumber)) {
uError("invalid client version:%s", client_version);

7
src/util/tests/cacheTest.cpp
View File

@ -19,8 +19,7 @@ int32_t tsMaxMeterConnections = 200;
// test cache
TEST(testCase, client_cache_test) {
const int32_t REFRESH_TIME_IN_SEC = 2;
void* tscTmr = taosTmrInit (tsMaxMgmtConnections*2, 200, 6000, "TSC");
SCacheObj* tscCacheHandle = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
SCacheObj* tscCacheHandle = taosCacheInit(REFRESH_TIME_IN_SEC);
const char* key1 = "test1";
char data1[] = "test11";
@ -106,9 +105,7 @@ TEST(testCase, client_cache_test) {
TEST(testCase, cache_resize_test) {
const int32_t REFRESH_TIME_IN_SEC = 2;
void* tscTmr = taosTmrInit (1000*2, 200, 6000, "TSC");
auto* pCache = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
auto* pCache = taosCacheInit(REFRESH_TIME_IN_SEC);
char key[256] = {0};
char data[1024] = "abcdefghijk";

2
src/vnode/inc/vnodeInt.h
View File

@ -51,7 +51,7 @@ typedef struct {
SSyncCfg syncCfg;
SWalCfg walCfg;
char *rootDir;
char db[TSDB_DB_NAME_LEN + 1];
char db[TSDB_DB_NAME_LEN];
} SVnodeObj;
int vnodeWriteToQueue(void *param, void *pHead, int type);

47
src/vnode/src/vnodeMain.c
View File

@ -321,6 +321,22 @@ void vnodeRelease(void *pVnodeRaw) {
return;
}
if (pVnode->tsdb)
tsdbCloseRepo(pVnode->tsdb, 1);
pVnode->tsdb = NULL;
if (pVnode->wal)
walClose(pVnode->wal);
pVnode->wal = NULL;
if (pVnode->wqueue)
dnodeFreeVnodeWqueue(pVnode->wqueue);
pVnode->wqueue = NULL;
if (pVnode->rqueue)
dnodeFreeVnodeRqueue(pVnode->rqueue);
pVnode->rqueue = NULL;
tfree(pVnode->rootDir);
if (pVnode->status == TAOS_VN_STATUS_DELETING) {
@ -381,16 +397,18 @@ void *vnodeGetWal(void *pVnode) {
static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
if (pVnode->status == TAOS_VN_STATUS_DELETING) return;
if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
int64_t totalStorage, compStorage, pointsWritten = 0;
tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage);
SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
pLoad->vgId = htonl(pVnode->vgId);
pLoad->cfgVersion = htonl(pVnode->cfgVersion);
pLoad->totalStorage = htobe64(pLoad->totalStorage);
pLoad->compStorage = htobe64(pLoad->compStorage);
pLoad->pointsWritten = htobe64(pLoad->pointsWritten);
pLoad->totalStorage = htobe64(totalStorage);
pLoad->compStorage = htobe64(compStorage);
pLoad->pointsWritten = htobe64(pointsWritten);
pLoad->status = pVnode->status;
pLoad->role = pVnode->role;
pLoad->replica = pVnode->syncCfg.replica;
pLoad->replica = pVnode->syncCfg.replica;
}
void vnodeBuildStatusMsg(void *param) {
@ -409,33 +427,21 @@ void vnodeBuildStatusMsg(void *param) {
}
static void vnodeCleanUp(SVnodeObj *pVnode) {
// remove from hash, so new messages wont be consumed
taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
// stop replication module
if (pVnode->sync) {
syncStop(pVnode->sync);
pVnode->sync = NULL;
}
if (pVnode->tsdb)
tsdbCloseRepo(pVnode->tsdb, 1);
pVnode->tsdb = NULL;
if (pVnode->wal)
walClose(pVnode->wal);
pVnode->wal = NULL;
// stop continuous query
if (pVnode->cq)
cqClose(pVnode->cq);
pVnode->cq = NULL;
if (pVnode->wqueue)
dnodeFreeVnodeWqueue(pVnode->wqueue);
pVnode->wqueue = NULL;
if (pVnode->rqueue)
dnodeFreeVnodeRqueue(pVnode->rqueue);
pVnode->rqueue = NULL;
// release local resources only after cutting off outside connections
vnodeRelease(pVnode);
}
@ -551,6 +557,7 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
fflush(fp);
fclose(fp);
free(content);

87
src/vnode/src/vnodeRead.c
View File

@ -27,17 +27,18 @@
#include "vnodeLog.h"
#include "query.h"
static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *pCont, int32_t contLen, SRspRet *pRet);
static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet);
static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet);
static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SReadMsg *pReadMsg);
static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg);
static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg);
void vnodeInitReadFp(void) {
vnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg;
vnodeProcessReadMsgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessFetchMsg;
}
int32_t vnodeProcessRead(void *param, int msgType, void *pCont, int32_t contLen, SRspRet *ret) {
int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) {
SVnodeObj *pVnode = (SVnodeObj *)param;
int msgType = pReadMsg->rpcMsg.msgType;
if (vnodeProcessReadMsgFp[msgType] == NULL) {
vTrace("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]);
@ -55,16 +56,46 @@ int32_t vnodeProcessRead(void *param, int msgType, void *pCont, int32_t contLen,
return TSDB_CODE_RPC_NOT_READY;
}
return (*vnodeProcessReadMsgFp[msgType])(pVnode, pCont, contLen, ret);
return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg);
}
static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet) {
// notify connection(handle) that current qhandle is created, if current connection from
// client is broken, the query needs to be killed immediately.
static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) {
SRetrieveTableMsg* killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg));
killQueryMsg->qhandle = htobe64((uint64_t) qhandle);
killQueryMsg->free = htons(1);
killQueryMsg->header.vgId = htonl(vgId);
killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));
vTrace("QInfo:%p register qhandle to connect:%p", qhandle, handle);
return rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg));
}
static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
void * pCont = pReadMsg->pCont;
int32_t contLen = pReadMsg->contLen;
SRspRet *pRet = &pReadMsg->rspRet;
SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pCont;
memset(pRet, 0, sizeof(SRspRet));
int32_t code = TSDB_CODE_SUCCESS;
// qHandle needs to be freed correctly
if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
SRetrieveTableMsg* killQueryMsg = (SRetrieveTableMsg*) pReadMsg->pCont;
killQueryMsg->free = htons(killQueryMsg->free);
killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
vWarn("QInfo:%p connection %p broken, kill query", killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1);
qKillQuery((qinfo_t) killQueryMsg->qhandle);
return TSDB_CODE_TSC_QUERY_CANCELLED; // todo change the error code
}
int32_t code = TSDB_CODE_SUCCESS;
qinfo_t pQInfo = NULL;
if (contLen != 0) {
code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo);
@ -74,7 +105,19 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t cont
pRet->len = sizeof(SQueryTableRsp);
pRet->rsp = pRsp;
// current connect is broken
if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, pQInfo, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vError("vgId:%d, QInfo:%p, dnode query discarded since link is broken, %p", pVnode->vgId, pQInfo, pReadMsg->rpcMsg.handle);
pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
//NOTE: there two refcount, needs to kill twice, todo refactor
qKillQuery(pQInfo);
qKillQuery(pQInfo);
return pRsp->code;
}
vTrace("vgId:%d, QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo);
} else {
assert(pCont != NULL);
@ -91,13 +134,34 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t cont
return code;
}
static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet) {
static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
void * pCont = pReadMsg->pCont;
SRspRet *pRet = &pReadMsg->rspRet;
SRetrieveTableMsg *pRetrieve = pCont;
void *pQInfo = (void*) htobe64(pRetrieve->qhandle);
pRetrieve->free = htons(pRetrieve->free);
memset(pRet, 0, sizeof(SRspRet));
if (pRetrieve->free == 1) {
vTrace("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, pQInfo);
int32_t ret = qKillQuery(pQInfo);
pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
pRet->len = sizeof(SRetrieveTableRsp);
memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
SRetrieveTableRsp* pRsp = pRet->rsp;
pRsp->numOfRows = 0;
pRsp->completed = true;
pRsp->useconds = 0;
return ret;
}
vTrace("vgId:%d, QInfo:%p, retrieve msg is received", pVnode->vgId, pQInfo);
int32_t code = qRetrieveQueryResultInfo(pQInfo);
if (code != TSDB_CODE_SUCCESS) {
//TODO
@ -110,8 +174,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, void *pCont, int32_t cont
if (qHasMoreResultsToRetrieve(pQInfo)) {
pRet->qhandle = pQInfo;
code = TSDB_CODE_VND_ACTION_NEED_REPROCESSED;
} else {
// no further execution invoked, release the ref to vnode
} else { // no further execution invoked, release the ref to vnode
qDestroyQueryInfo(pQInfo);
vnodeRelease(pVnode);
}

22
src/wal/src/walMain.c
View File

@ -44,7 +44,7 @@ typedef struct {
uint32_t id; // increase continuously
int num; // number of wal files
char path[TSDB_FILENAME_LEN];
char name[TSDB_FILENAME_LEN];
char name[TSDB_FILENAME_LEN+16];
pthread_mutex_t mutex;
} SWal;
@ -108,7 +108,7 @@ void walClose(void *handle) {
if (pWal->keep == 0) {
// remove all files in the directory
for (int i=0; i<pWal->num; ++i) {
sprintf(pWal->name, "%s/%s%d", pWal->path, walPrefix, pWal->id-i);
snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id-i);
if (remove(pWal->name) <0) {
wError("wal:%s, failed to remove", pWal->name);
} else {
@ -140,7 +140,7 @@ int walRenew(void *handle) {
pWal->num++;
sprintf(pWal->name, "%s/%s%d", pWal->path, walPrefix, pWal->id);
snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id);
pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (pWal->fd < 0) {
@ -152,7 +152,7 @@ int walRenew(void *handle) {
if (pWal->num > pWal->max) {
// remove the oldest wal file
char name[TSDB_FILENAME_LEN * 3];
sprintf(name, "%s/%s%d", pWal->path, walPrefix, pWal->id - pWal->max);
snprintf(name, sizeof(name), "%s/%s%d", pWal->path, walPrefix, pWal->id - pWal->max);
if (remove(name) <0) {
wError("wal:%s, failed to remove(%s)", name, strerror(errno));
} else {
@ -214,7 +214,7 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
int plen = strlen(walPrefix);
char opath[TSDB_FILENAME_LEN+5];
int slen = sprintf(opath, "%s", pWal->path);
int slen = snprintf(opath, sizeof(opath), "%s", pWal->path);
if ( pWal->keep == 0)
strcpy(opath+slen, "/old");
@ -245,7 +245,7 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
wTrace("wal:%s, %d files will be restored", opath, count);
for (index = minId; index<=maxId; ++index) {
sprintf(pWal->name, "%s/%s%d", opath, walPrefix, index);
snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, index);
terrno = walRestoreWalFile(pWal, pVnode, writeFp);
if (terrno < 0) break;
}
@ -264,7 +264,7 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
// open the existing WAL file in append mode
pWal->num = count;
pWal->id = maxId;
sprintf(pWal->name, "%s/%s%d", opath, walPrefix, maxId);
snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, maxId);
pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
if (pWal->fd < 0) {
wError("wal:%s, failed to open file(%s)", pWal->name, strerror(errno));
@ -361,7 +361,7 @@ int walHandleExistingFiles(const char *path) {
char nname[TSDB_FILENAME_LEN * 3];
char opath[TSDB_FILENAME_LEN];
sprintf(opath, "%s/old", path);
snprintf(opath, sizeof(opath), "%s/old", path);
struct dirent *ent;
DIR *dir = opendir(path);
@ -377,8 +377,8 @@ int walHandleExistingFiles(const char *path) {
int count = 0;
while ((ent = readdir(dir))!= NULL) {
if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
sprintf(oname, "%s/%s", path, ent->d_name);
sprintf(nname, "%s/old/%s", path, ent->d_name);
snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name);
if (access(opath, F_OK) != 0) {
if (mkdir(opath, 0755) != 0) {
wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
@ -416,7 +416,7 @@ static int walRemoveWalFiles(const char *path) {
while ((ent = readdir(dir))!= NULL) {
if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
sprintf(name, "%s/%s", path, ent->d_name);
snprintf(name, sizeof(name), "%s/%s", path, ent->d_name);
if (remove(name) <0) {
wError("wal:%s, failed to remove(%s)", name, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);

12
tests/examples/c/demo.c
View File

@ -30,7 +30,7 @@ static int32_t doQuery(TAOS* taos, const char* sql) {
TAOS_RES* res = taos_query(taos, sql);
if (taos_errno(res) != 0) {
printf("failed to execute query, reason:%s\n", taos_errstr(res));
printf("failed to execute query, reason:%s\n", taos_errstr(taos));
return -1;
}
@ -77,7 +77,7 @@ static __attribute__((unused)) void multiThreadTest(int32_t numOfThreads, void*
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
pthread_t* threadId = malloc(sizeof(pthread_t)*numOfThreads);
pthread_t* threadId = (pthread_t*)malloc(sizeof(pthread_t)*(uint32_t)numOfThreads);
for (int i = 0; i < numOfThreads; ++i) {
pthread_create(&threadId[i], NULL, oneLoader, conn);
@ -115,15 +115,15 @@ int main(int argc, char *argv[]) {
printf("success to connect to server\n");
// doQuery(taos, "select c1,count(*) from group_db0.group_mt0 where c1<8 group by c1");
doQuery(taos, "select * from test.m1");
// doQuery(taos, "select * from test.m1");
// multiThreadTest(1, taos);
// doQuery(taos, "select tbname from test.m1");
// doQuery(taos, "select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_db0.lm2_stb0 where ts >= 1537146000000 and ts <= 1543145400000 and tbname in ('lm2_tb0') interval(1s) group by t1");
// doQuery(taos, "select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_db0.lm2_stb0 where ts >= 1537146000000 and ts <= 1543145400000 and tbname in ('lm2_tb0', 'lm2_tb1', 'lm2_tb2') interval(1s)");
// for(int32_t i = 0; i < 100000; ++i) {
// doQuery(taos, "insert into t1 values(now, 2)");
// }
for(int32_t i = 0; i < 200; ++i) {
doQuery(taos, "select * from lm2_db0.lm2_stb0");
}
// doQuery(taos, "create table t1(ts timestamp, k binary(12), f nchar(2))");
taos_close(taos);

Some files were not shown because too many files have changed in this diff Show More