xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/crash_gen

This commit is contained in:
Steven Li 2020-06-09 23:22:09 -07:00
parent ac1fb970d5 862144c59b
commit dd5e9774e4
159 changed files with 4998 additions and 978 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/client/inc/tsclient.h
View File

@ -285,8 +285,8 @@ typedef struct STscObj {
void * pTimer; void * pTimer;
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN]; char pass[TSDB_KEY_LEN];
char acctId[TSDB_DB_NAME_LEN]; char acctId[TSDB_ACCT_LEN];
char db[TSDB_TABLE_ID_LEN]; char db[TSDB_DB_NAME_LEN];
char sversion[TSDB_VERSION_LEN]; char sversion[TSDB_VERSION_LEN];
char writeAuth : 1; char writeAuth : 1;
char superAuth : 1; char superAuth : 1;

20
src/client/src/tscLocal.c
View File

@ -132,7 +132,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i; char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, TSDB_COL_NAME_LEN); STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, TSDB_COL_NAME_LEN - 1);
char *type = tDataTypeDesc[pSchema[i].type].aName; char *type = tDataTypeDesc[pSchema[i].type].aName;
@ -171,7 +171,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
// field name // field name
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i; char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, TSDB_COL_NAME_LEN); STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, TSDB_COL_NAME_LEN - 1);
// type name // type name
pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1);
@ -211,18 +211,18 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
pQueryInfo->order.order = TSDB_ORDER_ASC; pQueryInfo->order.order = TSDB_ORDER_ASC;
TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE}; TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE};
strncpy(f.name, "Field", TSDB_COL_NAME_LEN); tstrncpy(f.name, "Field", sizeof(f.name));
SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE, TSDB_COL_NAME_LEN, false); (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, (TSDB_COL_NAME_LEN - 1), false);
rowLen += (TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE); rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE);
f.bytes = typeColLength; f.bytes = typeColLength;
f.type = TSDB_DATA_TYPE_BINARY; f.type = TSDB_DATA_TYPE_BINARY;
strncpy(f.name, "Type", TSDB_COL_NAME_LEN); tstrncpy(f.name, "Type", sizeof(f.name));
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength,
@ -232,7 +232,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
f.bytes = sizeof(int32_t); f.bytes = sizeof(int32_t);
f.type = TSDB_DATA_TYPE_INT; f.type = TSDB_DATA_TYPE_INT;
strncpy(f.name, "Length", TSDB_COL_NAME_LEN); tstrncpy(f.name, "Length", sizeof(f.name));
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t),
@ -242,7 +242,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
f.bytes = noteColLength; f.bytes = noteColLength;
f.type = TSDB_DATA_TYPE_BINARY; f.type = TSDB_DATA_TYPE_BINARY;
strncpy(f.name, "Note", TSDB_COL_NAME_LEN); tstrncpy(f.name, "Note", sizeof(f.name));
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength,
@ -286,7 +286,7 @@ static void tscProcessCurrentUser(SSqlObj *pSql) {
} }
static void tscProcessCurrentDB(SSqlObj *pSql) { static void tscProcessCurrentDB(SSqlObj *pSql) {
char db[TSDB_DB_NAME_LEN + 1] = {0}; char db[TSDB_DB_NAME_LEN] = {0};
extractDBName(pSql->pTscObj->db, db); extractDBName(pSql->pTscObj->db, db);
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);

8
src/client/src/tscParseInsert.c
View File

@ -795,7 +795,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX);
tscSetTableFullName(pSTableMeterMetaInfo, &sToken, pSql); tscSetTableFullName(pSTableMeterMetaInfo, &sToken, pSql);
strncpy(pTag->name, pSTableMeterMetaInfo->name, TSDB_TABLE_ID_LEN); tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name));
code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); code = tscGetTableMeta(pSql, pSTableMeterMetaInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
@ -989,7 +989,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
int validateTableName(char *tblName, int len) { int validateTableName(char *tblName, int len) {
char buf[TSDB_TABLE_ID_LEN] = {0}; char buf[TSDB_TABLE_ID_LEN] = {0};
strncpy(buf, tblName, len); tstrncpy(buf, tblName, sizeof(buf));
SSQLToken token = {.n = len, .type = TK_ID, .z = buf}; SSQLToken token = {.n = len, .type = TK_ID, .z = buf};
tSQLGetToken(buf, &token.type); tSQLGetToken(buf, &token.type);
@ -1512,7 +1512,7 @@ void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) {
} }
pCmd->count = 1; pCmd->count = 1;
strncpy(path, pDataBlock->filename, PATH_MAX); tstrncpy(path, pDataBlock->filename, sizeof(path));
FILE *fp = fopen(path, "r"); FILE *fp = fopen(path, "r");
if (fp == NULL) { if (fp == NULL) {
@ -1520,7 +1520,7 @@ void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) {
continue; continue;
} }
strncpy(pTableMetaInfo->name, pDataBlock->tableId, TSDB_TABLE_ID_LEN); tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
memset(pDataBlock->pData, 0, pDataBlock->nAllocSize); memset(pDataBlock->pData, 0, pDataBlock->nAllocSize);
int32_t ret = tscGetTableMeta(pSql, pTableMetaInfo); int32_t ret = tscGetTableMeta(pSql, pTableMetaInfo);

6
src/client/src/tscProfile.c
View File

@ -239,8 +239,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
continue; continue;
} }
strncpy(pQdesc->sql, pSql->sqlstr, TSDB_SHOW_SQL_LEN - 1); tstrncpy(pQdesc->sql, pSql->sqlstr, sizeof(pQdesc->sql));
pQdesc->sql[TSDB_SHOW_SQL_LEN - 1] = 0;
pQdesc->stime = htobe64(pSql->stime); pQdesc->stime = htobe64(pSql->stime);
pQdesc->queryId = htonl(pSql->queryId); pQdesc->queryId = htonl(pSql->queryId);
pQdesc->useconds = htobe64(pSql->res.useconds); pQdesc->useconds = htobe64(pSql->res.useconds);
@ -256,8 +255,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
SSqlStream *pStream = pObj->streamList; SSqlStream *pStream = pObj->streamList;
while (pStream) { while (pStream) {
strncpy(pSdesc->sql, pStream->pSql->sqlstr, TSDB_SHOW_SQL_LEN - 1); tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql));
pSdesc->sql[TSDB_SHOW_SQL_LEN - 1] = 0;
pSdesc->streamId = htonl(pStream->streamId); pSdesc->streamId = htonl(pStream->streamId);
pSdesc->num = htobe64(pStream->num); pSdesc->num = htobe64(pStream->num);

57
src/client/src/tscSQLParser.c
View File

@ -348,7 +348,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
} }
if (pToken->n > TSDB_TABLE_NAME_LEN) { if (pToken->n >= TSDB_TABLE_NAME_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
} }
@ -933,7 +933,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
SSchema* pSchema = tscGetTableSchema(pTableMeta); SSchema* pSchema = tscGetTableSchema(pTableMeta);
for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { for (int32_t i = 0; i < numOfTags + numOfCols; ++i) {
if (strncasecmp(pTagField->name, pSchema[i].name, TSDB_COL_NAME_LEN) == 0) { if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
return false; return false;
} }
@ -993,7 +993,7 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
// field name must be unique // field name must be unique
for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { for (int32_t i = 0; i < numOfTags + numOfCols; ++i) {
if (strncasecmp(pColField->name, pSchema[i].name, TSDB_COL_NAME_LEN) == 0) { if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
return false; return false;
} }
@ -1005,7 +1005,8 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
/* is contained in pFieldList or not */ /* is contained in pFieldList or not */
static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) { static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) {
for (int32_t j = startIdx; j < pFieldList->nField; ++j) { for (int32_t j = startIdx; j < pFieldList->nField; ++j) {
if (strncasecmp(name, pFieldList->p[j].name, TSDB_COL_NAME_LEN) == 0) return true; TAOS_FIELD* field = pFieldList->p + j;
if (strncasecmp(name, field->name, sizeof(field->name) - 1) == 0) return true;
} }
return false; return false;
@ -1041,7 +1042,7 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL
/* db name is not specified, the tableName dose not include db name */ /* db name is not specified, the tableName dose not include db name */
if (pDB != NULL) { if (pDB != NULL) {
if (pDB->n > TSDB_DB_NAME_LEN) { if (pDB->n >= TSDB_DB_NAME_LEN) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
@ -1060,7 +1061,7 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL
} }
} else { // pDB == NULL, the db prefix name is specified in tableName } else { // pDB == NULL, the db prefix name is specified in tableName
/* the length limitation includes tablename + dbname + sep */ /* the length limitation includes tablename + dbname + sep */
if (tableName->n > TSDB_TABLE_NAME_LEN + TSDB_DB_NAME_LEN + tListLen(TS_PATH_DELIMITER)) { if (tableName->n > (TSDB_TABLE_NAME_LEN - 1) + (TSDB_DB_NAME_LEN - 1) + sizeof(TS_PATH_DELIMITER)) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
} }
@ -1166,7 +1167,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
/* todo alias name should use the original sql string */ /* todo alias name should use the original sql string */
char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr;
strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN); tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName));
tExprNode* pNode = NULL; tExprNode* pNode = NULL;
SArray* colList = taosArrayInit(10, sizeof(SColIndex)); SArray* colList = taosArrayInit(10, sizeof(SColIndex));
@ -1308,7 +1309,7 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName;
strncpy(pExpr->aliasName, colName, tListLen(pExpr->aliasName)); tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName));
SColumnList ids = {0}; SColumnList ids = {0};
ids.num = 1; ids.num = 1;
@ -1358,7 +1359,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum
for (int32_t j = 0; j < numOfTotalColumns; ++j) { for (int32_t j = 0; j < numOfTotalColumns; ++j) {
SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos + j, j, pIndex->tableIndex); SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos + j, j, pIndex->tableIndex);
strncpy(pExpr->aliasName, pSchema[j].name, tListLen(pExpr->aliasName)); tstrncpy(pExpr->aliasName, pSchema[j].name, sizeof(pExpr->aliasName));
pIndex->columnIndex = j; pIndex->columnIndex = j;
SColumnList ids = {0}; SColumnList ids = {0};
@ -1401,7 +1402,7 @@ int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pI
} }
if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE}; SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE};
strcpy(colSchema.name, TSQL_TBNAME_L); strcpy(colSchema.name, TSQL_TBNAME_L);
tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, true); tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, true);
@ -1448,11 +1449,11 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema,
if (aliasName != NULL) { if (aliasName != NULL) {
strcpy(columnName, aliasName); strcpy(columnName, aliasName);
} else { } else {
getRevisedName(columnName, functionID, TSDB_COL_NAME_LEN, pSchema[pColIndex->columnIndex].name); getRevisedName(columnName, functionID, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name);
} }
SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false); SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false);
strncpy(pExpr->aliasName, columnName, tListLen(pExpr->aliasName)); tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName));
// for all queries, the timestamp column needs to be loaded // for all queries, the timestamp column needs to be loaded
SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
@ -1536,7 +1537,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
} }
memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
getColumnName(pItem, pExpr->aliasName, TSDB_COL_NAME_LEN); getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1);
SColumnList ids = getColumnList(1, index.tableIndex, index.columnIndex); SColumnList ids = getColumnList(1, index.tableIndex, index.columnIndex);
if (finalResult) { if (finalResult) {
@ -1651,7 +1652,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
ids.ids[0] = index; ids.ids[0] = index;
memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
getColumnName(pItem, pExpr->aliasName, TSDB_COL_NAME_LEN); getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1);
if (finalResult) { if (finalResult) {
int32_t numOfOutput = tscNumOfFields(pQueryInfo); int32_t numOfOutput = tscNumOfFields(pQueryInfo);
@ -1851,7 +1852,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
} }
memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
getColumnName(pItem, pExpr->aliasName, TSDB_COL_NAME_LEN); getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1);
SColumnList ids = getColumnList(1, 0, index.columnIndex); SColumnList ids = getColumnList(1, 0, index.columnIndex);
if (finalResult) { if (finalResult) {
@ -2202,7 +2203,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
if (pDbPrefixToken->type != 0) { if (pDbPrefixToken->type != 0) {
assert(pDbPrefixToken->n >= 0); assert(pDbPrefixToken->n >= 0);
if (pDbPrefixToken->n > TSDB_DB_NAME_LEN) { // db name is too long if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
} }
@ -2229,7 +2230,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
} }
if (pCmd->payloadLen > TSDB_TABLE_NAME_LEN) { if (pCmd->payloadLen >= TSDB_TABLE_NAME_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
} }
} }
@ -3997,7 +3998,7 @@ int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name;
for (int32_t j = 0; j < TSDB_COL_NAME_LEN && fieldName[j] != 0; ++j) { for (int32_t j = 0; j < (TSDB_COL_NAME_LEN - 1) && fieldName[j] != 0; ++j) {
for (int32_t k = 0; k < tListLen(rep); ++k) { for (int32_t k = 0; k < tListLen(rep); ++k) {
if (fieldName[j] == rep[k]) { if (fieldName[j] == rep[k]) {
fieldName[j] = '_'; fieldName[j] = '_';
@ -4013,7 +4014,7 @@ int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name;
for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutput; ++j) { for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutput; ++j) {
if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, TSDB_COL_NAME_LEN) == 0) { if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, (TSDB_COL_NAME_LEN - 1)) == 0) {
const char* msg = "duplicated column name in new table"; const char* msg = "duplicated column name in new table";
return invalidSqlErrMsg(pQueryInfo->msg, msg); return invalidSqlErrMsg(pQueryInfo->msg, msg);
} }
@ -4368,7 +4369,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
} }
tVariantListItem* pItem = &pAlterSQL->varList->a[0]; tVariantListItem* pItem = &pAlterSQL->varList->a[0];
if (pItem->pVar.nLen > TSDB_COL_NAME_LEN) { if (pItem->pVar.nLen >= TSDB_COL_NAME_LEN) {
return invalidSqlErrMsg(pQueryInfo->msg, msg9); return invalidSqlErrMsg(pQueryInfo->msg, msg9);
} }
@ -4420,7 +4421,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
char name[TSDB_COL_NAME_LEN + 1] = {0}; char name[TSDB_COL_NAME_LEN] = {0};
strncpy(name, pVarList->a[0].pVar.pz, pVarList->a[0].pVar.nLen); strncpy(name, pVarList->a[0].pVar.pz, pVarList->a[0].pVar.nLen);
TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
@ -4522,8 +4523,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(pQueryInfo->msg, msg18); return invalidSqlErrMsg(pQueryInfo->msg, msg18);
} }
char name1[TSDB_COL_NAME_LEN + 1] = {0}; char name1[TSDB_COL_NAME_LEN] = {0};
strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen); tstrncpy(name1, pItem->pVar.pz, sizeof(name1));
TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
} }
@ -5232,7 +5233,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) {
int16_t colIndex = pColIndex->colIndex; int16_t colIndex = pColIndex->colIndex;
if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
type = TSDB_DATA_TYPE_BINARY; type = TSDB_DATA_TYPE_BINARY;
bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; // todo extract method bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE; // todo extract method
name = TSQL_TBNAME_L; name = TSQL_TBNAME_L;
} else { } else {
if (TSDB_COL_IS_TAG(pColIndex->flag)) { if (TSDB_COL_IS_TAG(pColIndex->flag)) {
@ -5254,8 +5255,8 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) {
SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex};
SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true); SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true);
memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); memset(pExpr->aliasName, 0, sizeof(pExpr->aliasName));
strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN); tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName));
pExpr->colInfo.flag = TSDB_COL_TAG; pExpr->colInfo.flag = TSDB_COL_TAG;
@ -5419,7 +5420,7 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false); tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false);
const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name; const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name;
strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName)); tstrncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -6019,7 +6020,7 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray*
if (pCols != NULL) { // record the involved columns if (pCols != NULL) { // record the involved columns
SColIndex colIndex = {0}; SColIndex colIndex = {0};
strncpy(colIndex.name, pSchema->name, TSDB_COL_NAME_LEN); tstrncpy(colIndex.name, pSchema->name, sizeof(colIndex.name));
colIndex.colId = pSchema->colId; colIndex.colId = pSchema->colId;
colIndex.colIndex = index.columnIndex; colIndex.colIndex = index.columnIndex;

2
src/client/src/tscSchemaUtil.c
View File

@ -115,7 +115,7 @@ bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols) {
// 3. valid column names // 3. valid column names
for (int32_t j = i + 1; j < numOfCols; ++j) { for (int32_t j = i + 1; j < numOfCols; ++j) {
if (strncasecmp(pSchema[i].name, pSchema[j].name, TSDB_COL_NAME_LEN) == 0) { if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) {
return false; return false;
} }
} }

41
src/client/src/tscSecondaryMerge.c
View File

@ -324,7 +324,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
tfree(pReducer->discardData); tfree(pReducer->discardData);
tfree(pReducer->pResultBuf); tfree(pReducer->pResultBuf);
tfree(pReducer->pFinalRes); tfree(pReducer->pFinalRes);
// tfree(pReducer->pBufForInterpo);
tfree(pReducer->prevRowOfInput); tfree(pReducer->prevRowOfInput);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -363,7 +362,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (pQueryInfo->fillType != TSDB_FILL_NONE) { if (pQueryInfo->fillType != TSDB_FILL_NONE) {
SFillColInfo* pFillCol = createFillColInfo(pQueryInfo); SFillColInfo* pFillCol = createFillColInfo(pQueryInfo);
pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
4096, numOfCols, pQueryInfo->slidingTime, pQueryInfo->fillType, pFillCol); 4096, numOfCols, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit,
tinfo.precision, pQueryInfo->fillType, pFillCol);
} }
int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols; int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
@ -494,7 +494,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
tscTrace("%p waiting for delete procedure, status: %d", pSql, status); tscTrace("%p waiting for delete procedure, status: %d", pSql, status);
} }
taosDestoryFillInfo(pLocalReducer->pFillInfo); pLocalReducer->pFillInfo = taosDestoryFillInfo(pLocalReducer->pFillInfo);
if (pLocalReducer->pCtx != NULL) { if (pLocalReducer->pCtx != NULL) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
@ -980,8 +980,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
} }
/* all output for current group are completed */ /* all output for current group are completed */
int32_t totalRemainRows = int32_t totalRemainRows = getFilledNumOfRes(pFillInfo, actualETime, pLocalReducer->resColModel->capacity);
taosGetNumOfResultWithFill(pFillInfo, rpoints, pFillInfo->slidingTime, actualETime);
if (totalRemainRows <= 0) { if (totalRemainRows <= 0) {
break; break;
} }
@ -1267,13 +1266,7 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no
SFillInfo* pFillInfo = pLocalReducer->pFillInfo; SFillInfo* pFillInfo = pLocalReducer->pFillInfo;
if (pFillInfo != NULL) { if (pFillInfo != NULL) {
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); taosFillSetStartInfo(pFillInfo, pResBuf->num, pQueryInfo->window.ekey);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
TSKEY ekey = taosGetRevisedEndKey(pQueryInfo->window.ekey, pFillInfo->order, pFillInfo->slidingTime,
pQueryInfo->slidingTimeUnit, tinfo.precision);
taosFillSetStartInfo(pFillInfo, pResBuf->num, ekey);
taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf); taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf);
} }
@ -1327,23 +1320,15 @@ static bool doBuildFilledResultForGroup(SSqlObj *pSql) {
SLocalReducer *pLocalReducer = pRes->pLocalReducer; SLocalReducer *pLocalReducer = pRes->pLocalReducer;
SFillInfo *pFillInfo = pLocalReducer->pFillInfo; SFillInfo *pFillInfo = pLocalReducer->pFillInfo;
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
int8_t p = tinfo.precision;
if (pFillInfo != NULL && taosNumOfRemainRows(pFillInfo) > 0) { if (pFillInfo != NULL && taosNumOfRemainRows(pFillInfo) > 0) {
assert(pQueryInfo->fillType != TSDB_FILL_NONE); assert(pQueryInfo->fillType != TSDB_FILL_NONE);
tFilePage *pFinalDataBuf = pLocalReducer->pResultBuf; tFilePage *pFinalDataBuf = pLocalReducer->pResultBuf;
int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1)); int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1));
int32_t remain = taosNumOfRemainRows(pFillInfo);
TSKEY ekey = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit, p);
// the first column must be the timestamp column // the first column must be the timestamp column
int32_t rows = taosGetNumOfResultWithFill(pFillInfo, remain, ekey, pLocalReducer->resColModel->capacity); int32_t rows = getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
if (rows > 0) { // do interpo if (rows > 0) { // do fill gap
doFillResult(pSql, pLocalReducer, false); doFillResult(pSql, pLocalReducer, false);
} }
@ -1362,10 +1347,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow; bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow;
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL || if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL ||
prevGroupCompleted) { prevGroupCompleted) {
@ -1373,9 +1355,8 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
if (pQueryInfo->fillType != TSDB_FILL_NONE) { if (pQueryInfo->fillType != TSDB_FILL_NONE) {
int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey;
etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, assert(pFillInfo->numOfRows == 0);
pQueryInfo->slidingTimeUnit, tinfo.precision); int32_t rows = getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
int32_t rows = taosGetNumOfResultWithFill(pFillInfo, 0, etime, pLocalReducer->resColModel->capacity);
if (rows > 0) { // do interpo if (rows > 0) { // do interpo
doFillResult(pSql, pLocalReducer, true); doFillResult(pSql, pLocalReducer, true);
} }

28
src/client/src/tscServer.c
View File

@ -866,7 +866,7 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
assert(pCmd->numOfClause == 1); assert(pCmd->numOfClause == 1);
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strncpy(pCreateDbMsg->db, pTableMetaInfo->name, tListLen(pCreateDbMsg->db)); tstrncpy(pCreateDbMsg->db, pTableMetaInfo->name, sizeof(pCreateDbMsg->db));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1018,7 +1018,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload; SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strcpy(pDrop->ep, pTableMetaInfo->name); tstrncpy(pDrop->ep, pTableMetaInfo->name, sizeof(pDrop->ep));
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE; pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1036,7 +1036,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload; SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strcpy(pDropMsg->user, pTableMetaInfo->name); tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1091,9 +1091,9 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
size_t nameLen = strlen(pTableMetaInfo->name); size_t nameLen = strlen(pTableMetaInfo->name);
if (nameLen > 0) { if (nameLen > 0) {
strcpy(pShowMsg->db, pTableMetaInfo->name); // prefix is set here tstrncpy(pShowMsg->db, pTableMetaInfo->name, sizeof(pShowMsg->db)); // prefix is set here
} else { } else {
strcpy(pShowMsg->db, pObj->db); tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db));
} }
SShowInfo *pShowInfo = &pInfo->pDCLInfo->showOpt; SShowInfo *pShowInfo = &pInfo->pDCLInfo->showOpt;
@ -1300,7 +1300,7 @@ int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload; SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strcpy(pAlterDbMsg->db, pTableMetaInfo->name); tstrncpy(pAlterDbMsg->db, pTableMetaInfo->name, sizeof(pAlterDbMsg->db));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1430,9 +1430,9 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
char *db; // ugly code to move the space char *db; // ugly code to move the space
db = strstr(pObj->db, TS_PATH_DELIMITER); db = strstr(pObj->db, TS_PATH_DELIMITER);
db = (db == NULL) ? pObj->db : db + 1; db = (db == NULL) ? pObj->db : db + 1;
strcpy(pConnect->db, db); tstrncpy(pConnect->db, db, sizeof(pConnect->db));
strcpy(pConnect->clientVersion, version); tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion));
strcpy(pConnect->msgVersion, ""); tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2150,11 +2150,11 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
SCMConnectRsp *pConnect = (SCMConnectRsp *)pRes->pRsp; SCMConnectRsp *pConnect = (SCMConnectRsp *)pRes->pRsp;
strcpy(pObj->acctId, pConnect->acctId); // copy acctId from response tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response
int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db);
assert(len <= tListLen(pObj->db)); assert(len <= sizeof(pObj->db));
strncpy(pObj->db, temp, tListLen(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db));
if (pConnect->ipList.numOfIps > 0) if (pConnect->ipList.numOfIps > 0)
tscSetMgmtIpList(&pConnect->ipList); tscSetMgmtIpList(&pConnect->ipList);
@ -2172,7 +2172,7 @@ int tscProcessUseDbRsp(SSqlObj *pSql) {
STscObj * pObj = pSql->pTscObj; STscObj * pObj = pSql->pTscObj;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
strcpy(pObj->db, pTableMetaInfo->name); tstrncpy(pObj->db, pTableMetaInfo->name, sizeof(pObj->db));
return 0; return 0;
} }
@ -2338,7 +2338,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo);
assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
strncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, tListLen(pNewMeterMetaInfo->name)); tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name));
memcpy(pNew->cmd.payload, pSql->cmd.payload, pSql->cmd.payloadLen); // tag information if table does not exists. memcpy(pNew->cmd.payload, pSql->cmd.payload, pSql->cmd.payloadLen); // tag information if table does not exists.
pNew->cmd.payloadLen = pSql->cmd.payloadLen; pNew->cmd.payloadLen = pSql->cmd.payloadLen;
tscTrace("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated); tscTrace("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated);

8
src/client/src/tscSql.c
View File

@ -86,21 +86,21 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
pObj->signature = pObj; pObj->signature = pObj;
strncpy(pObj->user, user, TSDB_USER_LEN); tstrncpy(pObj->user, user, sizeof(pObj->user));
taosEncryptPass((uint8_t *)pass, strlen(pass), pObj->pass); taosEncryptPass((uint8_t *)pass, strlen(pass), pObj->pass);
if (db) { if (db) {
int32_t len = strlen(db); int32_t len = strlen(db);
/* db name is too long */ /* db name is too long */
if (len > TSDB_DB_NAME_LEN) { if (len >= TSDB_DB_NAME_LEN) {
terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH;
rpcClose(pDnodeConn); rpcClose(pDnodeConn);
free(pObj); free(pObj);
return NULL; return NULL;
} }
char tmp[TSDB_DB_NAME_LEN + 1] = {0}; char tmp[TSDB_DB_NAME_LEN] = {0};
strcpy(tmp, db); tstrncpy(tmp, db, sizeof(tmp));
strdequote(tmp); strdequote(tmp);
strtolower(pObj->db, tmp); strtolower(pObj->db, tmp);

2
src/client/src/tscSubquery.c
View File

@ -1921,7 +1921,7 @@ static char *getArithemicInputSrc(void *param, const char *name, int32_t colId)
for (int32_t i = 0; i < pSupport->numOfCols; ++i) { for (int32_t i = 0; i < pSupport->numOfCols; ++i) {
pExpr = taosArrayGetP(pSupport->exprList, i); pExpr = taosArrayGetP(pSupport->exprList, i);
if (strncmp(name, pExpr->aliasName, TSDB_COL_NAME_LEN) == 0) { if (strncmp(name, pExpr->aliasName, sizeof(pExpr->aliasName) - 1) == 0) {
index = i; index = i;
break; break;
} }

18
src/client/src/tscSystem.c
View File

@ -144,11 +144,11 @@ void taos_init_imp() {
} }
int64_t refreshTime = tsTableMetaKeepTimer; int64_t refreshTime = tsTableMetaKeepTimer;
refreshTime = refreshTime > 2 ? 2 : refreshTime; refreshTime = refreshTime > 10 ? 10 : refreshTime;
refreshTime = refreshTime < 1 ? 1 : refreshTime; refreshTime = refreshTime < 10 ? 10 : refreshTime;
if (tscCacheHandle == NULL) { if (tscCacheHandle == NULL) {
tscCacheHandle = taosCacheInit(tscTmr, refreshTime); tscCacheHandle = taosCacheInit(refreshTime);
} }
tscTrace("client is initialized successfully"); tscTrace("client is initialized successfully");
@ -180,7 +180,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
assert(cfg != NULL); assert(cfg != NULL);
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
strncpy(configDir, pStr, TSDB_FILENAME_LEN); tstrncpy(configDir, pStr, TSDB_FILENAME_LEN);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscPrint("set config file directory:%s", pStr); tscPrint("set config file directory:%s", pStr);
} else { } else {
@ -201,7 +201,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
tscPrint("set shellActivityTimer:%d", tsShellActivityTimer); tscPrint("set shellActivityTimer:%d", tsShellActivityTimer);
} else { } else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, pStr, tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, pStr,
tsCfgStatusStr[cfg->cfgStatus], (int32_t *)cfg->ptr); tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr);
} }
break; break;
@ -234,7 +234,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
tscPrint("failed to set locale:%s, current locale:%s", pStr, tsLocale); tscPrint("failed to set locale:%s, current locale:%s", pStr, tsLocale);
} }
strncpy(tsLocale, locale, tListLen(tsLocale)); tstrncpy(tsLocale, locale, sizeof(tsLocale));
char *charset = strrchr(tsLocale, sep); char *charset = strrchr(tsLocale, sep);
if (charset != NULL) { if (charset != NULL) {
@ -249,7 +249,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
tscPrint("charset changed from %s to %s", tsCharset, charset); tscPrint("charset changed from %s to %s", tsCharset, charset);
} }
strncpy(tsCharset, charset, tListLen(tsCharset)); tstrncpy(tsCharset, charset, sizeof(tsCharset));
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else { } else {
@ -286,7 +286,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
tscPrint("charset changed from %s to %s", tsCharset, pStr); tscPrint("charset changed from %s to %s", tsCharset, pStr);
} }
strncpy(tsCharset, pStr, tListLen(tsCharset)); tstrncpy(tsCharset, pStr, sizeof(tsCharset));
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else { } else {
tscPrint("charset:%s not valid", pStr); tscPrint("charset:%s not valid", pStr);
@ -324,7 +324,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
// return -1; // return -1;
// } // }
strncpy(tsSocketType, pStr, tListLen(tsSocketType)); tstrncpy(tsSocketType, pStr, sizeof(tsSocketType));
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscPrint("socket type is set:%s", tsSocketType); tscPrint("socket type is set:%s", tsSocketType);
} }

10
src/client/src/tscUtil.c
View File

@ -593,7 +593,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
dataBuf->size = startOffset; dataBuf->size = startOffset;
dataBuf->tsSource = -1; dataBuf->tsSource = -1;
strncpy(dataBuf->tableId, name, TSDB_TABLE_ID_LEN); tstrncpy(dataBuf->tableId, name, sizeof(dataBuf->tableId));
/* /*
* The table meta may be released since the table meta cache are completed clean by other thread * The table meta may be released since the table meta cache are completed clean by other thread
@ -801,7 +801,7 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) {
TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) { TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) {
TAOS_FIELD f = { .type = type, .bytes = bytes, }; TAOS_FIELD f = { .type = type, .bytes = bytes, };
strncpy(f.name, name, TSDB_COL_NAME_LEN); tstrncpy(f.name, name, sizeof(f.name));
return f; return f;
} }
@ -966,12 +966,12 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
if (isTagCol) { if (isTagCol) {
SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId; pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId;
strncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, TSDB_COL_NAME_LEN); tstrncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, sizeof(pExpr->colInfo.name));
} else if (pTableMetaInfo->pTableMeta != NULL) { } else if (pTableMetaInfo->pTableMeta != NULL) {
// in handling select database/version/server_status(), the pTableMeta is NULL // in handling select database/version/server_status(), the pTableMeta is NULL
SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex);
pExpr->colInfo.colId = pSchema->colId; pExpr->colInfo.colId = pSchema->colId;
strncpy(pExpr->colInfo.name, pSchema->name, TSDB_COL_NAME_LEN); tstrncpy(pExpr->colInfo.name, pSchema->name, sizeof(pExpr->colInfo.name));
} }
} }
@ -1666,7 +1666,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
assert(pTableMetaInfo != NULL); assert(pTableMetaInfo != NULL);
if (name != NULL) { if (name != NULL) {
strncpy(pTableMetaInfo->name, name, TSDB_TABLE_ID_LEN); tstrncpy(pTableMetaInfo->name, name, sizeof(pTableMetaInfo->name));
} }
pTableMetaInfo->pTableMeta = pTableMeta; pTableMetaInfo->pTableMeta = pTableMeta;

11
src/common/inc/tdataformat.h
View File

@ -31,7 +31,7 @@ extern "C" {
do { \ do { \
VarDataLenT __len = strlen(str); \ VarDataLenT __len = strlen(str); \
*(VarDataLenT *)(x) = __len; \ *(VarDataLenT *)(x) = __len; \
strncpy(varDataVal(x), (str), __len); \ memcpy(varDataVal(x), (str), __len); \
} while (0); } while (0);
#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \
@ -43,7 +43,7 @@ extern "C" {
#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ #define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \
do { \ do { \
*(VarDataLenT *)(x) = (_size); \ *(VarDataLenT *)(x) = (_size); \
strncpy(varDataVal(x), (str), (_size)); \ memcpy(varDataVal(x), (str), (_size)); \
} while (0); } while (0);
// ----------------- TSDB COLUMN DEFINITION // ----------------- TSDB COLUMN DEFINITION
@ -69,7 +69,8 @@ typedef struct {
int version; // version int version; // version
int numOfCols; // Number of columns appended int numOfCols; // Number of columns appended
int tlen; // maximum length of a SDataRow without the header part int tlen; // maximum length of a SDataRow without the header part
int flen; // First part length in a SDataRow after the header part int16_t flen; // First part length in a SDataRow after the header part
int16_t vlen; // pure value part length, excluded the overhead
STColumn columns[]; STColumn columns[];
} STSchema; } STSchema;
@ -77,6 +78,7 @@ typedef struct {
#define schemaVersion(s) ((s)->version) #define schemaVersion(s) ((s)->version)
#define schemaTLen(s) ((s)->tlen) #define schemaTLen(s) ((s)->tlen)
#define schemaFLen(s) ((s)->flen) #define schemaFLen(s) ((s)->flen)
#define schemaVLen(s) ((s)->vlen)
#define schemaColAt(s, i) ((s)->columns + i) #define schemaColAt(s, i) ((s)->columns + i)
#define tdFreeSchema(s) tfree((s)) #define tdFreeSchema(s) tfree((s))
@ -105,7 +107,8 @@ typedef struct {
int tCols; int tCols;
int nCols; int nCols;
int tlen; int tlen;
int flen; int16_t flen;
int16_t vlen;
int version; int version;
STColumn *columns; STColumn *columns;
} STSchemaBuilder; } STSchemaBuilder;

9
src/common/src/tdataformat.c
View File

@ -100,6 +100,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
pBuilder->nCols = 0; pBuilder->nCols = 0;
pBuilder->tlen = 0; pBuilder->tlen = 0;
pBuilder->flen = 0; pBuilder->flen = 0;
pBuilder->vlen = 0;
pBuilder->version = version; pBuilder->version = version;
} }
@ -124,10 +125,12 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int3
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
colSetBytes(pCol, bytes); colSetBytes(pCol, bytes);
pBuilder->tlen += (TYPE_BYTES[type] + sizeof(VarDataLenT) + bytes); pBuilder->tlen += (TYPE_BYTES[type] + bytes);
pBuilder->vlen += bytes - sizeof(VarDataLenT);
} else { } else {
colSetBytes(pCol, TYPE_BYTES[type]); colSetBytes(pCol, TYPE_BYTES[type]);
pBuilder->tlen += TYPE_BYTES[type]; pBuilder->tlen += TYPE_BYTES[type];
pBuilder->vlen += TYPE_BYTES[type];
} }
pBuilder->nCols++; pBuilder->nCols++;
@ -150,6 +153,7 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
schemaNCols(pSchema) = pBuilder->nCols; schemaNCols(pSchema) = pBuilder->nCols;
schemaTLen(pSchema) = pBuilder->tlen; schemaTLen(pSchema) = pBuilder->tlen;
schemaFLen(pSchema) = pBuilder->flen; schemaFLen(pSchema) = pBuilder->flen;
schemaVLen(pSchema) = pBuilder->vlen;
memcpy(schemaColAt(pSchema, 0), pBuilder->columns, sizeof(STColumn) * pBuilder->nCols); memcpy(schemaColAt(pSchema, 0), pBuilder->columns, sizeof(STColumn) * pBuilder->nCols);
@ -269,8 +273,7 @@ void dataColSetNullAt(SDataCol *pCol, int index) {
if (IS_VAR_DATA_TYPE(pCol->type)) { if (IS_VAR_DATA_TYPE(pCol->type)) {
pCol->dataOff[index] = pCol->len; pCol->dataOff[index] = pCol->len;
char *ptr = POINTER_SHIFT(pCol->pData, pCol->len); char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
varDataLen(ptr) = (pCol->type == TSDB_DATA_TYPE_BINARY) ? sizeof(char) : TSDB_NCHAR_SIZE; setVardataNull(ptr, pCol->type);
setNull(varDataVal(ptr), pCol->type, pCol->bytes);
pCol->len += varDataTLen(ptr); pCol->len += varDataTLen(ptr);
} else { } else {
setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes); setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes);

4
src/common/src/tglobal.c
View File

@ -708,7 +708,7 @@ static void doInitGlobalConfig() {
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0; cfg.minValue = 0;
cfg.maxValue = 0; cfg.maxValue = 0;
cfg.ptrLength = TSDB_DB_NAME_LEN; cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
@ -923,7 +923,7 @@ static void doInitGlobalConfig() {
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0; cfg.minValue = 0;
cfg.maxValue = 0; cfg.maxValue = 0;
cfg.ptrLength = TSDB_DB_NAME_LEN; cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);

3
src/common/src/tname.c
View File

@ -28,8 +28,7 @@ void extractTableName(const char* tableId, char* name) {
size_t s1 = strcspn(tableId, &TS_PATH_DELIMITER[0]); size_t s1 = strcspn(tableId, &TS_PATH_DELIMITER[0]);
size_t s2 = strcspn(&tableId[s1 + 1], &TS_PATH_DELIMITER[0]); size_t s2 = strcspn(&tableId[s1 + 1], &TS_PATH_DELIMITER[0]);
strncpy(name, &tableId[s1 + s2 + 2], TSDB_TABLE_NAME_LEN); tstrncpy(name, &tableId[s1 + s2 + 2], TSDB_TABLE_NAME_LEN);
name[TSDB_TABLE_NAME_LEN] = 0;
} }
char* extractDBName(const char* tableId, char* name) { char* extractDBName(const char* tableId, char* name) {

6
src/connector/python/linux/python3/taos/cinterface.py
View File

@ -130,9 +130,9 @@ _CONVERT_FUNC = {
# Corresponding TAOS_FIELD structure in C # Corresponding TAOS_FIELD structure in C
class TaosField(ctypes.Structure): class TaosField(ctypes.Structure):
_fields_ = [('name', ctypes.c_char * 64), _fields_ = [('name', ctypes.c_char * 65),
('bytes', ctypes.c_short), ('type', ctypes.c_char),
('type', ctypes.c_char)] ('bytes', ctypes.c_short)]
# C interface class # C interface class
class CTaosInterface(object): class CTaosInterface(object):

8
src/cq/src/cqMain.c
View File

@ -73,8 +73,8 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
return NULL; return NULL;
} }
strcpy(pContext->user, pCfg->user); tstrncpy(pContext->user, pCfg->user, sizeof(pContext->user));
strcpy(pContext->pass, pCfg->pass); tstrncpy(pContext->pass, pCfg->pass, sizeof(pContext->pass));
const char* db = pCfg->db; const char* db = pCfg->db;
for (const char* p = db; *p != 0; p++) { for (const char* p = db; *p != 0; p++) {
if (*p == '.') { if (*p == '.') {
@ -82,7 +82,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
break; break;
} }
} }
strcpy(pContext->db, db); tstrncpy(pContext->db, db, sizeof(pContext->db));
pContext->vgId = pCfg->vgId; pContext->vgId = pCfg->vgId;
pContext->cqWrite = pCfg->cqWrite; pContext->cqWrite = pCfg->cqWrite;
pContext->ahandle = ahandle; pContext->ahandle = ahandle;
@ -215,7 +215,7 @@ void cqDrop(void *handle) {
cTrace("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); cTrace("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr);
free(pObj); free(pObj);
pthread_mutex_lock(&pContext->mutex); pthread_mutex_unlock(&pContext->mutex);
} }
static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {

8
src/dnode/src/dnodeMain.c
View File

@ -61,12 +61,10 @@ static const SDnodeComponent tsDnodeComponents[] = {
}; };
static int dnodeCreateDir(const char *dir) { static int dnodeCreateDir(const char *dir) {
struct stat dirstat; if (mkdir(dir, 0755) != 0 && errno != EEXIST) {
if (stat(dir, &dirstat) < 0) { return -1;
if (mkdir(dir, 0755) != 0 && errno != EEXIST) {
return -1;
}
} }
return 0; return 0;
} }

18
src/dnode/src/dnodeMgmt.c
View File

@ -177,10 +177,12 @@ void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
memcpy(item, pMsg, sizeof(SRpcMsg)); memcpy(item, pMsg, sizeof(SRpcMsg));
taosWriteQitem(tsMgmtQueue, 1, item); taosWriteQitem(tsMgmtQueue, 1, item);
} else { } else {
SRpcMsg rsp; SRpcMsg rsp = {
rsp.handle = pMsg->handle; .handle = pMsg->handle,
rsp.pCont = NULL; .pCont = NULL,
rsp.code = TSDB_CODE_DND_OUT_OF_MEMORY; .code = TSDB_CODE_DND_OUT_OF_MEMORY
};
rpcSendResponse(&rsp); rpcSendResponse(&rsp);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
} }
@ -188,9 +190,9 @@ void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
static void *dnodeProcessMgmtQueue(void *param) { static void *dnodeProcessMgmtQueue(void *param) {
SRpcMsg *pMsg; SRpcMsg *pMsg;
SRpcMsg rsp; SRpcMsg rsp = {0};
int type; int type;
void *handle; void * handle;
while (1) { while (1) {
if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) { if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) {
@ -251,6 +253,7 @@ static int32_t dnodeOpenVnodes() {
if (status != TSDB_CODE_SUCCESS) { if (status != TSDB_CODE_SUCCESS) {
dPrint("Get dnode list failed"); dPrint("Get dnode list failed");
free(vnodeList);
return status; return status;
} }
@ -290,6 +293,7 @@ static void dnodeCloseVnodes() {
if (status != TSDB_CODE_SUCCESS) { if (status != TSDB_CODE_SUCCESS) {
dPrint("Get dnode list failed"); dPrint("Get dnode list failed");
free(vnodeList);
return; return;
} }
@ -456,6 +460,7 @@ static bool dnodeReadMnodeInfos() {
return false; return false;
} }
content[len] = 0;
cJSON* root = cJSON_Parse(content); cJSON* root = cJSON_Parse(content);
if (root == NULL) { if (root == NULL) {
dError("failed to read mnodeIpList.json, invalid json format"); dError("failed to read mnodeIpList.json, invalid json format");
@ -628,6 +633,7 @@ static bool dnodeReadDnodeCfg() {
return false; return false;
} }
content[len] = 0;
cJSON* root = cJSON_Parse(content); cJSON* root = cJSON_Parse(content);
if (root == NULL) { if (root == NULL) {
dError("failed to read dnodeCfg.json, invalid json format"); dError("failed to read dnodeCfg.json, invalid json format");

9
src/dnode/src/dnodePeer.c
View File

@ -83,10 +83,11 @@ void dnodeCleanupServer() {
} }
static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
SRpcMsg rspMsg; SRpcMsg rspMsg = {
rspMsg.handle = pMsg->handle; .handle = pMsg->handle,
rspMsg.pCont = NULL; .pCont = NULL,
rspMsg.contLen = 0; .contLen = 0
};
if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) { if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
rspMsg.code = TSDB_CODE_RPC_NOT_READY; rspMsg.code = TSDB_CODE_RPC_NOT_READY;

14
src/dnode/src/dnodeShell.c
View File

@ -19,6 +19,7 @@
#include "taosdef.h" #include "taosdef.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "tglobal.h" #include "tglobal.h"
#include "tutil.h"
#include "http.h" #include "http.h"
#include "mnode.h" #include "mnode.h"
#include "dnode.h" #include "dnode.h"
@ -108,10 +109,11 @@ void dnodeCleanupShell() {
} }
void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
SRpcMsg rpcMsg; SRpcMsg rpcMsg = {
rpcMsg.handle = pMsg->handle; .handle = pMsg->handle,
rpcMsg.pCont = NULL; .pCont = NULL,
rpcMsg.contLen = 0; .contLen = 0
};
if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) { if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
@ -143,7 +145,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
if (code != TSDB_CODE_RPC_NOT_READY) return code; if (code != TSDB_CODE_RPC_NOT_READY) return code;
SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg)); SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg));
strcpy(pMsg->user, user); tstrncpy(pMsg->user, user, TSDB_USER_LEN);
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pMsg; rpcMsg.pCont = pMsg;
@ -201,7 +203,7 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid) {
int16_t numOfTags = htons(pTable->numOfTags); int16_t numOfTags = htons(pTable->numOfTags);
int32_t sid = htonl(pTable->sid); int32_t sid = htonl(pTable->sid);
uint64_t uid = htobe64(pTable->uid); uint64_t uid = htobe64(pTable->uid);
dPrint("table:%s, numOfColumns:%d numOfTags:%d sid:%d uid:%d", pTable->tableId, numOfColumns, numOfTags, sid, uid); dPrint("table:%s, numOfColumns:%d numOfTags:%d sid:%d uid:%" PRIu64, pTable->tableId, numOfColumns, numOfTags, sid, uid);
return rpcRsp.pCont; return rpcRsp.pCont;
} }

3
src/dnode/src/dnodeSystem.c
View File

@ -33,7 +33,7 @@ int32_t main(int32_t argc, char *argv[]) {
printf("config file path overflow"); printf("config file path overflow");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
strcpy(configDir, argv[i]); tstrncpy(configDir, argv[i], TSDB_FILENAME_LEN);
} else { } else {
printf("'-c' requires a parameter, default:%s\n", configDir); printf("'-c' requires a parameter, default:%s\n", configDir);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
@ -92,6 +92,7 @@ int32_t main(int32_t argc, char *argv[]) {
// Initialize the system // Initialize the system
if (dnodeInitSystem() < 0) { if (dnodeInitSystem() < 0) {
syslog(LOG_ERR, "Error initialize TDengine system"); syslog(LOG_ERR, "Error initialize TDengine system");
dPrint("Failed to start TDengine, please check the log at:%s", tsLogDir);
closelog(); closelog();
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }

4
src/inc/taos.h
View File

@ -53,9 +53,9 @@ typedef enum {
} TSDB_OPTION; } TSDB_OPTION;
typedef struct taosField { typedef struct taosField {
char name[64]; char name[65];
short bytes;
uint8_t type; uint8_t type;
short bytes;
} TAOS_FIELD; } TAOS_FIELD;
#ifdef _TD_GO_DLL_ #ifdef _TD_GO_DLL_

10
src/inc/taosdef.h
View File

@ -189,7 +189,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_USERID_LEN 9 #define TSDB_USERID_LEN 9
#define TS_PATH_DELIMITER_LEN 1 #define TS_PATH_DELIMITER_LEN 1
#define TSDB_METER_ID_LEN_MARGIN 10 #define TSDB_METER_ID_LEN_MARGIN 8
#define TSDB_TABLE_ID_LEN (TSDB_DB_NAME_LEN+TSDB_TABLE_NAME_LEN+2*TS_PATH_DELIMITER_LEN+TSDB_USERID_LEN+TSDB_METER_ID_LEN_MARGIN) //TSDB_DB_NAME_LEN+TSDB_TABLE_NAME_LEN+2*strlen(TS_PATH_DELIMITER)+strlen(USERID) #define TSDB_TABLE_ID_LEN (TSDB_DB_NAME_LEN+TSDB_TABLE_NAME_LEN+2*TS_PATH_DELIMITER_LEN+TSDB_USERID_LEN+TSDB_METER_ID_LEN_MARGIN) //TSDB_DB_NAME_LEN+TSDB_TABLE_NAME_LEN+2*strlen(TS_PATH_DELIMITER)+strlen(USERID)
#define TSDB_UNI_LEN 24 #define TSDB_UNI_LEN 24
#define TSDB_USER_LEN TSDB_UNI_LEN #define TSDB_USER_LEN TSDB_UNI_LEN
@ -200,13 +200,13 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns
#define TSDB_NODE_NAME_LEN 64 #define TSDB_NODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 192 #define TSDB_TABLE_NAME_LEN 193
#define TSDB_DB_NAME_LEN 32 #define TSDB_DB_NAME_LEN 33
#define TSDB_COL_NAME_LEN 64 #define TSDB_COL_NAME_LEN 65
#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64
#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE
#define TSDB_MAX_SQL_SHOW_LEN 256 #define TSDB_MAX_SQL_SHOW_LEN 256
#define TSDB_MAX_ALLOWED_SQL_LEN (8*1024*1024U) // sql length should be less than 6mb #define TSDB_MAX_ALLOWED_SQL_LEN (8*1024*1024U) // sql length should be less than 8mb
#define TSDB_MAX_BYTES_PER_ROW TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_BYTES_PER_ROW TSDB_MAX_COLUMNS * 64
#define TSDB_MAX_TAGS_LEN 65536 #define TSDB_MAX_TAGS_LEN 65536

14
src/inc/taosmsg.h
View File

@ -224,7 +224,7 @@ typedef struct {
typedef struct SSchema { typedef struct SSchema {
uint8_t type; uint8_t type;
char name[TSDB_COL_NAME_LEN + 1]; char name[TSDB_COL_NAME_LEN];
int16_t colId; int16_t colId;
int16_t bytes; int16_t bytes;
} SSchema; } SSchema;
@ -250,7 +250,7 @@ typedef struct {
typedef struct { typedef struct {
char tableId[TSDB_TABLE_ID_LEN + 1]; char tableId[TSDB_TABLE_ID_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1]; char db[TSDB_DB_NAME_LEN];
int8_t igExists; int8_t igExists;
int8_t getMeta; int8_t getMeta;
int16_t numOfTags; int16_t numOfTags;
@ -268,7 +268,7 @@ typedef struct {
typedef struct { typedef struct {
char tableId[TSDB_TABLE_ID_LEN + 1]; char tableId[TSDB_TABLE_ID_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1]; char db[TSDB_DB_NAME_LEN];
int16_t type; /* operation type */ int16_t type; /* operation type */
int16_t numOfCols; /* number of schema */ int16_t numOfCols; /* number of schema */
int32_t tagValLen; int32_t tagValLen;
@ -296,7 +296,7 @@ typedef struct {
} SCMConnectMsg; } SCMConnectMsg;
typedef struct { typedef struct {
char acctId[TSDB_ACCT_LEN + 1]; char acctId[TSDB_ACCT_LEN];
char serverVersion[TSDB_VERSION_LEN]; char serverVersion[TSDB_VERSION_LEN];
int8_t writeAuth; int8_t writeAuth;
int8_t superAuth; int8_t superAuth;
@ -502,7 +502,7 @@ typedef struct {
typedef struct { typedef struct {
char acct[TSDB_USER_LEN + 1]; char acct[TSDB_USER_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1]; char db[TSDB_DB_NAME_LEN];
int32_t cacheBlockSize; //MB int32_t cacheBlockSize; //MB
int32_t totalBlocks; int32_t totalBlocks;
int32_t maxTables; int32_t maxTables;
@ -606,7 +606,7 @@ typedef struct {
} SMDVnodeDesc; } SMDVnodeDesc;
typedef struct { typedef struct {
char db[TSDB_DB_NAME_LEN + 1]; char db[TSDB_DB_NAME_LEN];
SMDVnodeCfg cfg; SMDVnodeCfg cfg;
SMDVnodeDesc nodes[TSDB_MAX_REPLICA]; SMDVnodeDesc nodes[TSDB_MAX_REPLICA];
} SMDCreateVnodeMsg; } SMDCreateVnodeMsg;
@ -671,7 +671,7 @@ typedef struct {
*/ */
typedef struct { typedef struct {
int8_t type; int8_t type;
char db[TSDB_DB_NAME_LEN + 1]; char db[TSDB_DB_NAME_LEN];
uint16_t payloadLen; uint16_t payloadLen;
char payload[]; char payload[];
} SCMShowMsg; } SCMShowMsg;

2
src/inc/tcq.h
View File

@ -27,7 +27,7 @@ typedef struct {
int vgId; int vgId;
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN]; char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_NAME_LEN + 1]; char db[TSDB_DB_NAME_LEN];
FCqWrite cqWrite; FCqWrite cqWrite;
} SCqCfg; } SCqCfg;

16
src/inc/tsdb.h
View File

@ -65,6 +65,13 @@ typedef struct {
int8_t compression; int8_t compression;
} STsdbCfg; } STsdbCfg;
// --------- TSDB REPOSITORY USAGE STATISTICS
typedef struct {
int64_t totalStorage; // total bytes occupie
int64_t compStorage;
int64_t pointsWritten; // total data points written
} STsdbStat;
typedef void TsdbRepoT; // use void to hide implementation details from outside typedef void TsdbRepoT; // use void to hide implementation details from outside
void tsdbSetDefaultCfg(STsdbCfg *pCfg); void tsdbSetDefaultCfg(STsdbCfg *pCfg);
@ -306,6 +313,15 @@ int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, uint64_t uid, STableGroupInfo *pGr
*/ */
void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle); void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
/**
* get the statistics of repo usage
* @param repo. point to the tsdbrepo
* @param totalPoints. total data point written
* @param totalStorage. total bytes took by the tsdb
* @param compStorage. total bytes took by the tsdb after compressed
*/
void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

12
src/kit/shell/src/shellEngine.c
View File

@ -727,10 +727,6 @@ void read_history() {
char f_history[TSDB_FILENAME_LEN]; char f_history[TSDB_FILENAME_LEN];
get_history_path(f_history); get_history_path(f_history);
if (access(f_history, R_OK) == -1) {
return;
}
FILE *f = fopen(f_history, "r"); FILE *f = fopen(f_history, "r");
if (f == NULL) { if (f == NULL) {
fprintf(stderr, "Opening file %s\n", f_history); fprintf(stderr, "Opening file %s\n", f_history);
@ -809,14 +805,6 @@ void source_file(TAOS *con, char *fptr) {
return; return;
} }
if (access(fname, R_OK) != 0) {
fprintf(stderr, "ERROR: file %s is not readable\n", fptr);
wordfree(&full_path);
free(cmd);
return;
}
FILE *f = fopen(fname, "r"); FILE *f = fopen(fname, "r");
if (f == NULL) { if (f == NULL) {
fprintf(stderr, "ERROR: failed to open file %s\n", fname); fprintf(stderr, "ERROR: failed to open file %s\n", fname);

12
src/kit/taosdemo/taosdemo.c
View File

@ -710,7 +710,7 @@ void *readTable(void *sarg) {
int32_t code = taos_errno(pSql); int32_t code = taos_errno(pSql);
if (code != 0) { if (code != 0) {
fprintf(stderr, "Failed to query:%s\n", taos_errstr(taos)); fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql));
taos_free_result(pSql); taos_free_result(pSql);
taos_close(taos); taos_close(taos);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
@ -756,7 +756,7 @@ void *readMetric(void *sarg) {
for (int j = 0; j < n; j++) { for (int j = 0; j < n; j++) {
char condition[BUFFER_SIZE - 30] = "\0"; char condition[BUFFER_SIZE - 30] = "\0";
char tempS[BUFFER_SIZE] = "\0"; char tempS[64] = "\0";
int m = 10 < num_of_tables ? 10 : num_of_tables; int m = 10 < num_of_tables ? 10 : num_of_tables;
@ -779,7 +779,7 @@ void *readMetric(void *sarg) {
int32_t code = taos_errno(pSql); int32_t code = taos_errno(pSql);
if (code != 0) { if (code != 0) {
fprintf(stderr, "Failed to query:%s\n", taos_errstr(taos)); fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql));
taos_free_result(pSql); taos_free_result(pSql);
taos_close(taos); taos_close(taos);
exit(1); exit(1);
@ -818,7 +818,9 @@ void queryDB(TAOS *taos, char *command) {
} }
if (i == 0) { if (i == 0) {
fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(taos)); fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
taos_free_result(pSql);
taos_close(taos); taos_close(taos);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
@ -914,7 +916,7 @@ void callBack(void *param, TAOS_RES *res, int code) {
int64_t tmp_time = tb_info->timestamp; int64_t tmp_time = tb_info->timestamp;
if (code < 0) { if (code < 0) {
fprintf(stderr, "failed to insert data %d:reason; %s\n", code, taos_errstr(tb_info->taos)); fprintf(stderr, "failed to insert data %d:reason; %s\n", code, taos_errstr(res));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }

15
src/kit/taosdump/taosdump.c
View File

@ -88,21 +88,21 @@ enum _describe_table_index {
}; };
typedef struct { typedef struct {
char field[TSDB_COL_NAME_LEN + 1]; char field[TSDB_COL_NAME_LEN];
char type[16]; char type[16];
int length; int length;
char note[128]; char note[128];
} SColDes; } SColDes;
typedef struct { typedef struct {
char name[TSDB_COL_NAME_LEN + 1]; char name[TSDB_COL_NAME_LEN];
SColDes cols[]; SColDes cols[];
} STableDef; } STableDef;
extern char version[]; extern char version[];
typedef struct { typedef struct {
char name[TSDB_DB_NAME_LEN + 1]; char name[TSDB_DB_NAME_LEN];
int32_t replica; int32_t replica;
int32_t days; int32_t days;
int32_t keep; int32_t keep;
@ -643,13 +643,14 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) {
lseek(fd, 0, SEEK_SET); lseek(fd, 0, SEEK_SET);
while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) { while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) {
tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp); taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
} }
tclose(fd); close(fd);
remove(".table.tmp");
return 0; return remove(".table.tmp");
} }
void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArguments *arguments, FILE *fp) { void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArguments *arguments, FILE *fp) {
@ -903,6 +904,8 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
lseek(fd, 0, SEEK_SET); lseek(fd, 0, SEEK_SET);
while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) { while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) {
tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp); taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
} }

10
src/mnode/inc/mnodeDef.h
View File

@ -32,8 +32,8 @@ struct SMnodeObj;
typedef struct SDnodeObj { typedef struct SDnodeObj {
int32_t dnodeId; int32_t dnodeId;
uint16_t dnodePort; uint16_t dnodePort;
char dnodeFqdn[TSDB_FQDN_LEN + 1]; char dnodeFqdn[TSDB_FQDN_LEN];
char dnodeEp[TSDB_EP_LEN + 1]; char dnodeEp[TSDB_EP_LEN];
int64_t createdTime; int64_t createdTime;
uint32_t lastAccess; uint32_t lastAccess;
int32_t openVnodes; int32_t openVnodes;
@ -115,7 +115,7 @@ typedef struct {
typedef struct SVgObj { typedef struct SVgObj {
uint32_t vgId; uint32_t vgId;
char dbName[TSDB_DB_NAME_LEN + 1]; char dbName[TSDB_DB_NAME_LEN];
int64_t createdTime; int64_t createdTime;
SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
int32_t numOfVnodes; int32_t numOfVnodes;
@ -154,7 +154,7 @@ typedef struct {
} SDbCfg; } SDbCfg;
typedef struct SDbObj { typedef struct SDbObj {
char name[TSDB_DB_NAME_LEN + 1]; char name[TSDB_DB_NAME_LEN];
char acct[TSDB_USER_LEN + 1]; char acct[TSDB_USER_LEN + 1];
int64_t createdTime; int64_t createdTime;
int32_t cfgVersion; int32_t cfgVersion;
@ -219,7 +219,7 @@ typedef struct SAcctObj {
typedef struct { typedef struct {
int8_t type; int8_t type;
int32_t index; int32_t index;
char db[TSDB_DB_NAME_LEN + 1]; char db[TSDB_DB_NAME_LEN];
void * pIter; void * pIter;
int16_t numOfColumns; int16_t numOfColumns;
int32_t rowSize; int32_t rowSize;

12
src/mnode/src/mnodeDb.c
View File

@ -328,8 +328,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
if (code != 0) return code; if (code != 0) return code;
pDb = calloc(1, sizeof(SDbObj)); pDb = calloc(1, sizeof(SDbObj));
strncpy(pDb->name, pCreate->db, TSDB_DB_NAME_LEN); tstrncpy(pDb->name, pCreate->db, sizeof(pDb->name));
strncpy(pDb->acct, pAcct->user, TSDB_USER_LEN); tstrncpy(pDb->acct, pAcct->user, sizeof(pDb->acct));
pDb->createdTime = taosGetTimestampMs(); pDb->createdTime = taosGetTimestampMs();
pDb->cfg = (SDbCfg) { pDb->cfg = (SDbCfg) {
.cacheBlockSize = pCreate->cacheBlockSize, .cacheBlockSize = pCreate->cacheBlockSize,
@ -373,7 +373,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
} }
bool mnodeCheckIsMonitorDB(char *db, char *monitordb) { bool mnodeCheckIsMonitorDB(char *db, char *monitordb) {
char dbName[TSDB_DB_NAME_LEN + 1] = {0}; char dbName[TSDB_DB_NAME_LEN] = {0};
extractDBName(db, dbName); extractDBName(db, dbName);
size_t len = strlen(dbName); size_t len = strlen(dbName);
@ -453,7 +453,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
SUserObj *pUser = mnodeGetUserFromConn(pConn); SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0; if (pUser == NULL) return 0;
pShow->bytes[cols] = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE; pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY; pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name"); strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
@ -610,7 +610,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
char* name = mnodeGetDbStr(pDb->name); char* name = mnodeGetDbStr(pDb->name);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, TSDB_DB_NAME_LEN); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, TSDB_DB_NAME_LEN - 1);
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -882,7 +882,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
} }
if (pDb->cfg.replications - replications >= 2) { if (pDb->cfg.replications - replications >= 2) {
mError("db:%s, replica number can't change from 3 to 1", pDb->name, replications); mError("db:%s, replica number can't change from %d to %d", pDb->name, pDb->cfg.replications, replications);
terrno = TSDB_CODE_MND_INVALID_DB_OPTION; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
} }
} }

8
src/mnode/src/mnodeDnode.c
View File

@ -90,7 +90,7 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) { static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj; SDnodeObj *pDnode = pOper->pObj;
SDnodeObj *pSaved = mnodeGetDnode(pDnode->dnodeId); SDnodeObj *pSaved = mnodeGetDnode(pDnode->dnodeId);
if (pDnode != pSaved) { if (pDnode != pSaved && pDnode != NULL && pSaved != NULL) {
memcpy(pSaved, pDnode, pOper->rowSize); memcpy(pSaved, pDnode, pOper->rowSize);
free(pDnode); free(pDnode);
} }
@ -237,7 +237,9 @@ void mnodeUpdateDnode(SDnodeObj *pDnode) {
.pObj = pDnode .pObj = pDnode
}; };
sdbUpdateRow(&oper); if (sdbUpdateRow(&oper) != 0) {
mError("dnodeId:%d, failed update", pDnode->dnodeId);
}
} }
static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
@ -383,7 +385,7 @@ static int32_t mnodeCreateDnode(char *ep) {
pDnode->createdTime = taosGetTimestampMs(); pDnode->createdTime = taosGetTimestampMs();
pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->totalVnodes = TSDB_INVALID_VNODE_NUM; pDnode->totalVnodes = TSDB_INVALID_VNODE_NUM;
strcpy(pDnode->dnodeEp, ep); tstrncpy(pDnode->dnodeEp, ep, TSDB_EP_LEN);
taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort); taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort);
SSdbOper oper = { SSdbOper oper = {

8
src/mnode/src/mnodeMain.c
View File

@ -134,8 +134,12 @@ void mnodeStopSystem() {
} }
mnodeCleanupSystem(); mnodeCleanupSystem();
mPrint("mnode file is removed");
remove(tsMnodeDir); if (remove(tsMnodeDir) != 0) {
mPrint("failed to remove mnode file, reason:%s", strerror(errno));
} else {
mPrint("mnode file is removed");
}
} }
static void mnodeInitTimer() { static void mnodeInitTimer() {

4
src/mnode/src/mnodeProfile.c
View File

@ -67,7 +67,7 @@ int32_t mnodeInitProfile() {
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg);
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_CONN, mnodeProcessKillConnectionMsg); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_CONN, mnodeProcessKillConnectionMsg);
tsMnodeConnCache = taosCacheInitWithCb(tsMnodeTmr, CONN_CHECK_TIME, mnodeFreeConn); tsMnodeConnCache = taosCacheInitWithCb(CONN_CHECK_TIME, mnodeFreeConn);
return 0; return 0;
} }
@ -97,7 +97,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) {
.connId = connId, .connId = connId,
.stime = taosGetTimestampMs() .stime = taosGetTimestampMs()
}; };
strcpy(connObj.user, user); tstrncpy(connObj.user, user, TSDB_USER_LEN);
char key[10]; char key[10];
sprintf(key, "%u", connId); sprintf(key, "%u", connId);

16
src/mnode/src/mnodeSdb.c
View File

@ -30,6 +30,8 @@
#include "mnodeDnode.h" #include "mnodeDnode.h"
#include "mnodeSdb.h" #include "mnodeSdb.h"
#define SDB_TABLE_LEN 12
typedef enum { typedef enum {
SDB_ACTION_INSERT, SDB_ACTION_INSERT,
SDB_ACTION_DELETE, SDB_ACTION_DELETE,
@ -43,7 +45,7 @@ typedef enum {
} ESdbStatus; } ESdbStatus;
typedef struct _SSdbTable { typedef struct _SSdbTable {
char tableName[TSDB_DB_NAME_LEN + 1]; char tableName[SDB_TABLE_LEN];
ESdbTable tableId; ESdbTable tableId;
ESdbKey keyType; ESdbKey keyType;
int32_t hashSessions; int32_t hashSessions;
@ -174,10 +176,10 @@ static void sdbRestoreTables() {
totalRows += pTable->numOfRows; totalRows += pTable->numOfRows;
numOfTables++; numOfTables++;
sdbTrace("table:%s, is restored, numOfRows:%d", pTable->tableName, pTable->numOfRows); sdbTrace("table:%s, is restored, numOfRows:%" PRId64, pTable->tableName, pTable->numOfRows);
} }
sdbTrace("sdb is restored, version:%d totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables); sdbTrace("sdb is restored, version:%" PRId64 " totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables);
} }
void sdbUpdateMnodeRoles() { void sdbUpdateMnodeRoles() {
@ -449,7 +451,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
pthread_mutex_unlock(&pTable->mutex); pthread_mutex_unlock(&pTable->mutex);
sdbTrace("table:%s, insert record:%s to hash, rowSize:%d vnumOfRows:%d version:%" PRIu64, pTable->tableName, sdbTrace("table:%s, insert record:%s to hash, rowSize:%d vnumOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion()); sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion());
(*pTable->insertFp)(pOper); (*pTable->insertFp)(pOper);
@ -473,7 +475,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
pTable->numOfRows--; pTable->numOfRows--;
pthread_mutex_unlock(&pTable->mutex); pthread_mutex_unlock(&pTable->mutex);
sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d version:%" PRIu64, pTable->tableName, sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d" PRId64 "version:%" PRIu64, pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1; int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1;
@ -484,7 +486,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
} }
static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) { static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
sdbTrace("table:%s, update record:%s in hash, numOfRows:%d version:%" PRIu64, pTable->tableName, sdbTrace("table:%s, update record:%s in hash, numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
(*pTable->updateFp)(pOper); (*pTable->updateFp)(pOper);
@ -717,7 +719,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
if (pTable == NULL) return NULL; if (pTable == NULL) return NULL;
strcpy(pTable->tableName, pDesc->tableName); tstrncpy(pTable->tableName, pDesc->tableName, SDB_TABLE_LEN);
pTable->keyType = pDesc->keyType; pTable->keyType = pDesc->keyType;
pTable->tableId = pDesc->tableId; pTable->tableId = pDesc->tableId;
pTable->hashSessions = pDesc->hashSessions; pTable->hashSessions = pDesc->hashSessions;

3
src/mnode/src/mnodeShow.c
View File

@ -65,7 +65,7 @@ int32_t mnodeInitShow() {
mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mnodeProcessConnectMsg); mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mnodeProcessConnectMsg);
mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mnodeProcessUseMsg); mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mnodeProcessUseMsg);
tsMnodeShowCache = taosCacheInitWithCb(tsMnodeTmr, 10, mnodeFreeShowObj); tsMnodeShowCache = taosCacheInitWithCb(10, mnodeFreeShowObj);
return 0; return 0;
} }
@ -324,6 +324,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
connect_over: connect_over:
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
rpcFreeCont(pConnectRsp);
mLError("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); mLError("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code));
} else { } else {
mLPrint("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); mLPrint("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code));

38
src/mnode/src/mnodeTable.c
View File

@ -951,7 +951,7 @@ static int32_t mnodeModifySuperTableTagName(SSuperTableObj *pStable, char *oldTa
// update // update
SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col); SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col);
strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN); tstrncpy(schema->name, newTagName, sizeof(schema->name));
SSdbOper oper = { SSdbOper oper = {
.type = SDB_OPER_GLOBAL, .type = SDB_OPER_GLOBAL,
@ -1078,7 +1078,7 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow,
int32_t cols = 0; int32_t cols = 0;
SSchema *pSchema = pMeta->schema; SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pShow->bytes[cols] = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY; pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name"); strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
@ -1138,7 +1138,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
prefixLen = strlen(prefix); prefixLen = strlen(prefix);
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
char stableName[TSDB_TABLE_NAME_LEN + 1] = {0}; char stableName[TSDB_TABLE_NAME_LEN] = {0};
while (numOfRows < rows) { while (numOfRows < rows) {
pShow->pIter = mnodeGetNextSuperTable(pShow->pIter, &pTable); pShow->pIter = mnodeGetNextSuperTable(pShow->pIter, &pTable);
@ -1151,7 +1151,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
memset(stableName, 0, tListLen(stableName)); memset(stableName, 0, tListLen(stableName));
mnodeExtractTableName(pTable->info.tableId, stableName); mnodeExtractTableName(pTable->info.tableId, stableName);
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, stableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) { if (pShow->payloadLen > 0 && patternMatch(pShow->payload, stableName, sizeof(stableName) - 1, &info) != TSDB_PATTERN_MATCH) {
mnodeDecTableRef(pTable); mnodeDecTableRef(pTable);
continue; continue;
} }
@ -1160,7 +1160,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
int16_t len = strnlen(stableName, TSDB_DB_NAME_LEN); int16_t len = strnlen(stableName, TSDB_DB_NAME_LEN - 1);
*(int16_t*) pWrite = len; *(int16_t*) pWrite = len;
pWrite += sizeof(int16_t); // todo refactor pWrite += sizeof(int16_t); // todo refactor
@ -1228,7 +1228,7 @@ static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pT
assert(numOfCols <= TSDB_MAX_COLUMNS); assert(numOfCols <= TSDB_MAX_COLUMNS);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
strncpy(pSchema->name, pTable->schema[i].name, TSDB_COL_NAME_LEN); tstrncpy(pSchema->name, pTable->schema[i].name, sizeof(pSchema->name));
pSchema->type = pTable->schema[i].type; pSchema->type = pTable->schema[i].type;
pSchema->bytes = htons(pTable->schema[i].bytes); pSchema->bytes = htons(pTable->schema[i].bytes);
pSchema->colId = htons(pTable->schema[i].colId); pSchema->colId = htons(pTable->schema[i].colId);
@ -1769,8 +1769,8 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_OUT_OF_MEMORY; return TSDB_CODE_MND_OUT_OF_MEMORY;
} }
strncpy(pCreateMsg->tableId, pInfo->tableId, tListLen(pInfo->tableId)); tstrncpy(pCreateMsg->tableId, pInfo->tableId, sizeof(pInfo->tableId));
strcpy(pCreateMsg->db, pMsg->pDb->name); tstrncpy(pCreateMsg->db, pMsg->pDb->name, sizeof(pCreateMsg->db));
pCreateMsg->igExists = 1; pCreateMsg->igExists = 1;
pCreateMsg->getMeta = 1; pCreateMsg->getMeta = 1;
pCreateMsg->contLen = htonl(contLen); pCreateMsg->contLen = htonl(contLen);
@ -2079,7 +2079,7 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
int32_t cols = 0; int32_t cols = 0;
SSchema *pSchema = pMeta->schema; SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pShow->bytes[cols] = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY; pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "table_name"); strcpy(pSchema[cols].name, "table_name");
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
@ -2097,7 +2097,7 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++; cols++;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pShow->bytes[cols] = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY; pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "stable_name"); strcpy(pSchema[cols].name, "stable_name");
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
@ -2141,12 +2141,12 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
continue; continue;
} }
char tableName[TSDB_TABLE_NAME_LEN + 1] = {0}; char tableName[TSDB_TABLE_NAME_LEN] = {0};
// pattern compare for table name // pattern compare for table name
mnodeExtractTableName(pTable->info.tableId, tableName); mnodeExtractTableName(pTable->info.tableId, tableName);
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) { if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, sizeof(tableName) - 1, &info) != TSDB_PATTERN_MATCH) {
mnodeDecTableRef(pTable); mnodeDecTableRef(pTable);
continue; continue;
} }
@ -2155,7 +2155,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, TSDB_TABLE_NAME_LEN); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, sizeof(tableName) - 1);
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -2173,10 +2173,10 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
memset(tableName, 0, tListLen(tableName)); memset(tableName, 0, sizeof(tableName));
if (pTable->info.type == TSDB_CHILD_TABLE) { if (pTable->info.type == TSDB_CHILD_TABLE) {
mnodeExtractTableName(pTable->superTable->info.tableId, tableName); mnodeExtractTableName(pTable->superTable->info.tableId, tableName);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, TSDB_TABLE_NAME_LEN); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, sizeof(tableName) - 1);
} }
cols++; cols++;
@ -2268,7 +2268,7 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo
int32_t cols = 0; int32_t cols = 0;
SSchema *pSchema = pMeta->schema; SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pShow->bytes[cols] = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY; pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "table_name"); strcpy(pSchema[cols].name, "table_name");
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
@ -2331,12 +2331,12 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
continue; continue;
} }
char tableName[TSDB_TABLE_NAME_LEN + 1] = {0}; char tableName[TSDB_TABLE_NAME_LEN] = {0};
// pattern compare for table name // pattern compare for table name
mnodeExtractTableName(pTable->info.tableId, tableName); mnodeExtractTableName(pTable->info.tableId, tableName);
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) { if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, sizeof(tableName) - 1, &info) != TSDB_PATTERN_MATCH) {
mnodeDecTableRef(pTable); mnodeDecTableRef(pTable);
continue; continue;
} }
@ -2345,7 +2345,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, TSDB_TABLE_NAME_LEN); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, sizeof(tableName) - 1);
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

2
src/os/darwin/src/darwinPlatform.c
View File

@ -189,7 +189,7 @@ void taosGetSystemLocale() {
if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
char *locale = setlocale(LC_CTYPE, "chs"); char *locale = setlocale(LC_CTYPE, "chs");
if (locale != NULL) { if (locale != NULL) {
strncpy(tsLocale, locale, sizeof(tsLocale) / sizeof(tsLocale[0])); tstrncpy(tsLocale, locale, sizeof(tsLocale));
cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
uPrint("locale not configured, set to default:%s", tsLocale); uPrint("locale not configured, set to default:%s", tsLocale);
} }

99
src/os/linux/src/linuxSysPara.c
View File

@ -160,7 +160,7 @@ static void taosGetSystemTimezone() {
/* load time zone string from /etc/timezone */ /* load time zone string from /etc/timezone */
FILE *f = fopen("/etc/timezone", "r"); FILE *f = fopen("/etc/timezone", "r");
char buf[64] = {0}; char buf[65] = {0};
if (f != NULL) { if (f != NULL) {
fread(buf, 64, 1, f); fread(buf, 64, 1, f);
fclose(f); fclose(f);
@ -229,7 +229,7 @@ static void taosGetSystemLocale() { // get and set default locale
uError("can't get locale from system, set it to en_US.UTF-8"); uError("can't get locale from system, set it to en_US.UTF-8");
strcpy(tsLocale, "en_US.UTF-8"); strcpy(tsLocale, "en_US.UTF-8");
} else { } else {
tstrncpy(tsLocale, locale, tListLen(tsLocale)); tstrncpy(tsLocale, locale, sizeof(tsLocale));
uError("locale not configured, set to system default:%s", tsLocale); uError("locale not configured, set to system default:%s", tsLocale);
} }
} }
@ -242,7 +242,7 @@ static void taosGetSystemLocale() { // get and set default locale
str++; str++;
char *revisedCharset = taosCharsetReplace(str); char *revisedCharset = taosCharsetReplace(str);
strncpy(tsCharset, revisedCharset, tListLen(tsCharset)); tstrncpy(tsCharset, revisedCharset, sizeof(tsCharset));
free(revisedCharset); free(revisedCharset);
uWarn("charset not configured, set to system default:%s", tsCharset); uWarn("charset not configured, set to system default:%s", tsCharset);
@ -331,66 +331,7 @@ bool taosGetDisk() {
return true; return true;
} }
static bool taosGetCardName(char *ip, char *name) {
struct ifaddrs *ifaddr, *ifa;
int family, s;
char host[NI_MAXHOST];
bool ret = false;
if (getifaddrs(&ifaddr) == -1) {
return false;
}
/* Walk through linked list, maintaining head pointer so we can free list
* later */
for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
if (ifa->ifa_addr == NULL) continue;
family = ifa->ifa_addr->sa_family;
if (family != AF_INET) {
continue;
}
s = getnameinfo(ifa->ifa_addr, (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6), host,
NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
if (s != 0) {
break;
}
if (strcmp(host, "127.0.0.1") == 0) {
continue;
}
// TODO: the ip not config
// if (strcmp(host, ip) == 0) {
strcpy(name, ifa->ifa_name);
ret = true;
// }
}
freeifaddrs(ifaddr);
return ret;
}
static bool taosGetCardInfo(int64_t *bytes) { static bool taosGetCardInfo(int64_t *bytes) {
static char tsPublicCard[1000] = {0};
static char tsPrivateIp[40];
if (tsPublicCard[0] == 0) {
if (!taosGetCardName(tsPrivateIp, tsPublicCard)) {
uError("can't get card name from ip:%s", tsPrivateIp);
return false;
}
int cardNameLen = (int)strlen(tsPublicCard);
for (int i = 0; i < cardNameLen; ++i) {
if (tsPublicCard[i] == ':') {
tsPublicCard[i] = 0;
break;
}
}
// uTrace("card name of public ip:%s is %s", tsPublicIp, tsPublicCard);
}
FILE *fp = fopen(tsSysNetFile, "r"); FILE *fp = fopen(tsSysNetFile, "r");
if (fp == NULL) { if (fp == NULL) {
uError("open file:%s failed", tsSysNetFile); uError("open file:%s failed", tsSysNetFile);
@ -403,6 +344,7 @@ static bool taosGetCardInfo(int64_t *bytes) {
size_t len; size_t len;
char * line = NULL; char * line = NULL;
*bytes = 0;
while (!feof(fp)) { while (!feof(fp)) {
tfree(line); tfree(line);
@ -411,23 +353,20 @@ static bool taosGetCardInfo(int64_t *bytes) {
if (line == NULL) { if (line == NULL) {
break; break;
} }
if (strstr(line, tsPublicCard) != NULL) { if (strstr(line, "lo:") != NULL) {
break; continue;
} }
sscanf(line,
"%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64,
nouse0, &rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &tbytes, &tpackets);
*bytes += (rbytes + tbytes);
} }
if (line != NULL) {
sscanf(line, "%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64, nouse0, &rbytes, &rpackts, &nouse1, &nouse2, &nouse3, tfree(line);
&nouse4, &nouse5, &nouse6, &tbytes, &tpackets); fclose(fp);
*bytes = rbytes + tbytes;
tfree(line); return true;
fclose(fp);
return true;
} else {
uWarn("can't get card:%s info from device:%s", tsPublicCard, tsSysNetFile);
*bytes = 0;
fclose(fp);
return false;
}
} }
bool taosGetBandSpeed(float *bandSpeedKb) { bool taosGetBandSpeed(float *bandSpeedKb) {
@ -443,13 +382,15 @@ bool taosGetBandSpeed(float *bandSpeedKb) {
if (lastTime == 0 || lastBytes == 0) { if (lastTime == 0 || lastBytes == 0) {
lastTime = curTime; lastTime = curTime;
lastBytes = curBytes; lastBytes = curBytes;
return false; *bandSpeedKb = 0;
return true;
} }
if (lastTime >= curTime || lastBytes > curBytes) { if (lastTime >= curTime || lastBytes > curBytes) {
lastTime = curTime; lastTime = curTime;
lastBytes = curBytes; lastBytes = curBytes;
return false; *bandSpeedKb = 0;
return true;
} }
double totalBytes = (double)(curBytes - lastBytes) / 1024 * 8; // Kb double totalBytes = (double)(curBytes - lastBytes) / 1024 * 8; // Kb

2
src/os/windows/src/twindows.c
View File

@ -226,7 +226,7 @@ void taosGetSystemLocale() {
if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
char *locale = setlocale(LC_CTYPE, "chs"); char *locale = setlocale(LC_CTYPE, "chs");
if (locale != NULL) { if (locale != NULL) {
strncpy(tsLocale, locale, sizeof(tsLocale) / sizeof(tsLocale[0])); tstrncpy(tsLocale, locale, sizeof(tsLocale));
cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
uPrint("locale not configured, set to default:%s", tsLocale); uPrint("locale not configured, set to default:%s", tsLocale);
} }

22
src/plugins/http/src/gcHandle.c
View File

@ -22,9 +22,27 @@
#include "taosdef.h" #include "taosdef.h"
static HttpDecodeMethod gcDecodeMethod = {"grafana", gcProcessRequest}; static HttpDecodeMethod gcDecodeMethod = {"grafana", gcProcessRequest};
static HttpEncodeMethod gcHeartBeatMethod = {NULL, gcSendHeartBeatResp, NULL, NULL, NULL, NULL, NULL, NULL}; static HttpEncodeMethod gcHeartBeatMethod = {
.startJsonFp = NULL,
.stopJsonFp = gcSendHeartBeatResp,
.buildQueryJsonFp = NULL,
.buildAffectRowJsonFp = NULL,
.initJsonFp = NULL,
.cleanJsonFp = NULL,
.checkFinishedFp = NULL,
.setNextCmdFp = NULL
};
static HttpEncodeMethod gcQueryMethod = { static HttpEncodeMethod gcQueryMethod = {
NULL, gcStopQueryJson, gcBuildQueryJson, NULL, gcInitQueryJson, gcCleanQueryJson, NULL, NULL}; .startJsonFp = NULL,
.stopJsonFp = gcStopQueryJson,
.buildQueryJsonFp = gcBuildQueryJson,
.buildAffectRowJsonFp = NULL,
.initJsonFp = gcInitQueryJson,
.cleanJsonFp = gcCleanQueryJson,
.checkFinishedFp = NULL,
.setNextCmdFp = NULL
};
void gcInitHandle(HttpServer* pServer) { httpAddMethod(pServer, &gcDecodeMethod); } void gcInitHandle(HttpServer* pServer) { httpAddMethod(pServer, &gcDecodeMethod); }

4
src/plugins/http/src/httpAuth.c
View File

@ -88,8 +88,8 @@ bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len) {
free(base64); free(base64);
return false; return false;
} else { } else {
strncpy(pContext->user, descrypt, TSDB_USER_LEN); tstrncpy(pContext->user, descrypt, sizeof(pContext->user));
strncpy(pContext->pass, descrypt + TSDB_USER_LEN, TSDB_PASSWORD_LEN); tstrncpy(pContext->pass, descrypt + TSDB_USER_LEN, TSDB_PASSWORD_LEN);
httpTrace("context:%p, fd:%d, ip:%s, taosd token:%s parsed success, user:%s", pContext, pContext->fd, httpTrace("context:%p, fd:%d, ip:%s, taosd token:%s parsed success, user:%s", pContext, pContext->fd,
pContext->ipstr, token, pContext->user); pContext->ipstr, token, pContext->user);

2
src/plugins/http/src/httpSql.c
View File

@ -87,7 +87,7 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) {
} }
if (code < 0) { if (code < 0) {
if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code >= 0 ? 0 : code)) { if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, -code)) {
singleCmd->code = code; singleCmd->code = code;
httpTrace("context:%p, fd:%d, ip:%s, user:%s, process pos jump to:%d, last code:%s, last sql:%s", httpTrace("context:%p, fd:%d, ip:%s, user:%s, process pos jump to:%d, last code:%s, last sql:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos + 1, tstrerror(code), sql); pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos + 1, tstrerror(code), sql);

4
src/plugins/http/src/httpUtil.c
View File

@ -307,7 +307,7 @@ void httpTrimTableName(char *name) {
for (int i = 0; name[i] != 0; i++) { for (int i = 0; name[i] != 0; i++) {
if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'') if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'')
name[i] = '_'; name[i] = '_';
if (i == TSDB_TABLE_NAME_LEN + 1) { if (i == TSDB_TABLE_NAME_LEN) {
name[i] = 0; name[i] = 0;
break; break;
} }
@ -323,7 +323,7 @@ int httpShrinkTableName(HttpContext *pContext, int pos, char *name) {
len++; len++;
} }
if (len < TSDB_TABLE_NAME_LEN) { if (len < TSDB_TABLE_NAME_LEN - 1) {
return pos; return pos;
} }

32
src/plugins/http/src/restHandle.c
View File

@ -22,11 +22,37 @@
static HttpDecodeMethod restDecodeMethod = {"rest", restProcessRequest}; static HttpDecodeMethod restDecodeMethod = {"rest", restProcessRequest};
static HttpDecodeMethod restDecodeMethod2 = {"restful", restProcessRequest}; static HttpDecodeMethod restDecodeMethod2 = {"restful", restProcessRequest};
static HttpEncodeMethod restEncodeSqlTimestampMethod = { static HttpEncodeMethod restEncodeSqlTimestampMethod = {
restStartSqlJson, restStopSqlJson, restBuildSqlTimestampJson, restBuildSqlAffectRowsJson, NULL, NULL, NULL, NULL}; .startJsonFp = restStartSqlJson,
.stopJsonFp = restStopSqlJson,
.buildQueryJsonFp = restBuildSqlTimestampJson,
.buildAffectRowJsonFp = restBuildSqlAffectRowsJson,
.initJsonFp = NULL,
.cleanJsonFp = NULL,
.checkFinishedFp = NULL,
.setNextCmdFp = NULL
};
static HttpEncodeMethod restEncodeSqlLocalTimeStringMethod = { static HttpEncodeMethod restEncodeSqlLocalTimeStringMethod = {
restStartSqlJson, restStopSqlJson, restBuildSqlLocalTimeStringJson, restBuildSqlAffectRowsJson, NULL, NULL, NULL, NULL}; .startJsonFp = restStartSqlJson,
.stopJsonFp = restStopSqlJson,
.buildQueryJsonFp = restBuildSqlLocalTimeStringJson,
.buildAffectRowJsonFp = restBuildSqlAffectRowsJson,
.initJsonFp = NULL,
.cleanJsonFp = NULL,
.checkFinishedFp = NULL,
.setNextCmdFp = NULL
};
static HttpEncodeMethod restEncodeSqlUtcTimeStringMethod = { static HttpEncodeMethod restEncodeSqlUtcTimeStringMethod = {
restStartSqlJson, restStopSqlJson, restBuildSqlUtcTimeStringJson, restBuildSqlAffectRowsJson, NULL, NULL, NULL, NULL}; .startJsonFp = restStartSqlJson,
.stopJsonFp = restStopSqlJson,
.buildQueryJsonFp = restBuildSqlUtcTimeStringJson,
.buildAffectRowJsonFp = restBuildSqlAffectRowsJson,
.initJsonFp = NULL,
.cleanJsonFp = NULL,
.checkFinishedFp = NULL,
.setNextCmdFp = NULL
};
void restInitHandle(HttpServer* pServer) { void restInitHandle(HttpServer* pServer) {
httpAddMethod(pServer, &restDecodeMethod); httpAddMethod(pServer, &restDecodeMethod);

21
src/plugins/http/src/tgHandle.c
View File

@ -62,9 +62,16 @@
#define TG_MAX_SORT_TAG_SIZE 20 #define TG_MAX_SORT_TAG_SIZE 20
static HttpDecodeMethod tgDecodeMethod = {"telegraf", tgProcessRquest}; static HttpDecodeMethod tgDecodeMethod = {"telegraf", tgProcessRquest};
static HttpEncodeMethod tgQueryMethod = {tgStartQueryJson, tgStopQueryJson, NULL, static HttpEncodeMethod tgQueryMethod = {
tgBuildSqlAffectRowsJson, tgInitQueryJson, tgCleanQueryJson, .startJsonFp = tgStartQueryJson,
tgCheckFinished, tgSetNextCmd}; .stopJsonFp = tgStopQueryJson,
.buildQueryJsonFp = NULL,
.buildAffectRowJsonFp = tgBuildSqlAffectRowsJson,
.initJsonFp = tgInitQueryJson,
.cleanJsonFp = tgCleanQueryJson,
.checkFinishedFp = tgCheckFinished,
.setNextCmdFp = tgSetNextCmd
};
static const char DEFAULT_TELEGRAF_CFG[] = static const char DEFAULT_TELEGRAF_CFG[] =
"{\"metrics\":[" "{\"metrics\":["
@ -202,7 +209,7 @@ void tgParseSchemaMetric(cJSON *metric) {
goto ParseEnd; goto ParseEnd;
} }
int nameLen = (int)strlen(field->valuestring); int nameLen = (int)strlen(field->valuestring);
if (nameLen == 0 || nameLen > TSDB_TABLE_NAME_LEN) { if (nameLen == 0 || nameLen >= TSDB_TABLE_NAME_LEN) {
parsedOk = false; parsedOk = false;
goto ParseEnd; goto ParseEnd;
} }
@ -303,7 +310,7 @@ bool tgGetUserFromUrl(HttpContext *pContext) {
return false; return false;
} }
strcpy(pContext->user, pParser->path[TG_USER_URL_POS].pos); tstrncpy(pContext->user, pParser->path[TG_USER_URL_POS].pos, TSDB_USER_LEN);
return true; return true;
} }
@ -402,7 +409,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_NULL); httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_NULL);
return false; return false;
} }
if (nameLen >= TSDB_TABLE_NAME_LEN - 7) { if (nameLen >= TSDB_TABLE_NAME_LEN - 8) {
httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_LONG); httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_LONG);
return false; return false;
} }
@ -491,7 +498,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
return false; return false;
} }
if (strlen(host->valuestring) >= TSDB_TABLE_NAME_LEN) { if (strlen(host->valuestring) >= TSDB_TABLE_NAME_LEN - 1) {
httpSendErrorResp(pContext, HTTP_TG_TABLE_SIZE); httpSendErrorResp(pContext, HTTP_TG_TABLE_SIZE);
return false; return false;
} }

2
src/plugins/monitor/src/monitorMain.c
View File

@ -156,7 +156,7 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
", io_read float, io_write float" ", io_read float, io_write float"
", req_http int, req_select int, req_insert int" ", req_http int, req_select int, req_insert int"
") tags (dnodeid int, fqdn binary(%d))", ") tags (dnodeid int, fqdn binary(%d))",
tsMonitorDbName, TSDB_FQDN_LEN + 1); tsMonitorDbName, TSDB_FQDN_LEN);
} else if (cmd == MONITOR_CMD_CREATE_TB_DN) { } else if (cmd == MONITOR_CMD_CREATE_TB_DN) {
snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn%d using %s.dn tags(%d, '%s')", tsMonitorDbName, snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn%d using %s.dn tags(%d, '%s')", tsMonitorDbName,
dnodeGetDnodeId(), tsMonitorDbName, dnodeGetDnodeId(), tsLocalEp); dnodeGetDnodeId(), tsMonitorDbName, dnodeGetDnodeId(), tsLocalEp);

14
src/query/inc/qfill.h
View File

@ -50,7 +50,8 @@ typedef struct SFillInfo {
char * nextValues; // next row of data char * nextValues; // next row of data
char** pData; // original result data block involved in filling data char** pData; // original result data block involved in filling data
int32_t capacityInRows; // data buffer size in rows int32_t capacityInRows; // data buffer size in rows
int8_t slidingUnit; // sliding time unit
int8_t precision; // time resoluation
SFillColInfo* pFillCol; // column info for fill operations SFillColInfo* pFillCol; // column info for fill operations
} SFillInfo; } SFillInfo;
@ -61,12 +62,13 @@ typedef struct SPoint {
int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, char timeUnit, int16_t precision); int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, char timeUnit, int16_t precision);
SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int32_t numOfCols, int64_t slidingTime, int32_t fillType, SFillColInfo* pFillCol); int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
SFillColInfo* pFillCol);
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp); void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
void taosDestoryFillInfo(SFillInfo *pFillInfo); void* taosDestoryFillInfo(SFillInfo *pFillInfo);
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
@ -74,9 +76,7 @@ void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput)
void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput); void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput);
TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision); int64_t getFilledNumOfRes(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
int64_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows);
int32_t taosNumOfRemainRows(SFillInfo *pFillInfo); int32_t taosNumOfRemainRows(SFillInfo *pFillInfo);

30
src/query/src/qExecutor.c
View File

@ -1368,7 +1368,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
int32_t index = pSqlFuncMsg->colInfo.colIndex; int32_t index = pSqlFuncMsg->colInfo.colIndex;
if (TSDB_COL_IS_TAG(pIndex->flag)) { if (TSDB_COL_IS_TAG(pIndex->flag)) {
if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor
pCtx->inputBytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pCtx->inputBytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
pCtx->inputType = TSDB_DATA_TYPE_BINARY; pCtx->inputType = TSDB_DATA_TYPE_BINARY;
} else { } else {
pCtx->inputBytes = pQuery->tagColList[index].bytes; pCtx->inputBytes = pQuery->tagColList[index].bytes;
@ -1466,7 +1466,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
tfree(pRuntimeEnv->pCtx); tfree(pRuntimeEnv->pCtx);
} }
taosDestoryFillInfo(pRuntimeEnv->pFillInfo); pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo);
destroyResultBuf(pRuntimeEnv->pResultBuf, pQInfo); destroyResultBuf(pRuntimeEnv->pResultBuf, pQInfo);
tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
@ -3557,9 +3557,7 @@ bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) {
* first result row in the actual result set will fill nothing. * first result row in the actual result set will fill nothing.
*/ */
if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime, int32_t numOfTotal = getFilledNumOfRes(pFillInfo, pQuery->window.ekey, pQuery->rec.capacity);
pQuery->slidingTimeUnit, pQuery->precision);
int32_t numOfTotal = taosGetNumOfResultWithFill(pFillInfo, remain, ekey, pQuery->rec.capacity);
return numOfTotal > 0; return numOfTotal > 0;
} }
@ -3601,7 +3599,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
} }
} }
int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t numOfRows, int32_t *numOfInterpo) { int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t *numOfInterpo) {
SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo; SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo;
@ -4013,7 +4011,8 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool
if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) {
SFillColInfo* pColInfo = taosCreateFillColInfo(pQuery); SFillColInfo* pColInfo = taosCreateFillColInfo(pQuery);
pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, 0, 0, pQuery->rec.capacity, pQuery->numOfOutput, pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, 0, 0, pQuery->rec.capacity, pQuery->numOfOutput,
pQuery->slidingTime, pQuery->fillType, pColInfo); pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision,
pQuery->fillType, pColInfo);
} }
// todo refactor // todo refactor
@ -4666,13 +4665,11 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
limitResults(pRuntimeEnv); limitResults(pRuntimeEnv);
break; break;
} else { } else {
TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime, taosFillSetStartInfo(pRuntimeEnv->pFillInfo, pQuery->rec.rows, pQuery->window.ekey);
pQuery->slidingTimeUnit, pQuery->precision);
taosFillSetStartInfo(pRuntimeEnv->pFillInfo, pQuery->rec.rows, ekey);
taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata); taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata);
numOfInterpo = 0; numOfInterpo = 0;
pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, pQuery->rec.rows, &numOfInterpo); pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfInterpo);
if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
limitResults(pRuntimeEnv); limitResults(pRuntimeEnv);
break; break;
@ -4704,8 +4701,7 @@ static void tableQueryImpl(SQInfo *pQInfo) {
* So, we do keep in this procedure instead of launching retrieve procedure for next results. * So, we do keep in this procedure instead of launching retrieve procedure for next results.
*/ */
int32_t numOfInterpo = 0; int32_t numOfInterpo = 0;
int32_t remain = taosNumOfRemainRows(pRuntimeEnv->pFillInfo); pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfInterpo);
pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, remain, &numOfInterpo);
if (pQuery->rec.rows > 0) { if (pQuery->rec.rows > 0) {
limitResults(pRuntimeEnv); limitResults(pRuntimeEnv);
@ -5148,8 +5144,8 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
bytes = tDataTypeDesc[type].nSize; bytes = tDataTypeDesc[type].nSize;
} else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column
type = TSDB_DATA_TYPE_BINARY; type = TSDB_DATA_TYPE_BINARY;
bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
} else { } else{
int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols);
assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags || j == TSDB_TBNAME_COLUMN_INDEX); assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags || j == TSDB_TBNAME_COLUMN_INDEX);
@ -5242,7 +5238,7 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) {
if (pQuery->colList[i].numOfFilters > 0) { if (pQuery->colList[i].numOfFilters > 0) {
SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[j]; SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[j];
memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoData)); memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfo));
pFilterInfo->info = pQuery->colList[i]; pFilterInfo->info = pQuery->colList[i];
pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters; pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters;
@ -6038,7 +6034,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { for(int32_t j = 0; j < pQuery->numOfOutput; ++j) {
if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
char* data = tsdbGetTableName(pQInfo->tsdb, &item->id); char* data = tsdbGetTableName(pQInfo->tsdb, &item->id);
char* dst = pQuery->sdata[j]->data + count * (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); char* dst = pQuery->sdata[j]->data + count * ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE);
memcpy(dst, data, varDataTLen(data)); memcpy(dst, data, varDataTLen(data));
} else {// todo refactor } else {// todo refactor
int16_t type = pExprInfo[j].type; int16_t type = pExprInfo[j].type;

4
src/query/src/qast.c
View File

@ -138,7 +138,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken
memcpy(pNode->pSchema, &pSchema[i], sizeof(SSchema)); memcpy(pNode->pSchema, &pSchema[i], sizeof(SSchema));
} else { } else {
pNode->pSchema->type = TSDB_DATA_TYPE_BINARY; pNode->pSchema->type = TSDB_DATA_TYPE_BINARY;
pNode->pSchema->bytes = TSDB_TABLE_NAME_LEN; pNode->pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
strcpy(pNode->pSchema->name, TSQL_TBNAME_L); strcpy(pNode->pSchema->name, TSQL_TBNAME_L);
pNode->pSchema->colId = -1; pNode->pSchema->colId = -1;
} }
@ -1127,7 +1127,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
left->pSchema = pSchema; left->pSchema = pSchema;
pSchema->type = TSDB_DATA_TYPE_BINARY; pSchema->type = TSDB_DATA_TYPE_BINARY;
pSchema->bytes = TSDB_TABLE_NAME_LEN; pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
strcpy(pSchema->name, TSQL_TBNAME_L); strcpy(pSchema->name, TSQL_TBNAME_L);
pSchema->colId = -1; pSchema->colId = -1;

62
src/query/src/qfill.c
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "qfill.h"
#include "os.h" #include "os.h"
#include "qfill.h"
#include "qextbuffer.h" #include "qextbuffer.h"
#include "taosdef.h" #include "taosdef.h"
#include "taosmsg.h" #include "taosmsg.h"
@ -58,7 +58,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, ch
} }
SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int32_t fillType, SFillColInfo* pFillCol) { int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, SFillColInfo* pFillCol) {
if (fillType == TSDB_FILL_NONE) { if (fillType == TSDB_FILL_NONE) {
return NULL; return NULL;
} }
@ -72,7 +72,9 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_
pFillInfo->pFillCol = pFillCol; pFillInfo->pFillCol = pFillCol;
pFillInfo->numOfTags = numOfTags; pFillInfo->numOfTags = numOfTags;
pFillInfo->numOfCols = numOfCols; pFillInfo->numOfCols = numOfCols;
pFillInfo->precision = precision;
pFillInfo->slidingTime = slidingTime; pFillInfo->slidingTime = slidingTime;
pFillInfo->slidingUnit = slidingUnit;
pFillInfo->pData = malloc(POINTER_BYTES * numOfCols); pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
@ -102,9 +104,9 @@ void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
pFillInfo->numOfTotal = 0; pFillInfo->numOfTotal = 0;
} }
void taosDestoryFillInfo(SFillInfo* pFillInfo) { void* taosDestoryFillInfo(SFillInfo* pFillInfo) {
if (pFillInfo == NULL) { if (pFillInfo == NULL) {
return; return NULL;
} }
tfree(pFillInfo->prevValues); tfree(pFillInfo->prevValues);
@ -119,6 +121,15 @@ void taosDestoryFillInfo(SFillInfo* pFillInfo) {
tfree(pFillInfo->pFillCol); tfree(pFillInfo->pFillCol);
tfree(pFillInfo); tfree(pFillInfo);
return NULL;
}
static TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision) {
if (order == TSDB_ORDER_ASC) {
return ekey;
} else {
return taosGetIntervalStartTimestamp(ekey, timeInterval, slidingTimeUnit, precision);
}
} }
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) { void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
@ -126,8 +137,10 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey)
return; return;
} }
pFillInfo->endKey = taosGetRevisedEndKey(endKey, pFillInfo->order, pFillInfo->slidingTime, pFillInfo->slidingUnit,
pFillInfo->precision);
pFillInfo->rowIdx = 0; pFillInfo->rowIdx = 0;
pFillInfo->endKey = endKey;
pFillInfo->numOfRows = numOfRows; pFillInfo->numOfRows = numOfRows;
// ensure the space // ensure the space
@ -165,36 +178,29 @@ void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInpu
} }
} }
TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision) { int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
if (order == TSDB_ORDER_ASC) { int64_t* tsList = (int64_t*) pFillInfo->pData[0];
return ekey;
} else {
return taosGetIntervalStartTimestamp(ekey, timeInterval, slidingTimeUnit, precision);
}
}
static int32_t taosGetTotalNumOfFilledRes(SFillInfo* pFillInfo, const TSKEY* tsArray, int32_t remain, int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
int64_t nInterval, int64_t ekey) {
if (remain > 0) { // still fill gap within current data block, not generating data after the result set. TSKEY ekey1 = taosGetRevisedEndKey(ekey, pFillInfo->order, pFillInfo->slidingTime, pFillInfo->slidingUnit,
TSKEY lastKey = tsArray[pFillInfo->numOfRows - 1]; pFillInfo->precision);
int32_t total = (int32_t)(labs(lastKey - pFillInfo->start) / nInterval) + 1;
assert(total >= remain); int64_t numOfRes = -1;
return total; if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set.
TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];
numOfRes = (int64_t)(labs(lastKey - pFillInfo->start) / pFillInfo->slidingTime) + 1;
assert(numOfRes >= numOfRows);
} else { // reach the end of data } else { // reach the end of data
if ((ekey < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) || if ((ekey1 < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) ||
(ekey > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) { (ekey1 > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) {
return 0; return 0;
} else { } else { // the numOfRes rows are all filled with specified policy
return (int32_t)(labs(ekey - pFillInfo->start) / nInterval) + 1; numOfRes = (labs(ekey1 - pFillInfo->start) / pFillInfo->slidingTime) + 1;
} }
} }
}
int64_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows) {
int32_t numOfRes = taosGetTotalNumOfFilledRes(pFillInfo, (int64_t*) pFillInfo->pData[0], numOfRows,
pFillInfo->slidingTime, ekey);
return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
} }
@ -496,8 +502,8 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu
int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) { int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) {
int32_t remain = taosNumOfRemainRows(pFillInfo); // todo use iterator? int32_t remain = taosNumOfRemainRows(pFillInfo); // todo use iterator?
int32_t rows = taosGetNumOfResultWithFill(pFillInfo, remain, pFillInfo->endKey, capacity);
int32_t rows = getFilledNumOfRes(pFillInfo, pFillInfo->endKey, capacity);
int32_t numOfRes = generateDataBlockImpl(pFillInfo, output, remain, rows, pFillInfo->pData); int32_t numOfRes = generateDataBlockImpl(pFillInfo, output, remain, rows, pFillInfo->pData);
assert(numOfRes == rows); assert(numOfRes == rows);

2
src/query/src/qtsbuf.c
View File

@ -51,7 +51,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
return NULL; return NULL;
} }
strncpy(pTSBuf->path, path, PATH_MAX); tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path));
pTSBuf->f = fopen(pTSBuf->path, "r+"); pTSBuf->f = fopen(pTSBuf->path, "r+");
if (pTSBuf->f == NULL) { if (pTSBuf->f == NULL) {

2
src/rpc/src/rpcMain.c
View File

@ -458,7 +458,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
pInfo->clientPort = pConn->peerPort; pInfo->clientPort = pConn->peerPort;
// pInfo->serverIp = pConn->destIp; // pInfo->serverIp = pConn->destIp;
strncpy(pInfo->user, pConn->user, sizeof(pInfo->user)); tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user));
return 0; return 0;
} }

25
src/rpc/src/rpcTcp.c
View File

@ -67,7 +67,7 @@ static void *taosProcessTcpData(void *param);
static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd); static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd);
static void taosFreeFdObj(SFdObj *pFdObj); static void taosFreeFdObj(SFdObj *pFdObj);
static void taosReportBrokenLink(SFdObj *pFdObj); static void taosReportBrokenLink(SFdObj *pFdObj);
static void* taosAcceptTcpConnection(void *arg); static void *taosAcceptTcpConnection(void *arg);
void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
SServerObj *pServerObj; SServerObj *pServerObj;
@ -80,6 +80,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
return NULL; return NULL;
} }
pServerObj->fd = -1;
pServerObj->thread = 0; pServerObj->thread = 0;
pServerObj->ip = ip; pServerObj->ip = ip;
pServerObj->port = port; pServerObj->port = port;
@ -99,6 +100,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
pthread_attr_init(&thattr); pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
// initialize parameters in case it may encounter error later
pThreadObj = pServerObj->pThreadObj; pThreadObj = pServerObj->pThreadObj;
for (int i = 0; i < numOfThreads; ++i) { for (int i = 0; i < numOfThreads; ++i) {
pThreadObj->pollFd = -1; pThreadObj->pollFd = -1;
@ -106,18 +108,21 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
pThreadObj->processData = fp; pThreadObj->processData = fp;
tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label));
pThreadObj->shandle = shandle; pThreadObj->shandle = shandle;
pThreadObj++;
}
// initialize mutex, thread, fd which may fail
pThreadObj = pServerObj->pThreadObj;
for (int i = 0; i < numOfThreads; ++i) {
code = pthread_mutex_init(&(pThreadObj->mutex), NULL); code = pthread_mutex_init(&(pThreadObj->mutex), NULL);
if (code < 0) { if (code < 0) {
tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
break;; break;;
} }
pThreadObj->pollFd = epoll_create(10); // size does not matter pThreadObj->pollFd = epoll_create(10); // size does not matter
if (pThreadObj->pollFd < 0) { if (pThreadObj->pollFd < 0) {
tError("%s failed to create TCP epoll", label); tError("%s failed to create TCP epoll", label);
terrno = TAOS_SYSTEM_ERROR(errno);
code = -1; code = -1;
break; break;
} }
@ -125,7 +130,6 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj));
if (code != 0) { if (code != 0) {
tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); tError("%s failed to create TCP process data thread(%s)", label, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
break; break;
} }
@ -133,15 +137,18 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
pThreadObj++; pThreadObj++;
} }
pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
if (pServerObj->fd < 0) code = -1;
if (code == 0) { if (code == 0) {
code = pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)); code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj);
if (code != 0) { if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tError("%s failed to create TCP accept thread(%s)", label, strerror(errno)); tError("%s failed to create TCP accept thread(%s)", label, strerror(errno));
} }
} }
if (code != 0) { if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCleanUpTcpServer(pServerObj); taosCleanUpTcpServer(pServerObj);
pServerObj = NULL; pServerObj = NULL;
} else { } else {
@ -204,7 +211,7 @@ void taosCleanUpTcpServer(void *handle) {
tfree(pServerObj); tfree(pServerObj);
} }
static void* taosAcceptTcpConnection(void *arg) { static void *taosAcceptTcpConnection(void *arg) {
int connFd = -1; int connFd = -1;
struct sockaddr_in caddr; struct sockaddr_in caddr;
int threadId = 0; int threadId = 0;
@ -212,10 +219,6 @@ static void* taosAcceptTcpConnection(void *arg) {
SServerObj *pServerObj; SServerObj *pServerObj;
pServerObj = (SServerObj *)arg; pServerObj = (SServerObj *)arg;
pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
if (pServerObj->fd < 0) return NULL;
tTrace("%s TCP server is ready, ip:0x%x:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); tTrace("%s TCP server is ready, ip:0x%x:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
while (1) { while (1) {

54
src/rpc/src/rpcUdp.c
View File

@ -19,6 +19,7 @@
#include "ttimer.h" #include "ttimer.h"
#include "tutil.h" #include "tutil.h"
#include "taosdef.h" #include "taosdef.h"
#include "taoserror.h"
#include "rpcLog.h" #include "rpcLog.h"
#include "rpcUdp.h" #include "rpcUdp.h"
#include "rpcHead.h" #include "rpcHead.h"
@ -65,6 +66,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
pSet = (SUdpConnSet *)malloc((size_t)size); pSet = (SUdpConnSet *)malloc((size_t)size);
if (pSet == NULL) { if (pSet == NULL) {
tError("%s failed to allocate UdpConn", label); tError("%s failed to allocate UdpConn", label);
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL; return NULL;
} }
@ -73,30 +75,34 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
pSet->port = port; pSet->port = port;
pSet->shandle = shandle; pSet->shandle = shandle;
pSet->fp = fp; pSet->fp = fp;
pSet->threads = threads;
tstrncpy(pSet->label, label, sizeof(pSet->label)); tstrncpy(pSet->label, label, sizeof(pSet->label));
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
int i;
uint16_t ownPort; uint16_t ownPort;
for (int i = 0; i < threads; ++i) { for (i = 0; i < threads; ++i) {
pConn = pSet->udpConn + i; pConn = pSet->udpConn + i;
ownPort = (port ? port + i : 0); ownPort = (port ? port + i : 0);
pConn->fd = taosOpenUdpSocket(ip, ownPort); pConn->fd = taosOpenUdpSocket(ip, ownPort);
if (pConn->fd < 0) { if (pConn->fd < 0) {
tError("%s failed to open UDP socket %x:%hu", label, ip, port); tError("%s failed to open UDP socket %x:%hu", label, ip, port);
taosCleanUpUdpConnection(pSet); break;
return NULL;
} }
pConn->buffer = malloc(RPC_MAX_UDP_SIZE); pConn->buffer = malloc(RPC_MAX_UDP_SIZE);
if (NULL == pConn->buffer) { if (NULL == pConn->buffer) {
tError("%s failed to malloc recv buffer", label); tError("%s failed to malloc recv buffer", label);
taosCleanUpUdpConnection(pSet); break;
return NULL;
} }
struct sockaddr_in sin; struct sockaddr_in sin;
unsigned int addrlen = sizeof(sin); unsigned int addrlen = sizeof(sin);
if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 &&
addrlen == sizeof(sin)) { sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
pConn->localPort = (uint16_t)ntohs(sin.sin_port); pConn->localPort = (uint16_t)ntohs(sin.sin_port);
} }
@ -107,23 +113,22 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
pConn->pSet = pSet; pConn->pSet = pSet;
pConn->signature = pConn; pConn->signature = pConn;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn); int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn);
pthread_attr_destroy(&thAttr);
if (code != 0) { if (code != 0) {
tError("%s failed to create thread to process UDP data, reason:%s", label, strerror(errno)); tError("%s failed to create thread to process UDP data(%s)", label, strerror(errno));
taosCloseSocket(pConn->fd); break;
taosCleanUpUdpConnection(pSet);
return NULL;
} }
++pSet->threads;
} }
tTrace("%s UDP connection is initialized, ip:%x port:%hu threads:%d", label, ip, port, threads); pthread_attr_destroy(&thAttr);
if (i != threads) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCleanUpUdpConnection(pSet);
return NULL;
}
tTrace("%s UDP connection is initialized, ip:%x:%hu threads:%d", label, ip, port, threads);
return pSet; return pSet;
} }
@ -136,16 +141,17 @@ void taosCleanUpUdpConnection(void *handle) {
for (int i = 0; i < pSet->threads; ++i) { for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i; pConn = pSet->udpConn + i;
pConn->signature = NULL; pConn->signature = NULL;
// shutdown to signal the thread to exit // shutdown to signal the thread to exit
shutdown(pConn->fd, SHUT_RD); if ( pConn->fd >=0) shutdown(pConn->fd, SHUT_RD);
} }
for (int i = 0; i < pSet->threads; ++i) { for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i; pConn = pSet->udpConn + i;
pthread_join(pConn->thread, NULL); if (pConn->thread) pthread_join(pConn->thread, NULL);
free(pConn->buffer); if (pConn->fd >=0) taosCloseSocket(pConn->fd);
taosCloseSocket(pConn->fd); tfree(pConn->buffer);
tTrace("chandle:%p is closed", pConn); tTrace("UDP chandle:%p is closed", pConn);
} }
tfree(pSet); tfree(pSet);
@ -159,7 +165,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
SUdpConn *pConn = pSet->udpConn + pSet->index; SUdpConn *pConn = pSet->udpConn + pSet->index;
pConn->port = port; pConn->port = port;
tTrace("%s UDP connection is setup, ip:%x:%hu, local:%x:%d", pConn->label, ip, port, pSet->ip, pConn->localPort); tTrace("%s UDP connection is setup, ip:%x:%hu", pConn->label, ip, port);
return pConn; return pConn;
} }

2
src/tsdb/inc/tsdbMain.h
View File

@ -357,6 +357,8 @@ typedef struct STsdbRepo {
STsdbAppH appH; STsdbAppH appH;
STsdbStat stat;
// The meter meta handle of this TSDB repository // The meter meta handle of this TSDB repository
STsdbMeta *tsdbMeta; STsdbMeta *tsdbMeta;

16
src/tsdb/src/tsdbMain.c
View File

@ -953,6 +953,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) { static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) {
STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbRepo *pRepo = (STsdbRepo *)repo;
STsdbMeta *pMeta = pRepo->tsdbMeta; STsdbMeta *pMeta = pRepo->tsdbMeta;
int64_t points = 0;
STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid}; STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid};
STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId); STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId);
@ -964,7 +965,9 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY
// Check schema version // Check schema version
int32_t tversion = pBlock->sversion; int32_t tversion = pBlock->sversion;
int16_t nversion = schemaVersion(tsdbGetTableSchema(pMeta, pTable)); STSchema * pSchema = tsdbGetTableSchema(pMeta, pTable);
ASSERT(pSchema != NULL);
int16_t nversion = schemaVersion(pSchema);
if (tversion > nversion) { if (tversion > nversion) {
tsdbTrace("vgId:%d table:%s tid:%d server schema version %d is older than clien version %d, try to config.", tsdbTrace("vgId:%d table:%s tid:%d server schema version %d is older than clien version %d, try to config.",
pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, nversion, tversion); pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, nversion, tversion);
@ -1014,7 +1017,10 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY
return -1; return -1;
} }
(*affectedrows)++; (*affectedrows)++;
points++;
} }
atomic_fetch_add_64(&(pRepo->stat.pointsWritten), points * (pSchema->numOfCols));
atomic_fetch_add_64(&(pRepo->stat.totalStorage), points * pSchema->vlen);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1381,3 +1387,11 @@ uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t
return magic; return magic;
} }
void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage){
ASSERT(repo != NULL);
STsdbRepo * pRepo = repo;
*totalPoints = pRepo->stat.pointsWritten;
*totalStorage = pRepo->stat.totalStorage;
*compStorage = pRepo->stat.compStorage;
}

4
src/tsdb/src/tsdbMeta.c
View File

@ -334,7 +334,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
pTable->schema[0] = tdDupSchema(pCfg->schema); pTable->schema[0] = tdDupSchema(pCfg->schema);
pTable->tagSchema = tdDupSchema(pCfg->tagSchema); pTable->tagSchema = tdDupSchema(pCfg->tagSchema);
tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN); tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN - 1);
pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1); pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1);
if (pTable->name == NULL) { if (pTable->name == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@ -355,7 +355,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
pTable->tableId.tid = pCfg->tableId.tid; pTable->tableId.tid = pCfg->tableId.tid;
pTable->lastKey = TSKEY_INITIAL_VAL; pTable->lastKey = TSKEY_INITIAL_VAL;
tsize = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN); tsize = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN - 1);
pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1); pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1);
if (pTable->name == NULL) { if (pTable->name == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;

225
src/tsdb/src/tsdbRead.c
View File

@ -233,8 +233,6 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond*
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
// pQueryHandle->outputCapacity = 2; // only allowed two rows to be loaded
changeQueryHandleForInterpQuery(pQueryHandle); changeQueryHandleForInterpQuery(pQueryHandle);
return pQueryHandle; return pQueryHandle;
} }
@ -618,54 +616,19 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pQueryHandle->cur;
SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlock); SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlock);
/*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
SDataRow row = getSDataRowInTableMem(pCheckInfo);
TSKEY k1 = TSKEY_INITIAL_VAL, k2 = TSKEY_INITIAL_VAL; TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL;
if (pCheckInfo->iter != NULL && tSkipListIterGet(pCheckInfo->iter) != NULL) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
SDataRow row = SL_GET_NODE_DATA(node);
k1 = dataRowKey(row);
if (k1 == binfo.window.skey) {
if (tSkipListIterNext(pCheckInfo->iter)) {
node = tSkipListIterGet(pCheckInfo->iter);
row = SL_GET_NODE_DATA(node);
k1 = dataRowKey(row);
} else {
k1 = TSKEY_INITIAL_VAL;
}
}
}
if (pCheckInfo->iiter != NULL && tSkipListIterGet(pCheckInfo->iiter) != NULL) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
SDataRow row = SL_GET_NODE_DATA(node);
k2 = dataRowKey(row);
if (k2 == binfo.window.skey) {
if (tSkipListIterNext(pCheckInfo->iiter)) {
node = tSkipListIterGet(pCheckInfo->iiter);
row = SL_GET_NODE_DATA(node);
k2 = dataRowKey(row);
} else {
k2 = TSKEY_INITIAL_VAL;
}
}
}
cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1); cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1);
if ((ASCENDING_TRAVERSE(pQueryHandle->order) && if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey) || (k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey))) || (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
(!ASCENDING_TRAVERSE(pQueryHandle->order) &&
((k1 != TSKEY_INITIAL_VAL && k1 >= binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 >= binfo.window.skey)))) { if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
if ((ASCENDING_TRAVERSE(pQueryHandle->order) &&
((k1 != TSKEY_INITIAL_VAL && k1 < binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 < binfo.window.skey))) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) &&
(((k1 != TSKEY_INITIAL_VAL && k1 > binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 > binfo.window.skey))))) {
// do not load file block into buffer // do not load file block into buffer
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
@ -756,7 +719,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
return pQueryHandle->realNumOfRows > 0; return pQueryHandle->realNumOfRows > 0;
} }
static int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) { static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
int firstPos, lastPos, midPos = -1; int firstPos, lastPos, midPos = -1;
int numOfRows; int numOfRows;
TSKEY* keyList; TSKEY* keyList;
@ -868,14 +831,21 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap
return numOfRows + num; return numOfRows + num;
} }
static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t capacity, static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row,
int32_t numOfRows, SDataRow row, STSchema* pSchema) { STsdbMeta *pMeta, int32_t numOfCols, STable* pTable) {
int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
int32_t numOfTableCols = schemaNCols(pSchema);
char* pData = NULL; char* pData = NULL;
for (int32_t i = 0; i < numOfCols; ++i) {
// the schema version info is embeded in SDataRow
STSchema* pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, dataRowVersion(row));
int32_t numOfRowCols = schemaNCols(pSchema);
int32_t i = 0, j = 0;
while(i < numOfCols && j < numOfRowCols) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
if (pSchema->columns[j].colId < pColInfo->info.colId) {
j++;
continue;
}
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
pData = pColInfo->pData + numOfRows * pColInfo->info.bytes; pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
@ -883,22 +853,41 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, STableCheckInfo* p
pData = pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; pData = pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
} }
int32_t offset = 0; if (pSchema->columns[j].colId == pColInfo->info.colId) {
for (int32_t j = 0; j < numOfTableCols; ++j) { void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset);
if (pColInfo->info.colId == pSchema->columns[j].colId) { if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
offset = pSchema->columns[j].offset; memcpy(pData, value, varDataTLen(value));
break; } else {
memcpy(pData, value, pColInfo->info.bytes);
} }
}
assert(offset != -1); // todo handle error j++;
void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset); i++;
} else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pData, pColInfo->info.type);
} else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
}
i++;
}
}
while (i < numOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else {
pData = pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
}
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
memcpy(pData, value, varDataTLen(value)); setVardataNull(pData, pColInfo->info.type);
} else { } else {
memcpy(pData, value, pColInfo->info.bytes); setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
} }
i++;
} }
} }
@ -912,6 +901,15 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
initTableMemIterator(pQueryHandle, pCheckInfo); initTableMemIterator(pQueryHandle, pCheckInfo);
SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
// for search the endPos, so the order needs to reverse
int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
STable* pTable = pCheckInfo->pTableObj;
int32_t endPos = cur->pos; int32_t endPos = cur->pos;
if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) {
endPos = blockInfo.rows - 1; endPos = blockInfo.rows - 1;
@ -920,8 +918,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
endPos = 0; endPos = 0;
cur->mixBlock = (cur->pos != blockInfo.rows - 1); cur->mixBlock = (cur->pos != blockInfo.rows - 1);
} else { } else {
int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; assert(pCols->numOfRows > 0);
endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
cur->mixBlock = true; cur->mixBlock = true;
} }
@ -933,7 +931,6 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
int32_t numOfRows = 0; int32_t numOfRows = 0;
pQueryHandle->cur.win = TSWINDOW_INITIALIZER; pQueryHandle->cur.win = TSWINDOW_INITIALIZER;
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
// no data in buffer, load data from file directly // no data in buffer, load data from file directly
if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) { if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) {
@ -950,12 +947,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
// todo opt in case of no data in buffer // todo opt in case of no data in buffer
numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end); numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
// if the buffer is not full in case of descending order query, move the data in the front of the buffer // if the buffer is not full in case of descending order query, move the data in the front of the buffer
if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < pQueryHandle->outputCapacity) { if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < pQueryHandle->outputCapacity) {
int32_t emptySize = pQueryHandle->outputCapacity - numOfRows; int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
int32_t reqNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
for(int32_t i = 0; i < reqNumOfCols; ++i) { for(int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
} }
@ -969,20 +965,15 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
pQueryHandle->realNumOfRows = numOfRows; pQueryHandle->realNumOfRows = numOfRows;
cur->rows = numOfRows; cur->rows = numOfRows;
return; return;
} else if (pCheckInfo->iter != NULL && pCheckInfo->iiter == NULL) { } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
// } else if (pCheckInfo->iter == NULL && pCheckInfo->iiter != NULL) {
// } else { // iter and iiter are all not NULL, three-way merge data block
STSchema* pSchema = tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj);
SSkipListNode* node = NULL; SSkipListNode* node = NULL;
do { do {
node = tSkipListIterGet(pCheckInfo->iter); SDataRow row = getSDataRowInTableMem(pCheckInfo);
if (node == NULL) { if (row == NULL) {
break; break;
} }
SDataRow row = SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row);
TSKEY key = dataRowKey(row);
if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) || if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { (key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
break; break;
@ -995,7 +986,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
copyOneRowFromMem(pQueryHandle, pCheckInfo, pQueryHandle->outputCapacity, numOfRows, row, pSchema); copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, pMeta, numOfCols, pTable);
numOfRows += 1; numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) { if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key; cur->win.skey = key;
@ -1005,17 +996,16 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
cur->lastKey = key + step; cur->lastKey = key + step;
cur->mixBlock = true; cur->mixBlock = true;
tSkipListIterNext(pCheckInfo->iter); moveToNextRow(pCheckInfo);
} else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it
tSkipListIterNext(pCheckInfo->iter); moveToNextRow(pCheckInfo);
} else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
if (cur->win.skey == TSKEY_INITIAL_VAL) { if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = tsArray[pos]; cur->win.skey = tsArray[pos];
} }
int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order);
int32_t end = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order);
if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
tSkipListIterNext(pCheckInfo->iter); tSkipListIterNext(pCheckInfo->iter);
} }
@ -1093,9 +1083,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
// if the buffer is not full in case of descending order query, move the data in the front of the buffer // if the buffer is not full in case of descending order query, move the data in the front of the buffer
if (numOfRows < pQueryHandle->outputCapacity) { if (numOfRows < pQueryHandle->outputCapacity) {
int32_t emptySize = pQueryHandle->outputCapacity - numOfRows; int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
for(int32_t i = 0; i < numOfCols; ++i) {
int32_t requiredNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
for(int32_t i = 0; i < requiredNumOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
} }
@ -1567,9 +1555,6 @@ void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
for(int32_t i = 0; i < numOfTables; ++i) { for(int32_t i = 0; i < numOfTables; ++i) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
if (pCheckInfo->pTableObj->tableId.uid == 12094628167747) {
printf("abc\n");
}
if (pCheckInfo->pTableObj->lastKey > key) { if (pCheckInfo->pTableObj->lastKey > key) {
key = pCheckInfo->pTableObj->lastKey; key = pCheckInfo->pTableObj->lastKey;
index = i; index = i;
@ -1652,8 +1637,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
*skey = TSKEY_INITIAL_VAL; *skey = TSKEY_INITIAL_VAL;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
STSchema* pSchema = tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj); STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
int32_t numOfTableCols = schemaNCols(pSchema); STable* pTable = pCheckInfo->pTableObj;
do { do {
SDataRow row = getSDataRowInTableMem(pCheckInfo); SDataRow row = getSDataRowInTableMem(pCheckInfo);
@ -1662,10 +1647,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
} }
TSKEY key = dataRowKey(row); TSKEY key = dataRowKey(row);
if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
tsdbTrace("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, tsdbTrace("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey,
pQueryHandle->window.ekey); pQueryHandle->window.ekey);
@ -1677,58 +1659,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
} }
*ekey = key; *ekey = key;
char* pData = NULL; copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, pMeta, numOfCols, pTable);
int32_t i = 0, j = 0;
while(i < numOfCols && j < numOfTableCols) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
if (pSchema->columns[j].colId < pColInfo->info.colId) {
j++;
continue;
}
if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else {
pData = pColInfo->pData + (maxRowsToRead - numOfRows - 1) * pColInfo->info.bytes;
}
if (pSchema->columns[j].colId == pColInfo->info.colId) {
void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset);
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
memcpy(pData, value, varDataTLen(value));
} else {
memcpy(pData, value, pColInfo->info.bytes);
}
j++;
i++;
} else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pData, pColInfo->info.type);
} else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
}
i++;
}
}
while (i < numOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else {
pData = pColInfo->pData + (maxRowsToRead - numOfRows - 1) * pColInfo->info.bytes;
}
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pData, pColInfo->info.type);
} else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
}
i++;
}
if (++numOfRows >= maxRowsToRead) { if (++numOfRows >= maxRowsToRead) {
moveToNextRow(pCheckInfo); moveToNextRow(pCheckInfo);
@ -2011,7 +1942,7 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
f1 = (char*) pTable1->name; f1 = (char*) pTable1->name;
f2 = (char*) pTable2->name; f2 = (char*) pTable2->name;
type = TSDB_DATA_TYPE_BINARY; type = TSDB_DATA_TYPE_BINARY;
bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
} else { } else {
STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex); STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex);
bytes = pCol->bytes; bytes = pCol->bytes;

57
src/util/inc/tcache.h
View File

@ -37,8 +37,8 @@ typedef struct SCacheDataNode {
uint64_t expiredTime; // expiredTime expiredTime when this element should be remove from cache uint64_t expiredTime; // expiredTime expiredTime when this element should be remove from cache
uint64_t signature; uint64_t signature;
uint32_t size; // allocated size for current SCacheDataNode uint32_t size; // allocated size for current SCacheDataNode
uint16_t keySize : 15; uint16_t keySize: 15;
bool inTrash : 1; // denote if it is in trash or not bool inTrashCan: 1;// denote if it is in trash or not
T_REF_DECLARE() T_REF_DECLARE()
char *key; char *key;
char data[]; char data[];
@ -50,46 +50,49 @@ typedef struct STrashElem {
SCacheDataNode *pData; SCacheDataNode *pData;
} STrashElem; } STrashElem;
/*
* to accommodate the old data which has the same key value of new one in hashList
* when an new node is put into cache, if an existed one with the same key:
* 1. if the old one does not be referenced, update it.
* 2. otherwise, move the old one to pTrash, addedTime the new one.
*
* when the node in pTrash does not be referenced, it will be release at the expired expiredTime
*/
typedef struct { typedef struct {
int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included. int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included.
int64_t refreshTime; int64_t refreshTime;
STrashElem * pTrash;
/* void * tmrCtrl;
* to accommodate the old datanode which has the same key value of new one in hashList void * pTimer;
* when an new node is put into cache, if an existed one with the same key: SCacheStatis statistics;
* 1. if the old one does not be referenced, update it. SHashObj * pHashTable;
* 2. otherwise, move the old one to pTrash, addedTime the new one.
*
* when the node in pTrash does not be referenced, it will be release at the expired expiredTime
*/
STrashElem * pTrash;
void * tmrCtrl;
void * pTimer;
SCacheStatis statistics;
SHashObj * pHashTable;
_hash_free_fn_t freeFp; _hash_free_fn_t freeFp;
int numOfElemsInTrash; // number of element in trash uint32_t numOfElemsInTrash; // number of element in trash
int16_t deleting; // set the deleting flag to stop refreshing ASAP. uint8_t deleting; // set the deleting flag to stop refreshing ASAP.
T_REF_DECLARE() pthread_t refreshWorker;
#if defined(LINUX) #if defined(LINUX)
pthread_rwlock_t lock; pthread_rwlock_t lock;
#else #else
pthread_mutex_t lock; pthread_mutex_t lock;
#endif #endif
} SCacheObj; } SCacheObj;
/** /**
* * initialize the cache object
* @param maxSessions maximum slots available for hash elements
* @param tmrCtrl timer ctrl
* @param refreshTime refresh operation interval time, the maximum survival time when one element is expired and * @param refreshTime refresh operation interval time, the maximum survival time when one element is expired and
* not referenced by other objects * not referenced by other objects
* @return * @return
*/ */
SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTimeInSeconds); SCacheObj *taosCacheInit(int64_t refreshTimeInSeconds);
SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTimeInSeconds, void (*freeCb)(void *data));
/**
* initialize the cache object and set the free object callback function
* @param refreshTimeInSeconds
* @param freeCb
* @return
*/
SCacheObj *taosCacheInitWithCb(int64_t refreshTimeInSeconds, void (*freeCb)(void *data));
/** /**
* add data into cache * add data into cache

395
src/util/src/tcache.c
View File

@ -77,31 +77,7 @@ static FORCE_INLINE void taosFreeNode(void *data) {
* @param lifespan total survial expiredTime from now * @param lifespan total survial expiredTime from now
* @return SCacheDataNode * @return SCacheDataNode
*/ */
static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration);
uint64_t duration) {
size_t totalSize = size + sizeof(SCacheDataNode) + keyLen + 1;
SCacheDataNode *pNewNode = calloc(1, totalSize);
if (pNewNode == NULL) {
uError("failed to allocate memory, reason:%s", strerror(errno));
return NULL;
}
memcpy(pNewNode->data, pData, size);
pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + size;
pNewNode->keySize = keyLen;
memcpy(pNewNode->key, key, keyLen);
pNewNode->addedTime = (uint64_t)taosGetTimestampMs();
pNewNode->expiredTime = pNewNode->addedTime + duration;
pNewNode->signature = (uint64_t)pNewNode;
pNewNode->size = (uint32_t)totalSize;
return pNewNode;
}
/** /**
* addedTime object node into trash, and this object is closed for referencing if it is addedTime to trash * addedTime object node into trash, and this object is closed for referencing if it is addedTime to trash
@ -109,50 +85,15 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const
* @param pCacheObj Cache object * @param pCacheObj Cache object
* @param pNode Cache slot object * @param pNode Cache slot object
*/ */
static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode);
if (pNode->inTrash) { /* node is already in trash */
return;
}
STrashElem *pElem = calloc(1, sizeof(STrashElem)); /**
pElem->pData = pNode; * remove node in trash can
* @param pCacheObj
* @param pElem
*/
static void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem);
pElem->next = pCacheObj->pTrash;
if (pCacheObj->pTrash) {
pCacheObj->pTrash->prev = pElem;
}
pElem->prev = NULL;
pCacheObj->pTrash = pElem;
pNode->inTrash = true;
pCacheObj->numOfElemsInTrash++;
uTrace("key:%s %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash);
}
static void taosRemoveFromTrash(SCacheObj *pCacheObj, STrashElem *pElem) {
if (pElem->pData->signature != (uint64_t)pElem->pData) {
uError("key:sig:%d %p data has been released, ignore", pElem->pData->signature, pElem->pData);
return;
}
pCacheObj->numOfElemsInTrash--;
if (pElem->prev) {
pElem->prev->next = pElem->next;
} else { /* pnode is the header, update header */
pCacheObj->pTrash = pElem->next;
}
if (pElem->next) {
pElem->next->prev = pElem->prev;
}
pElem->pData->signature = 0;
if (pCacheObj->freeFp) pCacheObj->freeFp(pElem->pData->data);
free(pElem->pData);
free(pElem);
}
/** /**
* remove nodes in trash with refCount == 0 in cache * remove nodes in trash with refCount == 0 in cache
* @param pNode * @param pNode
@ -160,42 +101,7 @@ static void taosRemoveFromTrash(SCacheObj *pCacheObj, STrashElem *pElem) {
* @param force force model, if true, remove data in trash without check refcount. * @param force force model, if true, remove data in trash without check refcount.
* may cause corruption. So, forece model only applys before cache is closed * may cause corruption. So, forece model only applys before cache is closed
*/ */
static void taosTrashEmpty(SCacheObj *pCacheObj, bool force) { static void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force);
__cache_wr_lock(pCacheObj);
if (pCacheObj->numOfElemsInTrash == 0) {
if (pCacheObj->pTrash != NULL) {
uError("key:inconsistency data in cache, numOfElem in trash:%d", pCacheObj->numOfElemsInTrash);
}
pCacheObj->pTrash = NULL;
__cache_unlock(pCacheObj);
return;
}
STrashElem *pElem = pCacheObj->pTrash;
while (pElem) {
T_REF_VAL_CHECK(pElem->pData);
if (pElem->next == pElem) {
pElem->next = NULL;
}
if (force || (T_REF_VAL_GET(pElem->pData) == 0)) {
uTrace("key:%s %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData,
pCacheObj->numOfElemsInTrash - 1);
STrashElem *p = pElem;
pElem = pElem->next;
taosRemoveFromTrash(pCacheObj, p);
} else {
pElem = pElem->next;
}
}
assert(pCacheObj->numOfElemsInTrash >= 0);
__cache_unlock(pCacheObj);
}
/** /**
* release node * release node
@ -304,87 +210,20 @@ static FORCE_INLINE SCacheDataNode *taosAddToCacheImpl(SCacheObj *pCacheObj, con
return pNode; return pNode;
} }
static void doCleanupDataCache(SCacheObj *pCacheObj) { /**
__cache_wr_lock(pCacheObj); * do cleanup the taos cache
* @param pCacheObj
//if (taosHashGetSize(pCacheObj->pHashTable) > 0) { */
taosHashCleanup(pCacheObj->pHashTable); static void doCleanupDataCache(SCacheObj *pCacheObj);
//}
__cache_unlock(pCacheObj);
taosTrashEmpty(pCacheObj, true);
__cache_lock_destroy(pCacheObj);
memset(pCacheObj, 0, sizeof(SCacheObj));
free(pCacheObj);
}
/** /**
* refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pCacheObj->refreshTime * refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pCacheObj->refreshTime
* @param handle Cache object handle * @param handle Cache object handle
*/ */
static void taosCacheRefresh(void *handle, void *tmrId) { static void* taosCacheRefresh(void *handle);
SCacheObj *pCacheObj = (SCacheObj *)handle;
if (pCacheObj == NULL || T_REF_VAL_GET(pCacheObj) == 0) { SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) {
uTrace("object is destroyed. no refresh retry"); if (refreshTime <= 0) {
return;
}
int16_t ref = T_REF_INC(pCacheObj);
if (ref == 1) {
T_REF_DEC(pCacheObj);
return;
}
// todo add the ref before start the timer
int32_t num = taosHashGetSize(pCacheObj->pHashTable);
if (num == 0) {
ref = T_REF_DEC(pCacheObj);
if (ref == 0) {
doCleanupDataCache(pCacheObj);
} else {
taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
}
return;
}
uint64_t expiredTime = taosGetTimestampMs();
pCacheObj->statistics.refreshCount++;
SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable);
__cache_wr_lock(pCacheObj);
while (taosHashIterNext(pIter)) {
if (pCacheObj->deleting == 1) {
taosHashDestroyIter(pIter);
break;
}
SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) {
taosCacheReleaseNode(pCacheObj, pNode);
}
}
__cache_unlock(pCacheObj);
taosHashDestroyIter(pIter);
taosTrashEmpty(pCacheObj, false);
ref = T_REF_DEC(pCacheObj);
if (ref == 0) {
doCleanupDataCache(pCacheObj);
return;
} else {
taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
}
}
SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb)(void *data)) {
if (tmrCtrl == NULL || refreshTime <= 0) {
return NULL; return NULL;
} }
@ -394,7 +233,7 @@ SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb
return NULL; return NULL;
} }
pCacheObj->pHashTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false);
if (pCacheObj->pHashTable == NULL) { if (pCacheObj->pHashTable == NULL) {
free(pCacheObj); free(pCacheObj);
uError("failed to allocate memory, reason:%s", strerror(errno)); uError("failed to allocate memory, reason:%s", strerror(errno));
@ -406,12 +245,8 @@ SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb
pCacheObj->freeFp = freeCb; pCacheObj->freeFp = freeCb;
pCacheObj->refreshTime = refreshTime * 1000; pCacheObj->refreshTime = refreshTime * 1000;
pCacheObj->tmrCtrl = tmrCtrl;
taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
if (__cache_lock_init(pCacheObj) != 0) { if (__cache_lock_init(pCacheObj) != 0) {
taosTmrStopA(&pCacheObj->pTimer);
taosHashCleanup(pCacheObj->pHashTable); taosHashCleanup(pCacheObj->pHashTable);
free(pCacheObj); free(pCacheObj);
@ -419,12 +254,18 @@ SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb
return NULL; return NULL;
} }
T_REF_INC(pCacheObj); pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
pthread_create(&pCacheObj->refreshWorker, &thattr, taosCacheRefresh, pCacheObj);
pthread_attr_destroy(&thattr);
return pCacheObj; return pCacheObj;
} }
SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) { SCacheObj *taosCacheInit(int64_t refreshTime) {
return taosCacheInitWithCb(tmrCtrl, refreshTime, NULL); return taosCacheInitWithCb(refreshTime, NULL);
} }
void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, size_t dataSize, int duration) { void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, size_t dataSize, int duration) {
@ -600,7 +441,7 @@ void taosCacheEmpty(SCacheObj *pCacheObj) {
__cache_unlock(pCacheObj); __cache_unlock(pCacheObj);
taosHashDestroyIter(pIter); taosHashDestroyIter(pIter);
taosTrashEmpty(pCacheObj, false); taosTrashCanEmpty(pCacheObj, false);
} }
void taosCacheCleanup(SCacheObj *pCacheObj) { void taosCacheCleanup(SCacheObj *pCacheObj) {
@ -608,8 +449,180 @@ void taosCacheCleanup(SCacheObj *pCacheObj) {
return; return;
} }
int32_t ref = T_REF_DEC(pCacheObj); pCacheObj->deleting = 1;
if (ref == 0) { pthread_join(pCacheObj->refreshWorker, NULL);
doCleanupDataCache(pCacheObj);
} doCleanupDataCache(pCacheObj);
}
SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size,
uint64_t duration) {
size_t totalSize = size + sizeof(SCacheDataNode) + keyLen + 1;
SCacheDataNode *pNewNode = calloc(1, totalSize);
if (pNewNode == NULL) {
uError("failed to allocate memory, reason:%s", strerror(errno));
return NULL;
}
memcpy(pNewNode->data, pData, size);
pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + size;
pNewNode->keySize = keyLen;
memcpy(pNewNode->key, key, keyLen);
pNewNode->addedTime = (uint64_t)taosGetTimestampMs();
pNewNode->expiredTime = pNewNode->addedTime + duration;
pNewNode->signature = (uint64_t)pNewNode;
pNewNode->size = (uint32_t)totalSize;
return pNewNode;
}
void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
if (pNode->inTrashCan) { /* node is already in trash */
return;
}
STrashElem *pElem = calloc(1, sizeof(STrashElem));
pElem->pData = pNode;
pElem->next = pCacheObj->pTrash;
if (pCacheObj->pTrash) {
pCacheObj->pTrash->prev = pElem;
}
pElem->prev = NULL;
pCacheObj->pTrash = pElem;
pNode->inTrashCan = true;
pCacheObj->numOfElemsInTrash++;
uTrace("key:%s %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash);
}
void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) {
if (pElem->pData->signature != (uint64_t)pElem->pData) {
uError("key:sig:%d %p data has been released, ignore", pElem->pData->signature, pElem->pData);
return;
}
pCacheObj->numOfElemsInTrash--;
if (pElem->prev) {
pElem->prev->next = pElem->next;
} else { /* pnode is the header, update header */
pCacheObj->pTrash = pElem->next;
}
if (pElem->next) {
pElem->next->prev = pElem->prev;
}
pElem->pData->signature = 0;
if (pCacheObj->freeFp) pCacheObj->freeFp(pElem->pData->data);
free(pElem->pData);
free(pElem);
}
void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) {
__cache_wr_lock(pCacheObj);
if (pCacheObj->numOfElemsInTrash == 0) {
if (pCacheObj->pTrash != NULL) {
uError("key:inconsistency data in cache, numOfElem in trash:%d", pCacheObj->numOfElemsInTrash);
}
pCacheObj->pTrash = NULL;
__cache_unlock(pCacheObj);
return;
}
STrashElem *pElem = pCacheObj->pTrash;
while (pElem) {
T_REF_VAL_CHECK(pElem->pData);
if (pElem->next == pElem) {
pElem->next = NULL;
}
if (force || (T_REF_VAL_GET(pElem->pData) == 0)) {
uTrace("key:%s %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData,
pCacheObj->numOfElemsInTrash - 1);
STrashElem *p = pElem;
pElem = pElem->next;
taosRemoveFromTrashCan(pCacheObj, p);
} else {
pElem = pElem->next;
}
}
assert(pCacheObj->numOfElemsInTrash >= 0);
__cache_unlock(pCacheObj);
}
void doCleanupDataCache(SCacheObj *pCacheObj) {
__cache_wr_lock(pCacheObj);
taosHashCleanup(pCacheObj->pHashTable);
__cache_unlock(pCacheObj);
taosTrashCanEmpty(pCacheObj, true);
__cache_lock_destroy(pCacheObj);
memset(pCacheObj, 0, sizeof(SCacheObj));
free(pCacheObj);
}
void* taosCacheRefresh(void *handle) {
SCacheObj *pCacheObj = (SCacheObj *)handle;
if (pCacheObj == NULL) {
uTrace("object is destroyed. no refresh retry");
return NULL;
}
const int32_t SLEEP_DURATION = 500; //500 ms
int64_t totalTick = pCacheObj->refreshTime / SLEEP_DURATION;
int64_t count = 0;
while(1) {
taosMsleep(500);
// check if current cache object will be deleted every 500ms.
if (pCacheObj->deleting) {
break;
}
if (++count < totalTick) {
continue;
}
// reset the count value
count = 0;
size_t num = taosHashGetSize(pCacheObj->pHashTable);
if (num == 0) {
continue;
}
uint64_t expiredTime = taosGetTimestampMs();
pCacheObj->statistics.refreshCount++;
SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable);
__cache_wr_lock(pCacheObj);
while (taosHashIterNext(pIter)) {
SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) {
taosCacheReleaseNode(pCacheObj, pNode);
}
}
__cache_unlock(pCacheObj);
taosHashDestroyIter(pIter);
taosTrashCanEmpty(pCacheObj, false);
}
return NULL;
} }

15
src/util/src/tlog.c
View File

@ -191,15 +191,14 @@ void taosResetLog() {
} }
static bool taosCheckFileIsOpen(char *logFileName) { static bool taosCheckFileIsOpen(char *logFileName) {
int32_t exist = access(logFileName, F_OK); int32_t fd = open(logFileName, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
if (exist != 0) {
return false;
}
int32_t fd = open(logFileName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (fd < 0) { if (fd < 0) {
printf("\nfailed to open log file:%s, reason:%s\n", logFileName, strerror(errno)); if (errno == ENOENT) {
return true; return false;
} else {
printf("\nfailed to open log file:%s, reason:%s\n", logFileName, strerror(errno));
return true;
}
} }
if (taosLockFile(fd)) { if (taosLockFile(fd)) {

4
src/util/src/tsched.c
View File

@ -15,6 +15,7 @@
#include "os.h" #include "os.h"
#include "taosdef.h" #include "taosdef.h"
#include "tutil.h"
#include "tulog.h" #include "tulog.h"
#include "tsched.h" #include "tsched.h"
#include "ttimer.h" #include "ttimer.h"
@ -62,8 +63,7 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) {
} }
pSched->queueSize = queueSize; pSched->queueSize = queueSize;
strncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow tstrncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow
pSched->label[sizeof(pSched->label)-1] = '\0';
pSched->fullSlot = 0; pSched->fullSlot = 0;
pSched->emptySlot = 0; pSched->emptySlot = 0;

3
src/util/src/ttimer.c
View File

@ -552,8 +552,7 @@ void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* lab
return NULL; return NULL;
} }
strncpy(ctrl->label, label, sizeof(ctrl->label)); tstrncpy(ctrl->label, label, sizeof(ctrl->label));
ctrl->label[sizeof(ctrl->label) - 1] = 0;
tmrTrace("%s timer controller is initialized, number of timer controllers: %d.", label, numOfTmrCtrl); tmrTrace("%s timer controller is initialized, number of timer controllers: %d.", label, numOfTmrCtrl);
return ctrl; return ctrl;
} }

8
src/util/src/tutil.c
View File

@ -582,13 +582,13 @@ bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
} }
int taosCheckVersion(char *input_client_version, char *input_server_version, int comparedSegments) { int taosCheckVersion(char *input_client_version, char *input_server_version, int comparedSegments) {
char client_version[64] = {0}; char client_version[TSDB_VERSION_LEN] = {0};
char server_version[64] = {0}; char server_version[TSDB_VERSION_LEN] = {0};
int clientVersionNumber[4] = {0}; int clientVersionNumber[4] = {0};
int serverVersionNumber[4] = {0}; int serverVersionNumber[4] = {0};
strcpy(client_version, input_client_version); tstrncpy(client_version, input_client_version, sizeof(client_version));
strcpy(server_version, input_server_version); tstrncpy(server_version, input_server_version, sizeof(server_version));
if (!taosGetVersionNumber(client_version, clientVersionNumber)) { if (!taosGetVersionNumber(client_version, clientVersionNumber)) {
uError("invalid client version:%s", client_version); uError("invalid client version:%s", client_version);

7
src/util/tests/cacheTest.cpp
View File

@ -19,8 +19,7 @@ int32_t tsMaxMeterConnections = 200;
// test cache // test cache
TEST(testCase, client_cache_test) { TEST(testCase, client_cache_test) {
const int32_t REFRESH_TIME_IN_SEC = 2; const int32_t REFRESH_TIME_IN_SEC = 2;
void* tscTmr = taosTmrInit (tsMaxMgmtConnections*2, 200, 6000, "TSC"); SCacheObj* tscCacheHandle = taosCacheInit(REFRESH_TIME_IN_SEC);
SCacheObj* tscCacheHandle = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
const char* key1 = "test1"; const char* key1 = "test1";
char data1[] = "test11"; char data1[] = "test11";
@ -106,9 +105,7 @@ TEST(testCase, client_cache_test) {
TEST(testCase, cache_resize_test) { TEST(testCase, cache_resize_test) {
const int32_t REFRESH_TIME_IN_SEC = 2; const int32_t REFRESH_TIME_IN_SEC = 2;
void* tscTmr = taosTmrInit (1000*2, 200, 6000, "TSC"); auto* pCache = taosCacheInit(REFRESH_TIME_IN_SEC);
auto* pCache = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
char key[256] = {0}; char key[256] = {0};
char data[1024] = "abcdefghijk"; char data[1024] = "abcdefghijk";

2
src/vnode/inc/vnodeInt.h
View File

@ -51,7 +51,7 @@ typedef struct {
SSyncCfg syncCfg; SSyncCfg syncCfg;
SWalCfg walCfg; SWalCfg walCfg;
char *rootDir; char *rootDir;
char db[TSDB_DB_NAME_LEN + 1]; char db[TSDB_DB_NAME_LEN];
} SVnodeObj; } SVnodeObj;
int vnodeWriteToQueue(void *param, void *pHead, int type); int vnodeWriteToQueue(void *param, void *pHead, int type);

8
src/vnode/src/vnodeMain.c
View File

@ -381,13 +381,15 @@ void *vnodeGetWal(void *pVnode) {
static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) { static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
if (pVnode->status == TAOS_VN_STATUS_DELETING) return; if (pVnode->status == TAOS_VN_STATUS_DELETING) return;
if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
int64_t totalStorage, compStorage, pointsWritten = 0;
tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage);
SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
pLoad->vgId = htonl(pVnode->vgId); pLoad->vgId = htonl(pVnode->vgId);
pLoad->cfgVersion = htonl(pVnode->cfgVersion); pLoad->cfgVersion = htonl(pVnode->cfgVersion);
pLoad->totalStorage = htobe64(pLoad->totalStorage); pLoad->totalStorage = htobe64(totalStorage);
pLoad->compStorage = htobe64(pLoad->compStorage); pLoad->compStorage = htobe64(compStorage);
pLoad->pointsWritten = htobe64(pLoad->pointsWritten); pLoad->pointsWritten = htobe64(pointsWritten);
pLoad->status = pVnode->status; pLoad->status = pVnode->status;
pLoad->role = pVnode->role; pLoad->role = pVnode->role;
pLoad->replica = pVnode->syncCfg.replica; pLoad->replica = pVnode->syncCfg.replica;

4
tests/pytest/fulltest.sh
View File

@ -134,3 +134,7 @@ python3 ./test.py -f query/filter.py
python3 ./test.py -f query/filterCombo.py python3 ./test.py -f query/filterCombo.py
python3 ./test.py -f query/queryNormal.py python3 ./test.py -f query/queryNormal.py
python3 ./test.py -f query/queryError.py python3 ./test.py -f query/queryError.py
#stream
python3 ./test.py -f stream/stream1.py
python3 ./test.py -f stream/stream2.py

3
tests/pytest/random-test/random-test-multi-threading-3.py
View File

@ -205,6 +205,7 @@ class Test (Thread):
global written global written
dnodesDir = tdDnodes.getDnodesRootDir() dnodesDir = tdDnodes.getDnodesRootDir()
tdDnodes.forcestop(1)
dataDir = dnodesDir + '/dnode1/data/*' dataDir = dnodesDir + '/dnode1/data/*'
deleteCmd = 'rm -rf %s' % dataDir deleteCmd = 'rm -rf %s' % dataDir
os.system(deleteCmd) os.system(deleteCmd)
@ -261,7 +262,7 @@ class Test (Thread):
while True: while True:
self.queryEvent.wait() self.queryEvent.wait()
tdLog.notice("third thread") tdLog.notice("third thread")
randQueryOp = random.randint(1, 9) randQueryOp = random.randint(1, 2)
queryOp.get(randQueryOp, lambda: "ERROR")() queryOp.get(randQueryOp, lambda: "ERROR")()
self.queryEvent.clear() self.queryEvent.clear()
self.dbEvent.clear() self.dbEvent.clear()

2
tests/pytest/random-test/random-test-multi-threading.py
View File

@ -208,12 +208,12 @@ class Test (threading.Thread):
global written global written
dnodesDir = tdDnodes.getDnodesRootDir() dnodesDir = tdDnodes.getDnodesRootDir()
tdDnodes.forcestop(1)
dataDir = dnodesDir + '/dnode1/data/*' dataDir = dnodesDir + '/dnode1/data/*'
deleteCmd = 'rm -rf %s' % dataDir deleteCmd = 'rm -rf %s' % dataDir
os.system(deleteCmd) os.system(deleteCmd)
tdDnodes.start(1) tdDnodes.start(1)
# tdLog.sleep(10)
tdSql.prepare() tdSql.prepare()
last_tb = "" last_tb = ""
last_stb = "" last_stb = ""

3
tests/pytest/random-test/random-test.py
View File

@ -166,7 +166,8 @@ class Test:
def delete_datafiles(self): def delete_datafiles(self):
tdLog.info("delete_datafiles") tdLog.info("delete_datafiles")
dnodesDir = tdDnodes.getDnodesRootDir() dnodesDir = tdDnodes.getDnodesRootDir()
dataDir = dnodesDir + '/dnode1/*' tdDnodes.forcestop(1)
dataDir = dnodesDir + '/dnode1/data/*'
deleteCmd = 'rm -rf %s' % dataDir deleteCmd = 'rm -rf %s' % dataDir
os.system(deleteCmd) os.system(deleteCmd)

11
tests/pytest/regressiontest.sh
View File

@ -131,6 +131,11 @@ python3 ./test.py -f user/pass_len.py
#query #query
python3 ./test.py -f query/filter.py python3 ./test.py -f query/filter.py
# python3 ./test.py -f query/filterCombo.py
# python3 ./test.py -f query/queryNormal.py
# python3 ./test.py -f query/queryError.py #stream
python3 ./test.py -f stream/stream1.py
python3 ./test.py -f stream/stream2.py

8
tests/pytest/table/boundary.py
View File

@ -96,14 +96,14 @@ class TDTestCase:
tdLog.notice("table name max length is %d" % maxTableNameLen) tdLog.notice("table name max length is %d" % maxTableNameLen)
# create a super table with name exceed max length # create a super table with name exceed max length
sname = self.generateString(maxTableNameLen + 1) sname = self.generateString(maxTableNameLen)
tdLog.info("create a super table with length %d" % len(sname)) tdLog.info("create a super table with length %d" % len(sname))
tdSql.error( tdSql.error(
"create table %s (ts timestamp, value int) tags(id int)" % "create table %s (ts timestamp, value int) tags(id int)" %
sname) sname)
# create a super table with name of max length # create a super table with name of max length
sname = self.generateString(maxTableNameLen) sname = self.generateString(maxTableNameLen - 1)
tdLog.info("create a super table with length %d" % len(sname)) tdLog.info("create a super table with length %d" % len(sname))
tdSql.execute( tdSql.execute(
"create table %s (ts timestamp, value int) tags(id int)" % "create table %s (ts timestamp, value int) tags(id int)" %
@ -113,12 +113,12 @@ class TDTestCase:
tdSql.checkRows(1) tdSql.checkRows(1)
# create a child table with name exceed max length # create a child table with name exceed max length
name = self.generateString(maxTableNameLen + 1) name = self.generateString(maxTableNameLen)
tdLog.info("create a child table with length %d" % len(name)) tdLog.info("create a child table with length %d" % len(name))
tdSql.error("create table %s using %s tags(0)" % (name, sname)) tdSql.error("create table %s using %s tags(0)" % (name, sname))
# create a child table with name of max length # create a child table with name of max length
name = self.generateString(maxTableNameLen) name = self.generateString(maxTableNameLen - 1)
tdLog.info("create a child table with length %d" % len(name)) tdLog.info("create a child table with length %d" % len(name))
tdSql.execute("create table %s using %s tags(0)" % (name, sname)) tdSql.execute("create table %s using %s tags(0)" % (name, sname))
tdSql.query('show tables') tdSql.query('show tables')

4
tests/pytest/table/tablename-boundary.py
View File

@ -18,9 +18,7 @@ class TDTestCase:
tdSql.prepare() tdSql.prepare()
getTableNameLen = "grep -w '#define TSDB_TABLE_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'" getTableNameLen = "grep -w '#define TSDB_TABLE_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'"
tableNameMaxLen = int( tableNameMaxLen = int( subprocess.check_output(getTableNameLen, shell=True)) - 1
subprocess.check_output(
getTableNameLen, shell=True))
tdLog.info("table name max length is %d" % tableNameMaxLen) tdLog.info("table name max length is %d" % tableNameMaxLen)
chars = string.ascii_uppercase + string.ascii_lowercase chars = string.ascii_uppercase + string.ascii_lowercase
tb_name = ''.join(random.choices(chars, k=tableNameMaxLen)) tb_name = ''.join(random.choices(chars, k=tableNameMaxLen))

12
tests/script/fullGeneralSuite.sim
View File

@ -211,3 +211,15 @@ run general/vector/table_field.sim
run general/vector/table_mix.sim run general/vector/table_mix.sim
run general/vector/table_query.sim run general/vector/table_query.sim
run general/vector/table_time.sim run general/vector/table_time.sim
run general/stream/stream_1.sim
run general/stream/stream_2.sim
run general/stream/stream_3.sim
run general/stream/stream_restart.sim
run general/stream/table_1.sim
run general/stream/metrics_1.sim
run general/stream/table_n.sim
run general/stream/metrics_n.sim
run general/stream/table_del.sim
run general/stream/metrics_del.sim
run general/stream/table_replica1_vnoden.sim
run general/stream/metrics_replica1_vnoden.sim

2
tests/script/general/alter/cached_schema_after_alter.sim
View File

@ -53,7 +53,7 @@ endi
print ================== restart server to commit data into disk print ================== restart server to commit data into disk
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 2000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
print ================== server restart completed print ================== server restart completed
sql connect sql connect

4
tests/script/general/alter/metrics.sim
View File

@ -375,9 +375,9 @@ endi
print ======== step9 print ======== step9
print ======== step10 print ======== step10
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 3000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
sql use d2 sql use d2
sql describe tb sql describe tb

4
tests/script/general/alter/table.sim
View File

@ -326,9 +326,9 @@ endi
print ======== step9 print ======== step9
print ======== step10 print ======== step10
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 3000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
sql use d1 sql use d1
sql describe tb sql describe tb

3
tests/script/general/cache/restart_metrics.sim vendored
View File

@ -57,7 +57,8 @@ system sh/exec.sh -n dnode1 -s start
print =============== step3 print =============== step3
print ==> sleep 8 seconds to renew cache print ==> sleep 8 seconds to renew cache
sleep 8000 sql reset query cache
sleep 1000
print =============== step4 print =============== step4
sql create database $db sql create database $db

4
tests/script/general/column/commit.sim
View File

@ -89,9 +89,9 @@ endi
print =============== step4 print =============== step4
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 4000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
print =============== step5 print =============== step5

4
tests/script/general/column/metrics.sim
View File

@ -157,9 +157,9 @@ endi
print =============== step4 print =============== step4
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 1000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 2000 sleep 5000
print =============== step5 print =============== step5

4
tests/script/general/column/table.sim
View File

@ -129,9 +129,9 @@ endi
print =============== step4 print =============== step4
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 1000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 2000 sleep 5000
print ============== step5 print ============== step5

2
tests/script/general/compress/commitlog.sim
View File

@ -89,7 +89,7 @@ print =============== step4
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 5000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
print =============== step5 print =============== step5

2
tests/script/general/compress/compress.sim
View File

@ -84,7 +84,7 @@ print =============== step4
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 5000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
print =============== step5 print =============== step5

2
tests/script/general/compress/compress2.sim
View File

@ -84,7 +84,7 @@ print =============== step4
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 5000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
print =============== step5 print =============== step5

8
tests/script/general/db/alter_tables_d2.sim
View File

@ -211,10 +211,10 @@ endi
print ============================ step7 print ============================ step7
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep 1000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode2 -s start
sleep 3000 sleep 5000
sql reset query cache sql reset query cache
sleep 1000 sleep 1000
@ -334,10 +334,10 @@ endi
print ============================ step10 print ============================ step10
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep 1000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode2 -s start
sleep 3000 sleep 5000
sql reset query cache sql reset query cache
sleep 1000 sleep 1000

8
tests/script/general/db/alter_tables_v1.sim
View File

@ -143,9 +143,9 @@ endi
print ============================ step7 print ============================ step7
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 1000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
sql reset query cache sql reset query cache
sleep 1000 sleep 1000
@ -245,9 +245,9 @@ endi
print ============================ step10 print ============================ step10
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 1000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
sql reset query cache sql reset query cache
sleep 1000 sleep 1000

8
tests/script/general/db/alter_tables_v4.sim
View File

@ -204,9 +204,9 @@ endi
print ============================ step7 print ============================ step7
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 1000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
sql reset query cache sql reset query cache
sleep 1000 sleep 1000
@ -325,9 +325,9 @@ endi
print ============================ step10 print ============================ step10
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 1000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
sql reset query cache sql reset query cache
sleep 1000 sleep 1000

8
tests/script/general/db/alter_vgroups.sim
View File

@ -70,9 +70,9 @@ endi
print ============================ step3 print ============================ step3
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 2 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 2
sleep 1000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
sql create table db.t100 using db.st tags(0) sql create table db.t100 using db.st tags(0)
sql create table db.t101 using db.st tags(1) sql create table db.t101 using db.st tags(1)
@ -132,9 +132,9 @@ print ============================ step5
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 3 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 3
sleep 1000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 5000
sql create table db.t200 using db.st tags(0) sql create table db.t200 using db.st tags(0)
sql create table db.t201 using db.st tags(1) sql create table db.t201 using db.st tags(1)

8
tests/script/general/db/backup/keep.sim
View File

@ -50,9 +50,9 @@ endi
print ======== step2 stop dnode print ======== step2 stop dnode
system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep 3000 sleep 5000
system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode2 -s start
sleep 3000 sleep 5000
sql select * from tb sql select * from tb
print ===> rows $rows print ===> rows $rows
@ -112,9 +112,9 @@ endi
print ======== step5 stop dnode print ======== step5 stop dnode
system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep 3000 sleep 5000
system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode2 -s start
sleep 3000 sleep 5000
sql select * from tb sql select * from tb
print ===> rows $rows print ===> rows $rows

2
tests/script/general/db/delete.sim
View File

@ -43,7 +43,7 @@ endi
print ======= step3 print ======= step3
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 1000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
$x = 0 $x = 0

3
tests/script/general/db/dropdnodes.sim
View File

@ -70,7 +70,7 @@ endi
print ========== step3 print ========== step3
system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep 500 sleep 5000
sql drop dnode $hostname2 sql drop dnode $hostname2
sleep 2000 sleep 2000
@ -102,3 +102,4 @@ sql_error select * from db.t1
sql_error select * from db.t9 sql_error select * from db.t9
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT

4
tests/script/general/import/replica1.sim
View File

@ -93,7 +93,7 @@ endi
print ================== dnode restart print ================== dnode restart
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 3000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 5000 sleep 5000
@ -162,7 +162,7 @@ endi
print ================= step10 print ================= step10
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 3000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 5000 sleep 5000

4
tests/script/general/insert/insert_drop.sim
View File

@ -43,7 +43,7 @@ print ====== tables created
print ================== restart server to commit data into disk print ================== restart server to commit data into disk
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 3000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
print ================== server restart completed print ================== server restart completed
@ -69,7 +69,7 @@ endw
print ================== restart server to commit data into disk print ================== restart server to commit data into disk
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 4000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
print ================== server restart completed print ================== server restart completed

2
tests/script/general/parser/auto_create_tb.sim
View File

@ -208,7 +208,7 @@ endi
print ================== restart server to commit data into disk print ================== restart server to commit data into disk
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 2000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
print ================== server restart completed print ================== server restart completed
sql connect sql connect

2
tests/script/general/parser/commit.sim
View File

@ -81,7 +81,7 @@ endw
print ================== restart server to commit data into disk print ================== restart server to commit data into disk
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 3000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 3000
print ================== server restart completed print ================== server restart completed

2
tests/script/general/parser/first_last.sim
View File

@ -75,7 +75,7 @@ run general/parser/first_last_query.sim
print ================== restart server to commit data into disk print ================== restart server to commit data into disk
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 3000 sleep 5000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
print ================== server restart completed print ================== server restart completed
sql connect sql connect

Some files were not shown because too many files have changed in this diff Show More