xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '2.0' into refact/sync

This commit is contained in:
Jeff Tao 2020-03-30 18:02:20 +08:00 committed by GitHub
parent 7337f7593d 6824687dc9
commit 2f7fb6e115
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
113 changed files with 10141 additions and 6708 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/CMakeLists.txt
View File

@ -12,7 +12,6 @@ ADD_SUBDIRECTORY(client)
ADD_SUBDIRECTORY(query)
ADD_SUBDIRECTORY(kit)
ADD_SUBDIRECTORY(plugins)
ADD_SUBDIRECTORY(sdb)
ADD_SUBDIRECTORY(mnode)
ADD_SUBDIRECTORY(vnode)
ADD_SUBDIRECTORY(dnode)

10
src/client/inc/tscUtil.h
View File

@ -32,7 +32,7 @@ extern "C" {
#define UTIL_TABLE_IS_SUPERTABLE(metaInfo) \
(((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
#define UTIL_TABLE_IS_NOMRAL_TABLE(metaInfo) (!(UTIL_TABLE_IS_SUPERTABLE(metaInfo)))
#define UTIL_TABLE_CREATE_FROM_STABLE(metaInfo) \
#define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \
(((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE))
#define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0)
@ -106,7 +106,7 @@ bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo);
bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex);
bool tscQueryOnMetric(SSqlCmd* pCmd);
bool tscQueryMetricTags(SQueryInfo* pQueryInfo);
bool tscQueryTags(SQueryInfo* pQueryInfo);
bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd);
void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex,
@ -176,8 +176,8 @@ void tscIncStreamExecutionCount(void* pStream);
bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId);
// get starter position of metric query condition (query on tags) in SSqlCmd.payload
SCond* tsGetMetricQueryCondPos(STagCond* pCond, uint64_t tableIndex);
void tsSetMetricQueryCond(STagCond* pTagCond, uint64_t uid, const char* str);
SCond* tsGetSTableQueryCondPos(STagCond* pCond, uint64_t tableIndex);
void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str);
void tscTagCondCopy(STagCond* dest, const STagCond* src);
void tscTagCondRelease(STagCond* pCond);
@ -207,7 +207,7 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd);
void tscClearSubqueryInfo(SSqlCmd* pCmd);
void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* keyStr, uint64_t uid);
int tscGetMetricMeta(SSqlObj* pSql, int32_t clauseIndex);
int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex);
int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists);

14
src/client/inc/tsclient.h
View File

@ -70,8 +70,9 @@ typedef struct STableMeta {
typedef struct STableMetaInfo {
STableMeta * pTableMeta; // table meta, cached in client side and acquried by name
SSuperTableMeta *pMetricMeta; // metricmeta
// SSuperTableMeta *pMetricMeta; // metricmeta
SArray* vgroupIdList;
/*
* 1. keep the vnode index during the multi-vnode super table projection query
* 2. keep the vnode index for multi-vnode insertion
@ -294,20 +295,21 @@ typedef struct SResRec {
struct STSBuf;
typedef struct {
int32_t code;
int64_t numOfRows; // num of results in current retrieved
int64_t numOfTotal; // num of total results
int64_t numOfTotalInCurrentClause; // num of total result in current subclause
char * pRsp;
int rspType;
int rspLen;
int32_t rspType;
int32_t rspLen;
uint64_t qhandle;
int64_t uid;
int64_t useconds;
int64_t offset; // offset value from vnode during projection query of stable
int row;
int32_t row;
int16_t numOfCols;
int16_t precision;
bool completed;
int32_t code;
int32_t numOfGroups;
SResRec * pGroupRec;
char * data;

11
src/client/src/tscAsync.c
View File

@ -406,11 +406,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
if (pSql->fp == NULL) {
tscError("%p callBack is NULL!!!", pSql);
return;
}
if (pSql->fp == (void *)1) {
pSql->fp = NULL;
@ -464,7 +459,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
code = tscGetMetricMeta(pSql, 0);
code = tscGetSTableVgroupInfo(pSql, 0);
pRes->code = code;
if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
@ -494,7 +489,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
code = tscGetMetricMeta(pSql, pCmd->clauseIndex);
code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex);
pRes->code = code;
if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
@ -519,7 +514,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
tscTansformSQLFunctionForSTableQuery(pQueryInfo);
tscIncStreamExecutionCount(pSql->pStream);
} else {
tscTrace("%p get tableMeta/metricMeta successfully", pSql);
tscTrace("%p get tableMeta successfully", pSql);
}
tscDoQuery(pSql);

4
src/client/src/tscFunctionImpl.c
View File

@ -417,6 +417,10 @@ static void count_function(SQLFunctionCtx *pCtx) {
numOfElem += 1;
}
} else {
/*
* when counting on the primary time stamp column and no statistics data is provided,
* simple use the size value
*/
numOfElem = pCtx->size;
}
}

7
src/client/src/tscLocal.c
View File

@ -286,6 +286,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) {
// todo add order support
static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) {
#if 0
// the result structure has been completed in sql parse, so we
// only need to reorganize the results in the column format
SSqlCmd * pCmd = &pSql->cmd;
@ -337,6 +338,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) {
}
}
#endif
return 0;
}
@ -345,7 +347,7 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) {
SSqlRes *pRes = &pSql->res;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
#if 0
SSuperTableMeta *pMetricMeta = tscGetMetaInfo(pQueryInfo, 0)->pMetricMeta;
int32_t totalNumOfResults = 1; // count function only produce one result
int32_t rowLen = tscGetResRowLength(pQueryInfo);
@ -369,7 +371,8 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) {
}
rowIdx++;
}
#endif
return 0;
}

55
src/client/src/tscParseInsert.c
View File

@ -18,9 +18,10 @@
#define _XOPEN_SOURCE
#include "hash.h"
#include "os.h"
#include "tscSecondaryMerge.h"
#include "hash.h"
//#include "tscSecondaryMerge.h"
#include "tscUtil.h"
#include "tschemautil.h"
#include "tsclient.h"
@ -32,6 +33,8 @@
#include "tstoken.h"
#include "ttime.h"
#include "dataformat.h"
enum {
TSDB_USE_SERVER_TS = 0,
TSDB_USE_CLI_TS = 1,
@ -393,7 +396,6 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start
int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, char *error,
int16_t timePrec, int32_t *code, char *tmpTokenBuf) {
int32_t index = 0;
// bool isPrevOptr; //fang, never used
SSQLToken sToken = {0};
char * payload = pDataBlocks->pData + pDataBlocks->size;
@ -604,8 +606,8 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3
return TSDB_CODE_SUCCESS;
}
static void tsSetBlockInfo(SShellSubmitBlock *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) {
pBlocks->sid = pTableMeta->sid;
static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) {
pBlocks->tid = pTableMeta->sid;
pBlocks->uid = pTableMeta->uid;
pBlocks->sversion = pTableMeta->sversion;
pBlocks->numOfRows += numOfRows;
@ -613,10 +615,10 @@ static void tsSetBlockInfo(SShellSubmitBlock *pBlocks, const STableMeta *pTableM
// data block is disordered, sort it in ascending order
void sortRemoveDuplicates(STableDataBlocks *dataBuf) {
SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)dataBuf->pData;
SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData;
// size is less than the total size, since duplicated rows may be removed yet.
assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SShellSubmitBlock) == dataBuf->size);
assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SSubmitBlk) == dataBuf->size);
// if use server time, this block must be ordered
if (dataBuf->tsSource == TSDB_USE_SERVER_TS) {
@ -624,7 +626,7 @@ void sortRemoveDuplicates(STableDataBlocks *dataBuf) {
}
if (!dataBuf->ordered) {
char *pBlockData = pBlocks->payLoad;
char *pBlockData = pBlocks->data;
qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar);
int32_t i = 0;
@ -650,7 +652,7 @@ void sortRemoveDuplicates(STableDataBlocks *dataBuf) {
dataBuf->ordered = true;
pBlocks->numOfRows = i + 1;
dataBuf->size = sizeof(SShellSubmitBlock) + dataBuf->rowSize * pBlocks->numOfRows;
dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows;
}
}
@ -663,7 +665,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char
STableDataBlocks *dataBuf = NULL;
int32_t ret = tscGetDataBlockFromList(pTableHashList, pCmd->pDataBlocks, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE,
sizeof(SShellSubmitBlock), tinfo.rowSize, pTableMetaInfo->name,
sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name,
pTableMeta, &dataBuf);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
@ -691,11 +693,11 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char
SParamInfo *param = dataBuf->params + i;
if (param->idx == -1) {
param->idx = pCmd->numOfParams++;
param->offset -= sizeof(SShellSubmitBlock);
param->offset -= sizeof(SSubmitBlk);
}
}
SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(dataBuf->pData);
SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData);
tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
dataBuf->vgId = pTableMeta->vgId;
@ -1141,7 +1143,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
STableDataBlocks *pDataBlock = NULL;
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
int32_t ret = tscCreateDataBlock(PATH_MAX, tinfo.rowSize, sizeof(SShellSubmitBlock), pTableMetaInfo->name,
int32_t ret = tscCreateDataBlock(PATH_MAX, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name,
pTableMeta, &pDataBlock);
if (ret != TSDB_CODE_SUCCESS) {
goto _error_clean;
@ -1236,22 +1238,10 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
goto _clean;
}
// submit to more than one vnode
if (pCmd->pDataBlocks->nSize > 0) {
// merge according to vgId
if (pCmd->pDataBlocks->nSize > 0) { // merge according to vgId
if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) {
goto _error_clean;
}
STableDataBlocks *pDataBlock = pCmd->pDataBlocks->pData[0];
if ((code = tscCopyDataBlockToPayload(pSql, pDataBlock)) != TSDB_CODE_SUCCESS) {
goto _error_clean;
}
pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
// set the next sent data vnode index in data block arraylist
pTableMetaInfo->vnodeIndex = 1;
} else {
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
}
@ -1303,12 +1293,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion) {
int32_t ret = TSDB_CODE_SUCCESS;
if (NULL == pSql->asyncTblPos) {
tscCleanSqlCmd(&pSql->cmd);
} else {
tscTrace("continue parse sql: %s", pSql->asyncTblPos);
}
tscTrace("continue parse sql: %s", pSql->asyncTblPos);
if (tscIsInsertOrImportData(pSql->sqlstr)) {
/*
@ -1353,7 +1338,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock
assert(pCmd->numOfClause == 1);
STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta;
SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(pTableDataBlocks->pData);
SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData);
tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) {
@ -1394,7 +1379,7 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) {
pCmd->pDataBlocks = tscCreateBlockArrayList();
STableDataBlocks *pTableDataBlock = NULL;
int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, rowSize, sizeof(SShellSubmitBlock),
int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, rowSize, sizeof(SSubmitBlk),
pTableMetaInfo->name, pTableMeta, &pTableDataBlock);
if (ret != TSDB_CODE_SUCCESS) {
return -1;
@ -1435,7 +1420,7 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) {
}
pTableDataBlock = pCmd->pDataBlocks->pData[0];
pTableDataBlock->size = sizeof(SShellSubmitBlock);
pTableDataBlock->size = sizeof(SSubmitBlk);
pTableDataBlock->rowSize = tinfo.rowSize;
numOfRows += pSql->res.numOfRows;

16
src/client/src/tscPrepare.c
View File

@ -325,12 +325,12 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) {
STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i];
uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock);
uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk);
uint32_t dataSize = totalDataSize / alloced;
assert(dataSize * alloced == totalDataSize);
if (alloced == binded) {
totalDataSize += dataSize + sizeof(SShellSubmitBlock);
totalDataSize += dataSize + sizeof(SSubmitBlk);
if (totalDataSize > pBlock->nAllocSize) {
const double factor = 1.5;
void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor));
@ -342,7 +342,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
}
}
char* data = pBlock->pData + sizeof(SShellSubmitBlock) + dataSize * binded;
char* data = pBlock->pData + sizeof(SSubmitBlk) + dataSize * binded;
for (uint32_t j = 0; j < pBlock->numOfParams; ++j) {
SParamInfo* param = pBlock->params + j;
int code = doBindParam(data, param, bind + param->idx);
@ -365,10 +365,10 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) {
STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i];
uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock);
uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk);
pBlock->size += totalDataSize / alloced;
SShellSubmitBlock* pSubmit = (SShellSubmitBlock*)pBlock->pData;
SSubmitBlk* pSubmit = (SSubmitBlk*)pBlock->pData;
pSubmit->numOfRows += pSubmit->numOfRows / alloced;
}
@ -398,10 +398,10 @@ static int insertStmtReset(STscStmt* pStmt) {
for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) {
STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i];
uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock);
pBlock->size = sizeof(SShellSubmitBlock) + totalDataSize / alloced;
uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk);
pBlock->size = sizeof(SSubmitBlk) + totalDataSize / alloced;
SShellSubmitBlock* pSubmit = (SShellSubmitBlock*)pBlock->pData;
SSubmitBlk* pSubmit = (SSubmitBlk*)pBlock->pData;
pSubmit->numOfRows = pSubmit->numOfRows / alloced;
}
}

75
src/client/src/tscSQLParser.c
View File

@ -209,9 +209,14 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
int32_t code = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo);
assert(pQueryInfo->numOfTables == 0);
// assert(pQueryInfo->numOfTables == 0);
STableMetaInfo* pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo);
STableMetaInfo* pTableMetaInfo = NULL;
if (pQueryInfo->numOfTables == 0) {
pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo);
} else {
pTableMetaInfo = &pQueryInfo->pTableMetaInfo[0];
}
pCmd->command = pInfo->type;
@ -639,7 +644,7 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
* check invalid SQL:
* select tbname, tags_fields from super_table_name interval(1s)
*/
if (tscQueryMetricTags(pQueryInfo) && pQueryInfo->intervalTime > 0) {
if (tscQueryTags(pQueryInfo) && pQueryInfo->intervalTime > 0) {
return invalidSqlErrMsg(pQueryInfo->msg, msg1);
}
@ -746,7 +751,7 @@ int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlO
tscClearMeterMetaInfo(pTableMetaInfo, false);
}
} else {
assert(pTableMetaInfo->pTableMeta == NULL && pTableMetaInfo->pMetricMeta == NULL);
assert(pTableMetaInfo->pTableMeta == NULL);
}
tfree(oldName);
@ -1252,7 +1257,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
if (tscQueryMetricTags(pQueryInfo)) { // local handle the metric tag query
if (tscQueryTags(pQueryInfo)) { // local handle the metric tag query
pCmd->count = numOfCols; // the number of meter schema, tricky.
pQueryInfo->command = TSDB_SQL_RETRIEVE_TAGS;
}
@ -1293,7 +1298,7 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c
int16_t functionId = (int16_t)((colIdx >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ);
if (functionId == TSDB_FUNC_TAGPRJ) {
addRequiredTagColumn(pQueryInfo, colIdx - numOfCols, tableIndex);
// addRequiredTagColumn(pQueryInfo, colIdx - numOfCols, tableIndex);
pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY;
} else {
pQueryInfo->type = TSDB_QUERY_TYPE_PROJECTION_QUERY;
@ -1396,8 +1401,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum
SColumnList ids = {0};
ids.ids[0] = *pIndex;
// tag columns do not add to source list
ids.num = (j >= tscGetNumOfColumns(pTableMeta)) ? 0 : 1;
ids.num = 1;
insertResultField(pQueryInfo, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name, pExpr);
}
@ -1489,11 +1493,9 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema,
SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, resColIdx, functionID, pColIndex, type, bytes, bytes);
strncpy(pExpr->aliasName, columnName, tListLen(pExpr->aliasName));
// for point interpolation/last_row query, we need the timestamp column to be loaded
// for all querie, the timestamp column meeds to be loaded
SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) {
tscColumnBaseInfoInsert(pQueryInfo, &index);
}
tscColumnBaseInfoInsert(pQueryInfo, &index);
SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex);
insertResultField(pQueryInfo, resColIdx, &ids, bytes, type, columnName, pExpr);
@ -1581,7 +1583,10 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt
tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i]));
}
}
SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
tscColumnBaseInfoInsert(pQueryInfo, &tsCol);
return TSDB_CODE_SUCCESS;
}
case TK_SUM:
@ -1689,7 +1694,10 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt
tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i]));
}
}
SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
tscColumnBaseInfoInsert(pQueryInfo, &tsCol);
return TSDB_CODE_SUCCESS;
}
case TK_FIRST:
@ -1708,7 +1716,6 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt
}
/* in first/last function, multiple columns can be add to resultset */
for (int32_t i = 0; i < pItem->pNode->pParam->nExpr; ++i) {
tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[i]);
if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) {
@ -1753,7 +1760,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt
}
}
}
return TSDB_CODE_SUCCESS;
} else { // select * from xxx
int32_t numOfFields = 0;
@ -1773,6 +1780,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt
numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
}
return TSDB_CODE_SUCCESS;
}
}
@ -1891,6 +1899,8 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt
default:
return TSDB_CODE_INVALID_SQL;
}
}
// todo refactor
@ -2779,7 +2789,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString) {
return TSDB_CODE_SUCCESS;
}
static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) {
static int32_t tablenameListToString(tSQLExpr* pExpr, SStringBuilder* sb) {
tSQLExprList* pList = pExpr->pParam;
if (pList->nExpr <= 0) {
return TSDB_CODE_INVALID_SQL;
@ -2805,7 +2815,7 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuild
return TSDB_CODE_SUCCESS;
}
static int32_t tablenameCondToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) {
static int32_t tablenameCondToString(tSQLExpr* pExpr, SStringBuilder* sb) {
taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN);
taosStringBuilderAppendString(sb, pExpr->val.pz);
@ -3501,7 +3511,7 @@ int tableNameCompar(const void* lhs, const void* rhs) {
return ret > 0 ? 1 : -1;
}
static int32_t setTableCondForMetricQuery(SQueryInfo* pQueryInfo, const char* account, tSQLExpr* pExpr,
static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* account, tSQLExpr* pExpr,
int16_t tableCondIndex, SStringBuilder* sb) {
const char* msg = "table name too long";
@ -3730,7 +3740,7 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr,
return ret;
}
tsSetMetricQueryCond(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid, c);
tsSetSTableQueryCond(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid, c);
doCompactQueryExpr(pExpr);
tSQLExprDestroy(p1);
@ -3746,8 +3756,8 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql
return TSDB_CODE_SUCCESS;
}
const char* msg = "invalid filter expression";
const char* msg1 = "invalid expression";
const char* msg2 = "invalid filter expression";
int32_t ret = TSDB_CODE_SUCCESS;
@ -3805,11 +3815,11 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql
// 7. query condition for table name
pQueryInfo->tagCond.relType = (condExpr.relType == TK_AND) ? TSDB_RELATION_AND : TSDB_RELATION_OR;
ret = setTableCondForMetricQuery(pQueryInfo, getAccountId(pSql), condExpr.pTableCond, condExpr.tableCondIndex, &sb);
ret = setTableCondForSTableQuery(pQueryInfo, getAccountId(pSql), condExpr.pTableCond, condExpr.tableCondIndex, &sb);
taosStringBuilderDestroy(&sb);
if (!validateFilterExpr(pQueryInfo)) {
return invalidSqlErrMsg(pQueryInfo->msg, msg);
return invalidSqlErrMsg(pQueryInfo->msg, msg2);
}
doAddJoinTagsColumnsIntoTagList(pQueryInfo, &condExpr);
@ -4234,7 +4244,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
} else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo))) {
return invalidSqlErrMsg(pQueryInfo->msg, msg4);
} else if ((pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) &&
UTIL_TABLE_CREATE_FROM_STABLE(pTableMetaInfo)) {
UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) {
return invalidSqlErrMsg(pQueryInfo->msg, msg6);
}
@ -4660,14 +4670,13 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL*
* And then launching multiple async-queries against all qualified virtual nodes, during the first-stage
* query operation.
*/
int32_t code = tscGetMetricMeta(pSql, clauseIndex);
int32_t code = tscGetSTableVgroupInfo(pSql, clauseIndex);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
// No tables included. No results generated. Query results are empty.
SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta;
if (pTableMetaInfo->pTableMeta == NULL || pMetricMeta == NULL || pMetricMeta->numOfTables == 0) {
if (pTableMetaInfo->pTableMeta == NULL) {
tscTrace("%p no table in metricmeta, no output result", pSql);
pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
}
@ -5435,8 +5444,9 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
// two table: the first one is for current table, and the secondary is for the super table.
tscAddEmptyMetaInfo(pQueryInfo);
assert(pQueryInfo->numOfTables == 2);
if (pQueryInfo->numOfTables < 2) {
tscAddEmptyMetaInfo(pQueryInfo);
}
const int32_t TABLE_INDEX = 0;
const int32_t STABLE_INDEX = 1;
@ -5681,6 +5691,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
}
assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr);
if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
int32_t code = tscGetSTableVgroupInfo(pSql, index);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
// parse the group by clause in the first place
if (parseGroupbyClause(pQueryInfo, pQuerySql->pGroupby, pCmd) != TSDB_CODE_SUCCESS) {

27
src/client/src/tscSchemaUtil.c
View File

@ -32,7 +32,7 @@ int32_t tscGetNumOfTags(const STableMeta* pTableMeta) {
}
if (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE) {
assert(tinfo.numOfTags > 0);
assert(tinfo.numOfTags >= 0);
return tinfo.numOfTags;
}
@ -51,14 +51,14 @@ int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) {
SSchema *tscGetTableSchema(const STableMeta *pTableMeta) {
assert(pTableMeta != NULL);
if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
STableMeta* pSTableMeta = pTableMeta->pSTable;
assert (pSTableMeta != NULL);
return pSTableMeta->schema;
}
// if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
// STableMeta* pSTableMeta = pTableMeta->pSTable;
// assert (pSTableMeta != NULL);
//
// return pSTableMeta->schema;
// }
return pTableMeta->schema;
return (SSchema*) pTableMeta->schema;
}
SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) {
@ -72,12 +72,14 @@ SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) {
STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) {
assert(pTableMeta != NULL);
#if 0
if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
assert (pTableMeta->pSTable != NULL);
return pTableMeta->pSTable->tableInfo;
}
#endif
return pTableMeta->tableInfo;
}
@ -130,12 +132,13 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t startCol)
assert(pTableMeta != NULL);
SSchema* pSchema = pTableMeta->schema;
#if 0
if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
assert (pTableMeta->pSTable != NULL);
pSchema = pTableMeta->pSTable->schema;
}
#endif
return &pSchema[startCol];
}

5
src/client/src/tscSecondaryMerge.c
View File

@ -605,7 +605,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
(*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pTableMetaInfo->pMetricMeta->numOfVnodes);
(*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * 1);
if (*pMemBuffer == NULL) {
tscError("%p failed to allocate memory", pSql);
pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
@ -636,7 +636,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity);
for (int32_t i = 0; i < pTableMetaInfo->pMetricMeta->numOfVnodes; ++i) {
size_t numOfSubs = taosArrayGetSize(pTableMetaInfo->vgroupIdList);
for (int32_t i = 0; i < numOfSubs; ++i) {
(*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel);
(*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL;
}

472
src/client/src/tscServer.c
View File

@ -35,7 +35,6 @@ SRpcIpSet tscDnodeIpSet;
int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0};
int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql);
void (*tscUpdateVnodeMsg[TSDB_SQL_MAX])(SSqlObj *pSql, char *buf);
void tscProcessActivityTimer(void *handle, void *tmrId);
int tscKeepConn[TSDB_SQL_MAX] = {0};
TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid);
@ -188,7 +187,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
.pCont = pMsg,
.contLen = pSql->cmd.payloadLen,
.handle = pSql,
.code = 0
.code = 0
};
rpcSendRequest(pVnodeConn, pSql->ipList, &rpcMsg);
} else {
@ -209,7 +208,6 @@ int tscSendMsgToServer(SSqlObj *pSql) {
}
void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
tscTrace("response:%s is received, len:%d error:%s", taosMsg[rpcMsg->msgType], rpcMsg->contLen, tstrerror(rpcMsg->code));
SSqlObj *pSql = (SSqlObj *)rpcMsg->handle;
if (pSql == NULL || pSql->signature != pSql) {
tscError("%p sql is already released, signature:%p", pSql, pSql->signature);
@ -256,7 +254,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
rpcFreeCont(rpcMsg->pCont);
return;
} else {
tscTrace("%p it shall renew meter meta, code:%d", pSql, rpcMsg->code);
tscTrace("%p it shall renew meter meta, code:%d", pSql, tstrerror(rpcMsg->code));
pSql->maxRetry = TSDB_VNODES_SUPPORT * 2;
pSql->res.code = rpcMsg->code; // keep the previous error code
@ -273,19 +271,19 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
}
pSql->retry = 0;
pRes->rspLen = 0;
if (pRes->code != TSDB_CODE_QUERY_CANCELLED) {
pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_NETWORK_UNAVAIL;
} else {
tscTrace("%p query is cancelled, code:%d", pSql, pRes->code);
tscTrace("%p query is cancelled, code:%d", pSql, tstrerror(pRes->code));
}
if (pRes->code != TSDB_CODE_QUERY_CANCELLED) {
assert(rpcMsg->msgType == pCmd->msgType + 1);
pRes->code = (int32_t)rpcMsg->code;
pRes->code = rpcMsg->code;
pRes->rspType = rpcMsg->msgType;
pRes->rspLen = rpcMsg->contLen;
pRes->rspLen = rpcMsg->contLen;
char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen);
if (tmp == NULL) {
@ -316,21 +314,20 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
pRes->numOfRows += pMsg->affectedRows;
tscTrace("%p cmd:%d code:%d, inserted rows:%d, rsp len:%d", pSql, pCmd->command, pRes->code,
*(int32_t *)pRes->pRsp, pRes->rspLen);
pMsg->affectedRows, pRes->rspLen);
} else {
tscTrace("%p cmd:%d code:%d rsp len:%d", pSql, pCmd->command, pRes->code, pRes->rspLen);
tscTrace("%p cmd:%d code:%s rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pRes->rspLen);
}
}
if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command])
rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql);
if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) {
int command = pCmd->command;
void *taosres = tscKeepConn[command] ? pSql : NULL;
void *taosres = tscKeepConn[pCmd->command] ? pSql : NULL;
rpcMsg->code = pRes->code ? -pRes->code : pRes->numOfRows;
tscTrace("%p Async SQL result:%d res:%p", pSql, rpcMsg->code, taosres);
tscTrace("%p Async SQL result:%s res:%p", pSql, tstrerror(pRes->code), taosres);
/*
* Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj
@ -344,14 +341,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
(*pSql->fp)(pSql->param, taosres, rpcMsg->code);
if (shouldFree) {
// If it is failed, all objects allocated during execution taos_connect_a should be released
if (command == TSDB_SQL_CONNECT) {
taos_close(pObj);
tscTrace("%p Async sql close failed connection", pSql);
} else {
tscFreeSqlObj(pSql);
tscTrace("%p Async sql is automatically freed", pSql);
}
tscFreeSqlObj(pSql);
tscTrace("%p Async sql is automatically freed", pSql);
}
}
@ -369,7 +360,7 @@ int doProcessSql(SSqlObj *pSql) {
pCmd->command == TSDB_SQL_CONNECT ||
pCmd->command == TSDB_SQL_HB ||
pCmd->command == TSDB_SQL_META ||
pCmd->command == TSDB_SQL_METRIC) {
pCmd->command == TSDB_SQL_STABLEVGROUP) {
tscBuildMsg[pCmd->command](pSql, NULL);
}
@ -389,7 +380,7 @@ int tscProcessSql(SSqlObj *pSql) {
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = NULL;
uint16_t type = 0;
uint16_t type = 0;
if (pQueryInfo != NULL) {
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@ -508,76 +499,44 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pRetrieveMsg->free = htons(pQueryInfo->type);
pMsg += sizeof(pQueryInfo->type);
pSql->cmd.payloadLen = pMsg - pStart;
pRetrieveMsg->header.vgId = htonl(1);
pMsg += sizeof(SRetrieveTableMsg);
pRetrieveMsg->header.contLen = htonl(pSql->cmd.payloadLen);
pSql->cmd.msgType = TSDB_MSG_TYPE_RETRIEVE;
return TSDB_CODE_SUCCESS;
}
void tscUpdateVnodeInSubmitMsg(SSqlObj *pSql, char *buf) {
//SShellSubmitMsg *pShellMsg;
//char * pMsg;
//STableMetaInfo * pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
//STableMeta *pTableMeta = pTableMetaInfo->pTableMeta;
//pMsg = buf + tsRpcHeadSize;
//TODO set iplist
//pShellMsg = (SShellSubmitMsg *)pMsg;
//pShellMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode);
//tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pSql->index].ip),
// htons(pShellMsg->vnode));
}
int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SShellSubmitMsg *pShellMsg;
char * pMsg, *pStart;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta;
pStart = pSql->cmd.payload + tsRpcHeadSize;
pMsg = pStart;
pShellMsg = (SShellSubmitMsg *)pMsg;
char* pMsg = pSql->cmd.payload + tsRpcHeadSize;
pShellMsg->desc.numOfVnodes = htonl(1);
// NOTE: shell message size should not include SMsgDesc
int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc);
SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg;
pMsgDesc->numOfVnodes = htonl(1); //todo set the right number of vnodes
pMsg += sizeof(SMsgDesc);
SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg;
pShellMsg->import = htons(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT) ? 0 : 1);
pShellMsg->header.vgId = htonl(pTableMeta->vgId);
pShellMsg->header.contLen = htonl(pSql->cmd.payloadLen);
pShellMsg->header.contLen = htonl(size);
pShellMsg->length = pShellMsg->header.contLen;
pShellMsg->numOfTables = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted
pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted
// pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here
// pSql->cmd.payloadLen is set during copying data into paylaod
pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
// tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pTableMeta->index].ip),
// htons(pShellMsg->vnode));
// pSql->cmd.payloadLen = sizeof(SShellSubmitMsg);
tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htons(pMsgDesc->numOfVnodes));
return TSDB_CODE_SUCCESS;
}
void tscUpdateVnodeInQueryMsg(SSqlObj *pSql, char *buf) {
//TODO
// SSqlCmd * pCmd = &pSql->cmd;
// STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
//
// char * pStart = buf + tsRpcHeadSize;
// SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart;
//
// if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // pColumnModel == NULL, query on meter
// STableMeta *pTableMeta = pTableMetaInfo->pTableMeta;
// pQueryMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode);
// } else { // query on metric
// SSuperTableMeta * pMetricMeta = pTableMetaInfo->pMetricMeta;
// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex);
// pQueryMsg->vnode = htons(pVnodeSidList->vpeerDesc[pSql->index].vnode);
// }
}
/*
* for meter query, simply return the size <= 1k
* for metric query, estimate size according to meter tags
@ -595,7 +554,10 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize;
}
int32_t size = 4096;
#if 0
SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta;
SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex);
@ -606,45 +568,23 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
if (pQueryInfo->tsBuf != NULL) {
size += pQueryInfo->tsBuf->fileSize;
}
#endif
return size;
}
static char *doSerializeTableInfo(SSqlObj *pSql, int32_t numOfTables, int32_t vgId, char *pMsg) {
static char *doSerializeTableInfo(SSqlObj *pSql, int32_t vgId, char *pMsg) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta;
tscTrace("%p vgId:%d, query on %d tables", pSql, vgId, numOfTables);
if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
#ifdef _DEBUG_VIEW
tscTrace("%p sid:%d, uid:%" PRIu64, pSql, pTableMetaInfo->pTableMeta->sid, pTableMetaInfo->pTableMeta->uid);
#endif
STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
pTableIdInfo->sid = htonl(pTableMeta->sid);
pTableIdInfo->uid = htobe64(pTableMeta->uid);
pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid));
pMsg += sizeof(STableIdInfo);
} else {
SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex);
for (int32_t i = 0; i < numOfTables; ++i) {
STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
STableIdInfo *pQueryMeterInfo = tscGetMeterSidInfo(pVnodeSidList, i);
pTableIdInfo->sid = htonl(pQueryMeterInfo->sid);
pTableIdInfo->uid = htobe64(pQueryMeterInfo->uid);
pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pQueryMeterInfo->uid));
pMsg += sizeof(STableIdInfo);
#ifdef _DEBUG_VIEW
tscTrace("%p sid:%d, uid:%" PRId64, pSql, pQueryMeterInfo->sid, pQueryMeterInfo->uid);
#endif
}
}
tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, vgId, pTableMetaInfo->name, pTableMeta->uid);
STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
pTableIdInfo->sid = htonl(pTableMeta->sid);
pTableIdInfo->uid = htobe64(pTableMeta->uid);
pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid));
pMsg += sizeof(STableIdInfo);
return pMsg;
}
@ -657,14 +597,27 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
tscError("%p failed to malloc for query msg", pSql);
return -1;
}
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
char *pStart = pCmd->payload + tsRpcHeadSize;
STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta;
if (pQueryInfo->colList.numOfCols <= 0) {
tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta));
return -1;
}
if (pQueryInfo->intervalTime < 0) {
tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime);
return -1;
}
if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) {
tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols);
return -1;
}
char *pStart = pCmd->payload + tsRpcHeadSize;
SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart;
@ -673,18 +626,17 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
numOfTables = 1;
pQueryMsg->uid = pTableMeta->uid;
pQueryMsg->numOfTagsCols = 0;
pQueryMsg->vgId = htonl(pTableMeta->vgId);
pQueryMsg->head.vgId = htonl(pTableMeta->vgId);
tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name);
} else { // query on super table
if (pTableMetaInfo->vnodeIndex < 0) {
tscError("%p error vnodeIdx:%d", pSql, pTableMetaInfo->vnodeIndex);
return -1;
}
uint32_t vnodeId = 1;
#if 0
SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex);
uint32_t vnodeId = pVnodeSidList->vpeerDesc[pVnodeSidList->index].vnode;
@ -693,14 +645,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
tscError("%p vid:%d,error numOfTables in query message:%d", pSql, vnodeId, numOfTables);
return -1; // error
}
#endif
tscTrace("%p query on vid:%d, number of tables:%d", pSql, vnodeId, numOfTables);
pQueryMsg->vgId = htons(vnodeId);
pQueryMsg->head.vgId = htonl(vnodeId);
numOfTables = 1;
}
pQueryMsg->numOfTables = htonl(numOfTables);
pQueryMsg->numOfTagsCols = htons(pTableMetaInfo->numOfTags);
if (pQueryInfo->order.order == TSQL_SO_ASC) {
pQueryMsg->window.skey = htobe64(pQueryInfo->stime);
pQueryMsg->window.ekey = htobe64(pQueryInfo->etime);
@ -709,54 +660,29 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->window.ekey = htobe64(pQueryInfo->stime);
}
pQueryMsg->order = htons(pQueryInfo->order.order);
pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId);
pQueryMsg->interpoType = htons(pQueryInfo->interpoType);
pQueryMsg->limit = htobe64(pQueryInfo->limit.limit);
pQueryMsg->offset = htobe64(pQueryInfo->limit.offset);
pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols);
if (pQueryInfo->colList.numOfCols <= 0) {
tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta));
return -1;
}
pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime);
pQueryMsg->numOfTables = htonl(numOfTables);
pQueryMsg->order = htons(pQueryInfo->order.order);
pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId);
pQueryMsg->interpoType = htons(pQueryInfo->interpoType);
pQueryMsg->limit = htobe64(pQueryInfo->limit.limit);
pQueryMsg->offset = htobe64(pQueryInfo->limit.offset);
pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols);
pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime);
pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime);
pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit;
pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime);
if (pQueryInfo->intervalTime < 0) {
tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime);
return -1;
}
if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) {
tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols);
return -1;
}
pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols);
if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // query on meter
pQueryMsg->tagLength = 0;
} else { // query on super table
pQueryMsg->tagLength = htons(pMetricMeta->tagLen);
}
pQueryMsg->queryType = htons(pQueryInfo->type);
pQueryMsg->numOfOutputCols = htons(pQueryInfo->exprsInfo.numOfExprs);
if (pQueryInfo->fieldsInfo.numOfOutputCols < 0) {
tscError("%p illegal value of number of output columns in query msg: %d", pSql,
pQueryInfo->fieldsInfo.numOfOutputCols);
int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutputCols;
if (numOfOutput < 0) {
tscError("%p illegal value of number of output columns in query msg: %d", pSql, numOfOutput);
return -1;
}
// set column list ids
char * pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo);
char *pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo);
SSchema *pSchema = tscGetTableSchema(pTableMeta);
for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) {
@ -774,7 +700,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->colList[i].colId = htons(pColSchema->colId);
pQueryMsg->colList[i].bytes = htons(pColSchema->bytes);
pQueryMsg->colList[i].type = htons(pColSchema->type);
pQueryMsg->colList[i].type = htons(pColSchema->type);
pQueryMsg->colList[i].numOfFilters = htons(pCol->numOfFilters);
// append the filter information after the basic column information
@ -808,7 +734,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
bool hasArithmeticFunction = false;
SSqlFuncExprMsg *pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg;
for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) {
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
@ -822,11 +747,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return -1;
}
pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId);
pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId);
pSqlFuncExpr->colInfo.colIdx = htons(pExpr->colInfo.colIdx);
pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag);
pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag);
pSqlFuncExpr->functionId = htons(pExpr->functionId);
pSqlFuncExpr->functionId = htons(pExpr->functionId);
pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams);
pMsg += sizeof(SSqlFuncExprMsg);
@ -864,25 +789,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->colNameLen = htonl(len);
// serialize the table info (sid, uid, tags)
pMsg = doSerializeTableInfo(pSql, numOfTables, htons(pQueryMsg->vgId), pMsg);
// only include the required tag column schema. If a tag is not required, it won't be sent to vnode
if (pTableMetaInfo->numOfTags > 0) {
// always transfer tag schema to vnode if exists
SSchema *pTagSchema = tscGetTableTagSchema(pTableMeta);
for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) {
if (pTableMetaInfo->tagColumnIndex[j] == TSDB_TBNAME_COLUMN_INDEX) {
SSchema tbSchema = {
.bytes = TSDB_TABLE_NAME_LEN, .colId = TSDB_TBNAME_COLUMN_INDEX, .type = TSDB_DATA_TYPE_BINARY};
memcpy(pMsg, &tbSchema, sizeof(SSchema));
} else {
memcpy(pMsg, &pTagSchema[pTableMetaInfo->tagColumnIndex[j]], sizeof(SSchema));
}
pMsg += sizeof(SSchema);
}
}
pMsg = doSerializeTableInfo(pSql, htons(pQueryMsg->head.vgId), pMsg);
SSqlGroupbyExpr *pGroupbyExpr = &pQueryInfo->groupbyExpr;
if (pGroupbyExpr->numOfGroupCols != 0) {
@ -940,14 +847,42 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder);
}
// serialize tag column query condition
if (pQueryInfo->tagCond.numOfTagCond > 0) {
STagCond* pTagCond = &pQueryInfo->tagCond;
SCond *pCond = tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid);
if (pCond != NULL && pCond->cond != NULL) {
size_t condLen = strlen(pCond->cond) + 1;
bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE);
if (!ret) {
tscError("%p mbs to ucs4 failed:%d", pSql, tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid));
return 0;
}
pQueryMsg->tagCondLen = htons(condLen);
pMsg += condLen * TSDB_NCHAR_SIZE;
}
}
// tbname in/like query expression should be sent to mgmt node
STagCond* pTagCond = &pQueryInfo->tagCond;
if (pTagCond->tbnameCond.cond != NULL) {
size_t s = strlen(pTagCond->tbnameCond.cond);
memcpy(pMsg, pTagCond->tbnameCond.cond, s);
pQueryMsg->nameCondLen = htons(s);
pMsg += s;
}
msgLen = pMsg - pStart;
tscTrace("%p msg built success,len:%d bytes", pSql, msgLen);
pCmd->payloadLen = msgLen;
pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY;
pQueryMsg->contLen = htonl(msgLen);
pQueryMsg->head.contLen = htonl(msgLen);
assert(msgLen + minMsgSize() <= size);
return TSDB_CODE_SUCCESS;
@ -1478,11 +1413,15 @@ int tscProcessTagRetrieveRsp(SSqlObj *pSql) {
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
int32_t numOfRes = 0;
#if 0
if (tscSqlExprGet(pQueryInfo, 0)->functionId == TSDB_FUNC_TAGPRJ) {
numOfRes = pTableMetaInfo->pMetricMeta->numOfTables;
} else {
numOfRes = 1; // for count function, there is only one output.
}
#endif
return tscLocalResultCommonBuilder(pSql, numOfRes);
}
@ -1568,8 +1507,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pMsg += sizeof(STagData);
}
msgLen = pMsg - (char*)pInfoMsg;
pCmd->payloadLen = msgLen;
pCmd->payloadLen = pMsg - (char*)pInfoMsg;;
pCmd->msgType = TSDB_MSG_TYPE_CM_TABLE_META;
tfree(tmpData);
@ -1643,7 +1581,9 @@ static int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) {
return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE);
}
int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
#if 0
SSuperTableMetaMsg *pMetaMsg;
char * pMsg, *pStart;
int msgLen = 0;
@ -1706,13 +1646,13 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
// convert to unicode before sending to mnode for metric query
int32_t condLen = 0;
if (pTagCond->numOfTagCond > 0) {
SCond *pCond = tsGetMetricQueryCondPos(pTagCond, uid);
SCond *pCond = tsGetSTableQueryCondPos(pTagCond, uid);
if (pCond != NULL && pCond->cond != NULL) {
condLen = strlen(pCond->cond) + 1;
bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE);
if (!ret) {
tscError("%p mbs to ucs4 failed:%s", pSql, tsGetMetricQueryCondPos(pTagCond, uid));
tscError("%p mbs to ucs4 failed:%s", pSql, tsGetSTableQueryCondPos(pTagCond, uid));
return 0;
}
}
@ -1782,9 +1722,20 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
msgLen = pMsg - pStart;
pCmd->payloadLen = msgLen;
pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_META;
pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP;
assert(msgLen + minMsgSize() <= size);
#endif
SSqlCmd *pCmd = &pSql->cmd;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pCmd->payload;
strncpy(pStableVgroupMsg->tableId, pTableMetaInfo->name, tListLen(pStableVgroupMsg->tableId));
pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP;
pCmd->payloadLen = sizeof(SCMSTableVgroupMsg);
return TSDB_CODE_SUCCESS;
}
@ -1853,14 +1804,13 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
pMetaMsg->vgId = htonl(pMetaMsg->vgId);
pMetaMsg->uid = htobe64(pMetaMsg->uid);
pMetaMsg->contLen = htons(pMetaMsg->contLen);
pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns);
if (pMetaMsg->sid < 0 || pMetaMsg->vgId < 0) {
tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgId, pMetaMsg->sid);
return TSDB_CODE_INVALID_VALUE;
}
pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns);
if (pMetaMsg->numOfTags > TSDB_MAX_TAGS || pMetaMsg->numOfTags < 0) {
tscError("invalid numOfTags:%d", pMetaMsg->numOfTags);
return TSDB_CODE_INVALID_VALUE;
@ -1884,23 +1834,20 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
pSchema++;
}
// rsp += numOfTotalCols * sizeof(SSchema);
//
// int32_t tagLen = 0;
// SSchema *pTagsSchema = tscGetTableTagSchema(pMetaMsg);
//
// if (pMetaMsg->tableType == TSDB_CHILD_TABLE) {
// for (int32_t i = 0; i < pMetaMsg->numOfTags; ++i) {
// tagLen += pTagsSchema[i].bytes;
// }
// }
//
// rsp += tagLen;
// int32_t size = (int32_t)(rsp - (char *)pMetaMsg);
size_t size = 0;
STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size);
#if 0
// if current table is created according to super table, get the table meta of super table
if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
char id[TSDB_TABLE_ID_LEN + 1] = {0};
strncpy(id, pMetaMsg->stableId, TSDB_TABLE_ID_LEN);
// NOTE: if the table meta of super table is not cached at client side yet, the pSTable is NULL
pTableMeta->pSTable = taosCacheAcquireByName(tscCacheHandle, id);
}
#endif
// todo add one more function: taosAddDataIfNotExists();
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
assert(pTableMetaInfo->pTableMeta == NULL);
@ -1914,6 +1861,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
}
free(pTableMeta);
return TSDB_CODE_SUCCESS;
}
@ -2027,22 +1975,11 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
return TSDB_CODE_SUCCESS;
}
int tscProcessMetricMetaRsp(SSqlObj *pSql) {
SSuperTableMeta *pMeta;
uint8_t ieType;
int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
#if 0
void ** metricMetaList = NULL;
int32_t * sizes = NULL;
char *rsp = pSql->res.pRsp;
ieType = *rsp;
if (ieType != TSDB_IE_TYPE_META) {
tscError("invalid ie type:%d", ieType);
return TSDB_CODE_INVALID_IE;
}
rsp++;
int32_t num = htons(*(int16_t *)rsp);
rsp += sizeof(int16_t);
@ -2126,7 +2063,6 @@ int tscProcessMetricMetaRsp(SSqlObj *pSql) {
// release the used metricmeta
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false);
pTableMetaInfo->pMetricMeta = (SSuperTableMeta *)taosCachePut(tscCacheHandle, name, (char *)metricMetaList[i],
sizes[i], tsMetricMetaKeepTimer);
tfree(metricMetaList[i]);
@ -2146,7 +2082,23 @@ _error_clean:
free(sizes);
free(metricMetaList);
#endif
SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pSql->res.pRsp;
pStableVgroup->numOfDnodes = htonl(pStableVgroup->numOfDnodes);
SSqlObj* pparent = pSql->param;
assert(pparent != NULL);
SSqlCmd* pCmd = &pparent->cmd;
STableMetaInfo* pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
pInfo->vgroupIdList = taosArrayInit(pStableVgroup->numOfDnodes, sizeof(int32_t));
// todo opt performance
for(int32_t i = 0; i < pStableVgroup->numOfDnodes; ++i) {
taosArrayPush(pInfo->vgroupIdList, &pStableVgroup->dnodeIps[i]);
}
return pSql->res.code;
}
@ -2272,7 +2224,7 @@ int tscProcessDropTableRsp(SSqlObj *pSql) {
if (pTableMetaInfo->pTableMeta) {
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true);
// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true);
}
return 0;
@ -2293,7 +2245,7 @@ int tscProcessAlterTableMsgRsp(SSqlObj *pSql) {
bool isSuperTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo);
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true);
// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true);
if (isSuperTable) { // if it is a super table, reset whole query cache
tscTrace("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name);
@ -2329,9 +2281,10 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) {
pRes->numOfRows = htonl(pRetrieve->numOfRows);
pRes->precision = htons(pRetrieve->precision);
pRes->offset = htobe64(pRetrieve->offset);
pRes->useconds = htobe64(pRetrieve->useconds);
pRes->data = pRetrieve->data;
pRes->offset = htobe64(pRetrieve->offset);
pRes->useconds = htobe64(pRetrieve->useconds);
pRes->completed = (pRetrieve->completed == 1);
pRes->data = pRetrieve->data;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
tscSetResultPointer(pQueryInfo, pRes);
@ -2356,7 +2309,6 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) {
}
pRes->row = 0;
tscTrace("%p numOfRows:%d, offset:%d", pSql, pRes->numOfRows, pRes->offset);
return 0;
@ -2379,7 +2331,7 @@ int tscProcessRetrieveRspFromLocal(SSqlObj *pSql) {
void tscTableMetaCallBack(void *param, TAOS_RES *res, int code);
static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) {
static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) {
SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
if (NULL == pNew) {
tscError("%p malloc failed for new sqlobj to get meter meta", pSql);
@ -2406,7 +2358,7 @@ static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMet
STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo);
assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
strcpy(pNewMeterMetaInfo->name, pTableMetaInfo->name);
strncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, tListLen(pNewMeterMetaInfo->name));
memcpy(pNew->cmd.payload, pSql->cmd.payload, TSDB_DEFAULT_PAYLOAD_SIZE); // tag information if table does not exists.
tscTrace("%p new pSqlObj:%p to get tableMeta", pSql, pNew);
@ -2424,7 +2376,7 @@ static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMet
int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) {
assert(strlen(pTableMetaInfo->name) != 0);
// If this STableMetaInfo owns a metermeta, release it first
// If this STableMetaInfo owns a table meta, release it first
if (pTableMetaInfo->pTableMeta != NULL) {
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), false);
}
@ -2437,12 +2389,8 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) {
return TSDB_CODE_SUCCESS;
}
/*
* for async insert operation, release data block buffer before issue new object to get metermeta
* because in table meta callback function, the tscParse function will generate the submit data blocks
*/
return doGetMeterMetaFromServer(pSql, pTableMetaInfo);
return getTableMetaFromMgmt(pSql, pTableMetaInfo);
}
int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) {
@ -2491,7 +2439,7 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) {
tscWaitingForCreateTable(pCmd);
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
code = doGetMeterMetaFromServer(pSql, pTableMetaInfo); // todo ??
code = getTableMetaFromMgmt(pSql, pTableMetaInfo); // todo ??
} else {
tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql,
tscGetNumOfTags(pTableMetaInfo->pTableMeta), pCmd->numOfCols, pTableMetaInfo->pTableMeta->uid,
@ -2501,30 +2449,34 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) {
return code;
}
int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) {
int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
int code = TSDB_CODE_NETWORK_UNAVAIL;
SSqlCmd *pCmd = &pSql->cmd;
/*
* the query condition is serialized into pCmd->payload, we need to rebuild key for metricmeta info in cache.
*/
bool required = false;
//the query condition is serialized into pCmd->payload, we need to rebuild key for stable meta info in cache.
bool required = false;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
if (pQueryInfo->pTableMetaInfo[0]->vgroupIdList != NULL) {
return TSDB_CODE_SUCCESS;
}
#if 0
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0};
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
tscGetMetricMetaCacheKey(pQueryInfo, tagstr, pTableMetaInfo->pTableMeta->uid);
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false);
// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false);
SSuperTableMeta *ppMeta = (SSuperTableMeta *)taosCacheAcquireByName(tscCacheHandle, tagstr);
if (ppMeta == NULL) {
required = true;
break;
} else {
pTableMetaInfo->pMetricMeta = ppMeta;
// pTableMetaInfo->pMetricMeta = ppMeta;
}
}
@ -2532,12 +2484,13 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) {
if (!required) {
return TSDB_CODE_SUCCESS;
}
#endif
SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
pNew->pTscObj = pSql->pTscObj;
pNew->signature = pNew;
pNew->cmd.command = TSDB_SQL_METRIC;
pNew->cmd.command = TSDB_SQL_STABLEVGROUP;
SQueryInfo *pNewQueryInfo = NULL;
if ((code = tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo)) != TSDB_CODE_SUCCESS) {
@ -2574,7 +2527,7 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) {
// tscFreeSubqueryInfo(pCmd);
// }
tscTrace("%p allocate new pSqlObj:%p to get metricMeta", pSql, pNew);
tscTrace("%p allocate new pSqlObj:%p to get stable vgroupInfo", pSql, pNew);
pNew->fp = tscTableMetaCallBack;
pNew->param = pSql;
code = tscProcessSql(pNew);
@ -2611,7 +2564,7 @@ void tscInitMsgsFp() {
tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg;
tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg;
tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg;
tscBuildMsg[TSDB_SQL_METRIC] = tscBuildMetricMetaMsg;
tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg;
tscBuildMsg[TSDB_SQL_MULTI_META] = tscBuildMultiMeterMetaMsg;
tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg;
@ -2629,7 +2582,7 @@ void tscInitMsgsFp() {
tscProcessMsgRsp[TSDB_SQL_CONNECT] = tscProcessConnectRsp;
tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp;
tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp;
tscProcessMsgRsp[TSDB_SQL_METRIC] = tscProcessMetricMetaRsp;
tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp;
tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiMeterMetaRsp;
tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp;
@ -2655,7 +2608,4 @@ void tscInitMsgsFp() {
tscKeepConn[TSDB_SQL_SELECT] = 1;
tscKeepConn[TSDB_SQL_FETCH] = 1;
tscKeepConn[TSDB_SQL_HB] = 1;
tscUpdateVnodeMsg[TSDB_SQL_SELECT] = tscUpdateVnodeInQueryMsg;
tscUpdateVnodeMsg[TSDB_SQL_INSERT] = tscUpdateVnodeInSubmitMsg;
}

81
src/client/src/tscSql.c
View File

@ -147,7 +147,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
}
// tsRpcHeaderSize will be updated during RPC initialization, so only after it initialization, this value is valid
tsInsertHeadSize = tsRpcHeadSize + sizeof(SShellSubmitMsg);
tsInsertHeadSize = tsRpcHeadSize + sizeof(SMsgDesc) + sizeof(SSubmitMsg);
return pObj;
}
@ -546,11 +546,11 @@ static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) {
* if the global limitation is not reached, and current result has not exhausted, or next more vnodes are
* available, goes on
*/
if (pMetaInfo->vnodeIndex < pMetaInfo->pMetricMeta->numOfVnodes && pRes1->row < pRes1->numOfRows &&
(!tscHasReachLimitation(pQueryInfo1, pRes1))) {
allSubqueryExhausted = false;
break;
}
// if (pMetaInfo->vnodeIndex < pMetaInfo->pMetricMeta->numOfVnodes && pRes1->row < pRes1->numOfRows &&
// (!tscHasReachLimitation(pQueryInfo1, pRes1))) {
// allSubqueryExhausted = false;
// break;
// }
}
hasData = !allSubqueryExhausted;
@ -594,11 +594,7 @@ static void **tscBuildResFromSubqueries(SSqlObj *pSql) {
}
if (numOfTableHasRes >= 2) { // do merge result
success = (doSetResultRowData(pSql->pSubs[0]) != NULL) && (doSetResultRowData(pSql->pSubs[1]) != NULL);
// TSKEY key1 = *(TSKEY *)pRes1->tsrow[0];
// TSKEY key2 = *(TSKEY *)pRes2->tsrow[0];
// printf("first:%" PRId64 ", second:%" PRId64 "\n", key1, key2);
} else { // only one subquery
SSqlObj *pSub = pSql->pSubs[0];
if (pSub == NULL) {
@ -653,66 +649,9 @@ static void **tscBuildResFromSubqueries(SSqlObj *pSql) {
return pRes->tsrow;
}
TAOS_ROW taos_fetch_row_impl(TAOS_RES *res) {
SSqlObj *pSql = (SSqlObj *)res;
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
if (pRes->qhandle == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
return NULL;
}
if (pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) {
tscFetchDatablockFromSubquery(pSql);
if (pRes->code == TSDB_CODE_SUCCESS) {
tscTrace("%p data from all subqueries have been retrieved to client", pSql);
return tscBuildResFromSubqueries(pSql);
} else {
tscTrace("%p retrieve data from subquery failed, code:%d", pSql, pRes->code);
return NULL;
}
} else if (pRes->row >= pRes->numOfRows) {
/**
* NOT a join query
*
* If the data block of current result set have been consumed already, try fetch next result
* data block from virtual node.
*/
tscResetForNextRetrieve(pRes);
if (pCmd->command < TSDB_SQL_LOCAL) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
}
tscProcessSql(pSql); // retrieve data from virtual node
// if failed to retrieve data from current virtual node, try next one if exists
if (hasMoreVnodesToTry(pSql)) {
tscTryQueryNextVnode(pSql, NULL);
}
/*
* local reducer has handle this case,
* so no need to add the pRes->numOfRows for super table query
*/
if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC) {
pRes->numOfTotalInCurrentClause += pRes->numOfRows;
}
if (pRes->numOfRows == 0) {
return NULL;
}
}
return doSetResultRowData(pSql);
}
static void asyncFetchCallback(void *param, TAOS_RES *tres, int numOfRows) {
SSqlObj* pSql = (SSqlObj*) tres;
if (numOfRows < 0) {
// set the error code
if (numOfRows < 0) { // set the error code
pSql->res.code = -numOfRows;
}
@ -729,12 +668,14 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
if (pRes->qhandle == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT) {
if (pRes->qhandle == 0 ||
pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
pCmd->command == TSDB_SQL_INSERT) {
return NULL;
}
// current data are exhausted, fetch more data
if (pRes->data == NULL || (pRes->data != NULL && pRes->row >= pRes->numOfRows &&
if (pRes->data == NULL || (pRes->data != NULL && pRes->row >= pRes->numOfRows && pRes->completed != true &&
(pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_METRIC || pCmd->command == TSDB_SQL_FETCH))) {
taos_fetch_rows_a(res, asyncFetchCallback, pSql->pTscObj);

3
src/client/src/tscStream.c
View File

@ -80,7 +80,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) {
if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
if (code == 0 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
code = tscGetMetricMeta(pSql, 0);
code = tscGetSTableVgroupInfo(pSql, 0);
pSql->res.code = code;
if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
@ -504,7 +504,6 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
}
tsem_init(&pSql->rspSem, 0, 0);
tsem_init(&pSql->emptyRspSem, 0, 1);
SSqlInfo SQLInfo = {0};
tSQLParse(&SQLInfo, pSql->sqlstr);

32
src/client/src/tscSub.c
View File

@ -178,11 +178,11 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0);
int numOfTables = 0;
if (!UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta;
for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) {
SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i);
numOfTables += pVnodeSidList->numOfSids;
}
// SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta;
// for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) {
// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i);
// numOfTables += pVnodeSidList->numOfSids;
// }
}
SSubscriptionProgress* progress = (SSubscriptionProgress*)calloc(numOfTables, sizeof(SSubscriptionProgress));
@ -197,17 +197,17 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
progress[0].uid = uid;
progress[0].key = tscGetSubscriptionProgress(pSub, uid);
} else {
SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta;
numOfTables = 0;
for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) {
SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i);
for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) {
STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j);
int64_t uid = pTableMetaInfo->uid;
progress[numOfTables].uid = uid;
progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid);
}
}
// SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta;
// numOfTables = 0;
// for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) {
// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i);
// for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) {
// STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j);
// int64_t uid = pTableMetaInfo->uid;
// progress[numOfTables].uid = uid;
// progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid);
// }
// }
qsort(progress, numOfTables, sizeof(SSubscriptionProgress), tscCompareSubscriptionProgress);
}

231
src/client/src/tscSubquery.c
View File

@ -476,7 +476,8 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
assert(pQueryInfo->numOfTables == 1);
// for projection query, need to try next vnode
int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
int32_t totalVnode = 0;
if ((++pTableMetaInfo->vnodeIndex) < totalVnode) {
tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql,
pTableMetaInfo->vnodeIndex - 1, pTableMetaInfo->vnodeIndex, totalVnode, pRes->numOfTotal);
@ -541,16 +542,16 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
assert(pQueryInfo->numOfTables == 1);
// for projection query, need to try next vnode if current vnode is exhausted
if ((++pTableMetaInfo->vnodeIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) {
pSupporter->pState->numOfCompleted = 0;
pSupporter->pState->numOfTotal = 1;
pSql->cmd.command = TSDB_SQL_SELECT;
pSql->fp = tscJoinQueryCallback;
tscProcessSql(pSql);
return;
}
// if ((++pTableMetaInfo->vnodeIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) {
// pSupporter->pState->numOfCompleted = 0;
// pSupporter->pState->numOfTotal = 1;
//
// pSql->cmd.command = TSDB_SQL_SELECT;
// pSql->fp = tscJoinQueryCallback;
// tscProcessSql(pSql);
//
// return;
// }
}
int32_t numOfTotal = pSupporter->pState->numOfTotal;
@ -608,10 +609,10 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes &&
(!tscHasReachLimitation(pQueryInfo, pRes))) {
numOfFetch++;
}
// if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes &&
// (!tscHasReachLimitation(pQueryInfo, pRes))) {
// numOfFetch++;
// }
} else {
if (pRes->row >= pRes->numOfRows && (!tscHasReachLimitation(pQueryInfo, pRes))) {
numOfFetch++;
@ -788,7 +789,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
* data instead of returning to its invoker
*/
if (pTableMetaInfo->vnodeIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
assert(pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes);
// assert(pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes);
pSupporter->pState->numOfCompleted = 0; // reset the record value
pSql->fp = joinRetrieveCallback; // continue retrieve data
@ -1005,11 +1006,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
pRes->qhandle = 1; // hack the qhandle check
const uint32_t nBufferSize = (1 << 16); // 64KB
const uint32_t nBufferSize = (1u << 16); // 64KB
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
int32_t numOfSubQueries = pTableMetaInfo->pMetricMeta->numOfVnodes;
int32_t numOfSubQueries = taosArrayGetSize(pTableMetaInfo->vgroupIdList);
assert(numOfSubQueries > 0);
int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize);
@ -1115,7 +1117,7 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) {
tfree(trsupport);
}
static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows);
static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows);
static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t errCode) {
// set no disk space error info
@ -1137,7 +1139,7 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES
pthread_mutex_unlock(&trsupport->queryMutex);
tscRetrieveFromVnodeCallBack(trsupport, tres, trsupport->pState->code);
tscRetrieveFromDnodeCallBack(trsupport, tres, trsupport->pState->code);
}
static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) {
@ -1232,7 +1234,86 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq
}
}
static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) {
int32_t idx = trsupport->subqueryIndex;
SSqlObj * pPObj = trsupport->pParentSqlObj;
tOrderDescriptor *pDesc = trsupport->pOrderDescriptor;
SSubqueryState* pState = trsupport->pState;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
// data in from current vnode is stored in cache and disk
uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
// tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip,
// pSvd->vnode, numOfRowsFromSubquery, idx);
tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
#ifdef _DEBUG_VIEW
printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems);
SSrcColumnInfo colInfo[256] = {0};
tscGetSrcColumnInfo(colInfo, pQueryInfo);
tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems,
trsupport->localBuffer->numOfElems, colInfo);
#endif
if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) {
tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql,
tsAvailTmpDirGB, tsMinimalTmpDirGB);
tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE);
return;
}
// each result for a vnode is ordered as an independant list,
// then used as an input of loser tree for disk-based merge routine
int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer,
pQueryInfo->groupbyExpr.orderType);
if (ret != 0) { // set no disk space error info, and abort retry
return tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE);
}
// keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion
// increases the finished value up to pState->numOfTotal value, which means all subqueries are completed.
// In this case, the comparsion between finished value and released pState->numOfTotal is not safe.
int32_t numOfTotal = pState->numOfTotal;
int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1);
if (finished < numOfTotal) {
tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished);
return tscFreeSubSqlObj(trsupport, pSql);
}
// all sub-queries are returned, start to local merge process
pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage;
tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj,
pState->numOfTotal, pState->numOfRetrievedRows);
SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0);
tscClearInterpInfo(pPQueryInfo);
tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel,
&pPObj->cmd, &pPObj->res);
tscTrace("%p build loser tree completed", pPObj);
pPObj->res.precision = pSql->res.precision;
pPObj->res.numOfRows = 0;
pPObj->res.row = 0;
// only free once
tfree(trsupport->pState);
tscFreeSubSqlObj(trsupport, pSql);
// set the command flag must be after the semaphore been correctly set.
pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC;
if (pPObj->res.code == TSDB_CODE_SUCCESS) {
(*pPObj->fp)(pPObj->param, pPObj, 0);
} else {
tscQueueAsyncRes(pPObj);
}
}
static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
SRetrieveSupport *trsupport = (SRetrieveSupport *)param;
int32_t idx = trsupport->subqueryIndex;
SSqlObj * pPObj = trsupport->pParentSqlObj;
@ -1260,15 +1341,16 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx);
SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index];
// SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx);
// SVnodeSidList *vnodeInfo = 0;
// SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index];
if (numOfRows > 0) {
assert(pRes->numOfRows == numOfRows);
int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows);
tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql,
pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx);
// tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql,
// pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx);
if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) {
tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId64 " , current:%" PRId64,
@ -1294,85 +1376,16 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR
int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data,
pRes->numOfRows, pQueryInfo->groupbyExpr.orderType);
if (ret < 0) {
// set no disk space error info, and abort retry
if (ret < 0) { // set no disk space error info, and abort retry
tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE);
} else {
} else if (pRes->completed) {
tscAllDataRetrievedFromDnode(trsupport, pSql);
} else { // continue fetch data from dnode
pthread_mutex_unlock(&trsupport->queryMutex);
taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param);
}
} else { // all data has been retrieved to client
/* data in from current vnode is stored in cache and disk */
uint32_t numOfRowsFromVnode = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip,
pSvd->vnode, numOfRowsFromVnode, idx);
tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
#ifdef _DEBUG_VIEW
printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems);
SSrcColumnInfo colInfo[256] = {0};
tscGetSrcColumnInfo(colInfo, pQueryInfo);
tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems,
trsupport->localBuffer->numOfElems, colInfo);
#endif
if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) {
tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql,
tsAvailTmpDirGB, tsMinimalTmpDirGB);
tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE);
return;
}
// each result for a vnode is ordered as an independant list,
// then used as an input of loser tree for disk-based merge routine
int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer,
pQueryInfo->groupbyExpr.orderType);
if (ret != 0) {
/* set no disk space error info, and abort retry */
return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE);
}
// keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion
// increases the finished value up to pState->numOfTotal value, which means all subqueries are completed.
// In this case, the comparsion between finished value and released pState->numOfTotal is not safe.
int32_t numOfTotal = pState->numOfTotal;
int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1);
if (finished < numOfTotal) {
tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished);
return tscFreeSubSqlObj(trsupport, pSql);
}
// all sub-queries are returned, start to local merge process
pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage;
tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj,
pState->numOfTotal, pState->numOfRetrievedRows);
SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0);
tscClearInterpInfo(pPQueryInfo);
tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel,
&pPObj->cmd, &pPObj->res);
tscTrace("%p build loser tree completed", pPObj);
pPObj->res.precision = pSql->res.precision;
pPObj->res.numOfRows = 0;
pPObj->res.row = 0;
// only free once
tfree(trsupport->pState);
tscFreeSubSqlObj(trsupport, pSql);
// set the command flag must be after the semaphore been correctly set.
pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC;
if (pPObj->res.code == TSDB_CODE_SUCCESS) {
(*pPObj->fp)(pPObj->param, pPObj, 0);
} else {
tscQueueAsyncRes(pPObj);
taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param);
}
} else { // all data has been retrieved to client
tscAllDataRetrievedFromDnode(trsupport, pSql);
}
}
@ -1409,10 +1422,10 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
SVnodeSidList *vnodeInfo = NULL;
SVnodeDesc * pSvd = NULL;
if (pTableMetaInfo->pMetricMeta != NULL) {
vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx);
pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index];
}
// if (pTableMetaInfo->pMetricMeta != NULL) {
// vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx);
// pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index];
// }
SSubqueryState* pState = trsupport->pState;
assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 &&
@ -1432,7 +1445,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
/*
* if a query on a vnode is failed, all retrieve operations from vnode that occurs later
* than this one are actually not necessary, we simply call the tscRetrieveFromVnodeCallBack
* than this one are actually not necessary, we simply call the tscRetrieveFromDnodeCallBack
* function to abort current and remain retrieve process.
*
* NOTE: threadsafe is required.
@ -1453,7 +1466,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
} else {
SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
assert(pNewQueryInfo->pTableMetaInfo[0]->pTableMeta != NULL && pNewQueryInfo->pTableMetaInfo[0]->pMetricMeta != NULL);
// assert(pNewQueryInfo->pTableMetaInfo[0]->pTableMeta != NULL && pNewQueryInfo->pTableMetaInfo[0]->pMetricMeta != NULL);
tscProcessSql(pNew);
return;
}
@ -1470,7 +1483,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
trsupport->subqueryIndex, pState->code);
}
tscRetrieveFromVnodeCallBack(param, tres, pState->code);
tscRetrieveFromDnodeCallBack(param, tres, pState->code);
} else { // success, proceed to retrieve data from dnode
if (vnodeInfo != NULL) {
tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql,
@ -1481,7 +1494,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
trsupport->subqueryIndex);
}
taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param);
taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param);
}
}

140
src/client/src/tscUtil.c
View File

@ -53,7 +53,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) {
const int32_t maxKeySize = TSDB_MAX_TAGS_LEN; // allowed max key size
SCond* cond = tsGetMetricQueryCondPos(pTagCond, uid);
SCond* cond = tsGetSTableQueryCondPos(pTagCond, uid);
char join[512] = {0};
if (pTagCond->joinInfo.hasJoin) {
@ -92,7 +92,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) {
free(tmp);
}
SCond* tsGetMetricQueryCondPos(STagCond* pTagCond, uint64_t uid) {
SCond* tsGetSTableQueryCondPos(STagCond* pTagCond, uint64_t uid) {
for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) {
if (uid == pTagCond->cond[i].uid) {
return &pTagCond->cond[i];
@ -102,7 +102,8 @@ SCond* tsGetMetricQueryCondPos(STagCond* pTagCond, uint64_t uid) {
return NULL;
}
void tsSetMetricQueryCond(STagCond* pTagCond, uint64_t uid, const char* str) {
// todo refactor by using SArray
void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str) {
size_t len = strlen(str);
if (len == 0) {
return;
@ -122,7 +123,7 @@ bool tscQueryOnMetric(SSqlCmd* pCmd) {
(pCmd->msgType == TSDB_MSG_TYPE_QUERY);
}
bool tscQueryMetricTags(SQueryInfo* pQueryInfo) {
bool tscQueryTags(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) {
if (tscSqlExprGet(pQueryInfo, i)->functionId != TSDB_FUNC_TAGPRJ) {
return false;
@ -172,6 +173,7 @@ void tscGetDBInfoFromMeterId(char* tableId, char* db) {
}
SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx) {
#if 0
if (pMetricmeta == NULL) {
tscError("illegal metricmeta");
return 0;
@ -189,6 +191,8 @@ SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx
}
return (SVnodeSidList*)(pMetricmeta->list[vnodeIdx] + (char*)pMetricmeta);
#endif
}
STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) {
@ -221,12 +225,12 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
// for select query super table, the metricmeta can not be null in any cases.
if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
assert(pTableMetaInfo->pMetricMeta != NULL);
// assert(pTableMetaInfo->pMetricMeta != NULL);
}
if (pTableMetaInfo->pMetricMeta == NULL) {
return false;
}
// if (pTableMetaInfo->pMetricMeta == NULL) {
// return false;
// }
if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) {
return false;
@ -259,12 +263,12 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) {
}
// only query on tag, not a projection query
if (tscQueryMetricTags(pQueryInfo)) {
if (tscQueryTags(pQueryInfo)) {
return false;
}
// for project query, only the following two function is allowed
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) {
for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) {
int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId;
if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG &&
functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) {
@ -423,9 +427,6 @@ void tscFreeResData(SSqlObj* pSql) {
}
void tscFreeSqlResult(SSqlObj* pSql) {
//TODO not free
return;
tfree(pSql->res.pRsp);
pSql->res.row = 0;
pSql->res.numOfRows = 0;
@ -469,8 +470,6 @@ void tscFreeSqlObjPartial(SSqlObj* pSql) {
tscFreeSqlCmdData(pCmd);
tscTrace("%p free sqlObj partial completed", pSql);
tscFreeSqlCmdData(pCmd);
}
void tscFreeSqlObj(SSqlObj* pSql) {
@ -489,10 +488,7 @@ void tscFreeSqlObj(SSqlObj* pSql) {
pCmd->allocSize = 0;
if (pSql->fp == NULL) {
tsem_destroy(&pSql->rspSem);
tsem_destroy(&pSql->emptyRspSem);
}
tsem_destroy(&pSql->rspSem);
free(pSql);
}
@ -695,6 +691,49 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList,
return TSDB_CODE_SUCCESS;
}
static void trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
int32_t firstPartLen = 0;
STableMeta* pTableMeta = pTableDataBlock->pTableMeta;
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
SSchema* pSchema = tscGetTableSchema(pTableMeta);
memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk));
pDataBlock += sizeof(SSubmitBlk);
int32_t total = sizeof(int32_t)*2;
for(int32_t i = 0; i < tinfo.numOfColumns; ++i) {
switch (pSchema[i].type) {
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_BINARY: {
assert(0); // not support binary yet
firstPartLen += sizeof(int32_t);break;
}
default:
firstPartLen += tDataTypeDesc[pSchema[i].type].nSize;
total += tDataTypeDesc[pSchema[i].type].nSize;
}
}
char* p = pTableDataBlock->pData + sizeof(SSubmitBlk);
SSubmitBlk* pBlock = (SSubmitBlk*) pTableDataBlock->pData;
int32_t rows = htons(pBlock->numOfRows);
for(int32_t i = 0; i < rows; ++i) {
*(int32_t*) pDataBlock = total;
pDataBlock += sizeof(int32_t);
*(int32_t*) pDataBlock = firstPartLen;
pDataBlock += sizeof(int32_t);
memcpy(pDataBlock, p, pTableDataBlock->rowSize);
p += pTableDataBlock->rowSize;
pDataBlock += pTableDataBlock->rowSize;
}
}
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) {
SSqlCmd* pCmd = &pSql->cmd;
@ -716,7 +755,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
return ret;
}
int64_t destSize = dataBuf->size + pOneTableBlock->size;
int64_t destSize = dataBuf->size + pOneTableBlock->size + pOneTableBlock->size*sizeof(int32_t)*2;
if (dataBuf->nAllocSize < destSize) {
while (dataBuf->nAllocSize < destSize) {
dataBuf->nAllocSize = dataBuf->nAllocSize * 1.5;
@ -730,29 +769,33 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
tscError("%p failed to allocate memory for merging submit block, size:%d", pSql, dataBuf->nAllocSize);
taosHashCleanup(pVnodeDataBlockHashList);
tfree(dataBuf->pData);
tscDestroyBlockArrayList(pVnodeDataBlockList);
tfree(dataBuf->pData);
return TSDB_CODE_CLI_OUT_OF_MEMORY;
}
}
SShellSubmitBlock* pBlocks = (SShellSubmitBlock*)pOneTableBlock->pData;
SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
sortRemoveDuplicates(pOneTableBlock);
char* e = (char*)pBlocks->payLoad + pOneTableBlock->rowSize*(pBlocks->numOfRows-1);
char* e = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1);
tscTrace("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId, pBlocks->sid,
pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->payLoad), GET_INT64_VAL(e));
tscTrace("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId,
pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(e));
pBlocks->sid = htonl(pBlocks->sid);
int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + sizeof(int32_t) * 2);
pBlocks->tid = htonl(pBlocks->tid);
pBlocks->uid = htobe64(pBlocks->uid);
pBlocks->sversion = htonl(pBlocks->sversion);
pBlocks->numOfRows = htons(pBlocks->numOfRows);
memcpy(dataBuf->pData + dataBuf->size, pOneTableBlock->pData, pOneTableBlock->size);
dataBuf->size += pOneTableBlock->size;
pBlocks->len = htonl(len);
// erase the empty space reserved for binary data
trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
dataBuf->size += (len + sizeof(SSubmitBlk));
dataBuf->numOfTables += 1;
}
@ -1704,16 +1747,8 @@ bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql) {
}
int32_t command = pSql->cmd.command;
if (pTscObj->pSql == pSql) {
/*
* in case of taos_connect_a query, the object should all be released, even it is the
* master sql object. Otherwise, the master sql should not be released
*/
if (command == TSDB_SQL_CONNECT && pSql->res.code != TSDB_CODE_SUCCESS) {
return true;
}
return false;
if (command == TSDB_SQL_CONNECT) {
return true;
}
if (command == TSDB_SQL_INSERT) {
@ -1894,7 +1929,7 @@ STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
}
pTableMetaInfo->pTableMeta = pTableMeta;
pTableMetaInfo->pMetricMeta = pMetricMeta;
// pTableMetaInfo->pMetricMeta = pMetricMeta;
pTableMetaInfo->numOfTags = numOfTags;
if (tags != NULL) {
@ -1944,7 +1979,7 @@ void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache)
}
taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache);
taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache);
// taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache);
}
void tscResetForNextRetrieve(SSqlRes* pRes) {
@ -2083,15 +2118,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0);
STableMeta* pPrevMeterMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta);
SSuperTableMeta* pPrevMetricMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMetricMeta);
// SSuperTableMeta* pPrevMetricMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMetricMeta);
pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags,
pTableMetaInfo->tagColumnIndex);
// pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags,
// pTableMetaInfo->tagColumnIndex);
}
assert(pFinalInfo->pTableMeta != NULL && pNewQueryInfo->numOfTables == 1);
if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
assert(pFinalInfo->pMetricMeta != NULL);
// assert(pFinalInfo->pMetricMeta != NULL);
}
tscTrace(
@ -2191,13 +2226,13 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) {
// if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) {
return false;
}
// }
int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) &&
(!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vnodeIndex < totalVnode - 1);
// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
// return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) &&
// (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vnodeIndex < totalVnode - 1);
}
void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
@ -2213,7 +2248,8 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes));
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
int32_t totalVnode = 0;
// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
while (++pTableMetaInfo->vnodeIndex < totalVnode) {
tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql,

167
src/client/tests/timeParseTest.cpp Normal file
View File

@ -0,0 +1,167 @@
#include <gtest/gtest.h>
#include <cassert>
#include <iostream>
#include "taos.h"
#include "tstoken.h"
#include "ttime.h"
#include "tutil.h"
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
extern void deltaToUtcInitOnce();
/* test parse time function */
TEST(testCase, parse_time) {
taos_options(TSDB_OPTION_TIMEZONE, "GMT-8");
deltaToUtcInitOnce();
char t1[] = "2018-1-1 1:1:1.952798";
char t13[] = "1970-1-1 0:0:0";
int64_t time = 0, time1 = 0;
taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952);
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND);
char t2[] = "2018-1-1T1:1:1.952Z";
taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952 + 28800000);
char t3[] = "2018-1-1 1:01:01.952";
taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952);
char t4[] = "2018-1-1 1:01:01.9";
char t5[] = "2018-1-1 1:01:1.900";
char t6[] = "2018-01-01 1:1:1.90";
char t7[] = "2018-01-01 01:01:01.9";
char t8[] = "2018-01-01 01:01:01.9007865";
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t9[] = "2017-4-3 1:1:2.980";
char t10[] = "2017-4-3T2:1:2.98+9:00";
taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t11[] = "2017-4-3T2:1:2.98+09:00";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t12[] = "2017-4-3T2:1:2.98+0900";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taos_options(TSDB_OPTION_TIMEZONE, "UTC");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 0);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
deltaToUtcInitOnce();
char t14[] = "1970-1-1T0:0:0Z";
taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 0);
char t40[] = "1970-1-1 0:0:0.999999999";
taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND);
char t41[] = "1997-1-1 0:0:0.999999999";
taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 852048000999);
int64_t k = timezone;
char t42[] = "1997-1-1T0:0:0.999999999Z";
taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND);
////////////////////////////////////////////////////////////////////
// illegal timestamp format
char t15[] = "2017-12-33 0:0:0";
EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1);
char t16[] = "2017-12-31 99:0:0";
EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1);
char t17[] = "2017-12-31T9:0:0";
EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1);
char t18[] = "2017-12-31T9:0:0.Z";
EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1);
char t19[] = "2017-12-31 9:0:0.-1";
EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1);
char t20[] = "2017-12-31 9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0);
EXPECT_EQ(time, 1514682000100);
char t21[] = "2017-12-31T9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1);
char t22[] = "2017-12-31 9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0);
char t23[] = "2017-12-31T9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0);
//======================== add some case ============================//
char b1[] = "9999-12-31 23:59:59.999";
taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 253402271999999);
char b2[] = "2020-01-01 01:01:01.321";
taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1577811661321);
taos_options(TSDB_OPTION_TIMEZONE, "America/New_York");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND);
}

33
src/common/inc/dataformat.h
View File

@ -81,11 +81,13 @@ STSchema *tdDecodeSchema(void **psrc);
*/
typedef void *SDataRow;
#define TD_DATA_ROW_HEAD_SIZE (2 * sizeof(int32_t))
#define dataRowLen(r) (*(int32_t *)(r))
#define dataRowFLen(r) (*(int32_t *)((char *)(r) + sizeof(int32_t)))
#define dataRowTuple(r) ((char *)(r) + TD_DATA_ROW_HEAD_SIZE)
#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
#define dataRowSetLen(r, l) (dataRowLen(r) = (l))
#define dataRowSetFLen(r, l) (dataRowFLen(r) = (l))
#define dataRowIdx(r, i) ((char *)(r) + i)
@ -103,11 +105,36 @@ SDataRow tdDataRowDup(SDataRow row);
// ----------------- Data column structure
typedef struct SDataCol {
int64_t len;
char data[];
int8_t type;
int16_t colId;
int bytes;
int len;
int offset;
void * pData;
} SDataCol;
void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter);
typedef struct {
int maxRowSize;
int maxCols; // max number of columns
int maxPoints; // max number of points
int numOfPoints;
int numOfCols; // Total number of cols
int sversion; // TODO: set sversion
void * buf;
SDataCol cols[];
} SDataCols;
#define keyCol(pCols) (&((pCols)->cols[0])) // Key column
#define dataColsKeyAt(pCols, idx) ((int64_t *)(keyCol(pCols)->pData))[(idx)]
#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
#define dataColsKeyLast(pCols) dataColsKeyAt(pCols, (pCols)->numOfPoints - 1)
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
void tdResetDataCols(SDataCols *pCols);
void tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
void tdFreeDataCols(SDataCols *pCols);
void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols);
void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop);
#ifdef __cplusplus
}

77
src/common/src/dataformat.c
View File

@ -150,7 +150,7 @@ void tdFreeSchema(STSchema *pSchema) {
*/
void tdUpdateSchema(STSchema *pSchema) {
STColumn *pCol = NULL;
int32_t offset = 0;
int32_t offset = TD_DATA_ROW_HEAD_SIZE;
for (int i = 0; i < schemaNCols(pSchema); i++) {
pCol = schemaColAt(pSchema, i);
colSetOffset(pCol, offset);
@ -294,14 +294,79 @@ SDataRow tdDataRowDup(SDataRow row) {
return trow;
}
void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter) {
int row = *iter;
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * maxCols);
if (pCols == NULL) return NULL;
for (int i = 0; i < schemaNCols(pSchema); i++) {
// TODO
pCols->maxRowSize = maxRowSize;
pCols->maxCols = maxCols;
pCols->maxPoints = maxRows;
pCols->buf = malloc(maxRowSize * maxRows);
if (pCols->buf == NULL) {
free(pCols);
return NULL;
}
*iter = row + 1;
return pCols;
}
void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
// assert(schemaNCols(pSchema) <= pCols->numOfCols);
tdResetDataCols(pCols);
pCols->numOfCols = schemaNCols(pSchema);
pCols->cols[0].pData = pCols->buf;
for (int i = 0; i < schemaNCols(pSchema); i++) {
if (i > 0) {
pCols->cols[i].pData = (char *)(pCols->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * pCols->maxPoints;
}
pCols->cols[i].type = colType(schemaColAt(pSchema, i));
pCols->cols[i].bytes = colBytes(schemaColAt(pSchema, i));
pCols->cols[i].offset = colOffset(schemaColAt(pSchema, i));
pCols->cols[i].colId = colColId(schemaColAt(pSchema, i));
}
return pCols;
}
void tdFreeDataCols(SDataCols *pCols) {
if (pCols) {
if (pCols->buf) free(pCols->buf);
free(pCols);
}
}
void tdResetDataCols(SDataCols *pCols) {
pCols->numOfPoints = 0;
for (int i = 0; i < pCols->maxCols; i++) {
pCols->cols[i].len = 0;
}
}
void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols) {
TSKEY key = dataRowKey(row);
for (int i = 0; i < pCols->numOfCols; i++) {
SDataCol *pCol = pCols->cols + i;
memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, pCol->offset), pCol->bytes);
pCol->len += pCol->bytes;
}
pCols->numOfPoints++;
}
// Pop pointsToPop points from the SDataCols
void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
int pointsLeft = pCols->numOfPoints - pointsToPop;
for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
SDataCol *p_col = pCols->cols + iCol;
if (p_col->len > 0) {
p_col->len = TYPE_BYTES[p_col->type] * pointsLeft;
if (pointsLeft > 0) {
memmove((void *)(p_col->pData), (void *)((char *)(p_col->pData) + TYPE_BYTES[p_col->type] * pointsToPop), p_col->len);
}
}
}
pCols->numOfPoints = pointsLeft;
}
/**

2
src/dnode/CMakeLists.txt
View File

@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
AUX_SOURCE_DIRECTORY(src SRC)
ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode sdb taos_static monitor http tsdb)
TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb)
#IF (TD_CLUSTER)
# TARGET_LINK_LIBRARIES(taosd dcluster)

16
src/dnode/src/dnodeMgmt.c
View File

@ -225,17 +225,17 @@ static int32_t dnodeOpenVnode(int32_t vnode, char *rootDir) {
vnodeObj.status = TSDB_VN_STATUS_NOT_READY;
vnodeObj.refCount = 1;
vnodeObj.version = 0;
vnodeObj.wworker = dnodeAllocateWriteWorker(&vnodeObj);
vnodeObj.rworker = dnodeAllocateReadWorker(&vnodeObj);
vnodeObj.wal = NULL;
vnodeObj.tsdb = pTsdb;
vnodeObj.replica = NULL;
vnodeObj.events = NULL;
vnodeObj.cq = NULL;
taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *) (&vnodeObj));
SVnodeObj *pVnode = (SVnodeObj *)taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *)(&vnodeObj));
pVnode->wworker = dnodeAllocateWriteWorker(pVnode);
pVnode->rworker = dnodeAllocateReadWorker(pVnode);
dTrace("open vnode:%d in %s", vnodeObj.vgId, rootDir);
dTrace("open vnode:%d in %s", pVnode->vgId, rootDir);
return TSDB_CODE_SUCCESS;
}
@ -314,17 +314,17 @@ static int32_t dnodeCreateVnode(SMDCreateVnodeMsg *pVnodeCfg) {
vnodeObj.status = TSDB_VN_STATUS_NOT_READY;
vnodeObj.refCount = 1;
vnodeObj.version = 0;
vnodeObj.wworker = dnodeAllocateWriteWorker(&vnodeObj);
vnodeObj.rworker = dnodeAllocateReadWorker(&vnodeObj);
vnodeObj.wal = NULL;
vnodeObj.tsdb = pTsdb;
vnodeObj.replica = NULL;
vnodeObj.events = NULL;
vnodeObj.cq = NULL;
taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *) (&vnodeObj));
SVnodeObj *pVnode = (SVnodeObj *)taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *)(&vnodeObj));
pVnode->wworker = dnodeAllocateWriteWorker(pVnode);
pVnode->rworker = dnodeAllocateReadWorker(pVnode);
dPrint("vgroup:%d, vnode:%d is created", vnodeObj.vgId, vnodeObj.vgId);
dPrint("vgroup:%d, vnode:%d is created", pVnode->vgId, pVnode->vgId);
return TSDB_CODE_SUCCESS;
}

158
src/dnode/src/dnodeRead.c
View File

@ -76,7 +76,9 @@ void dnodeRead(SRpcMsg *pMsg) {
int32_t leftLen = pMsg->contLen;
char *pCont = (char *) pMsg->pCont;
SRpcContext *pRpcContext = NULL;
dTrace("dnode read msg disposal");
// SMsgDesc *pDesc = pCont;
// pDesc->numOfVnodes = htonl(pDesc->numOfVnodes);
// pCont += sizeof(SMsgDesc);
@ -90,8 +92,8 @@ void dnodeRead(SRpcMsg *pMsg) {
while (leftLen > 0) {
SMsgHead *pHead = (SMsgHead *) pCont;
pHead->vgId = 1; //htonl(pHead->vgId);
pHead->contLen = pMsg->contLen; //htonl(pHead->contLen);
pHead->vgId = htonl(pHead->vgId);
pHead->contLen = htonl(pHead->contLen);
void *pVnode = dnodeGetVnode(pHead->vgId);
if (pVnode == NULL) {
@ -162,7 +164,7 @@ static void *dnodeProcessReadQueue(void *param) {
void *pVnode;
while (1) {
if (taosReadQitemFromQset(qset, &type, &pReadMsg, &pVnode) == 0) {
if (taosReadQitemFromQset(qset, &type, (void **)&pReadMsg, (void **)&pVnode) == 0) {
dnodeHandleIdleReadWorker();
continue;
}
@ -174,10 +176,10 @@ static void *dnodeProcessReadQueue(void *param) {
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
}
dnodeProcessReadResult(pVnode, pReadMsg);
// dnodeProcessReadResult(pVnode, pReadMsg);
taosFreeQitem(pReadMsg);
dnodeReleaseVnode(pVnode);
dnodeReleaseVnode(pVnode);
}
return NULL;
@ -218,88 +220,92 @@ static void dnodeProcessReadResult(void *pVnode, SReadMsg *pRead) {
code = terrno;
}
SRpcMsg rsp;
rsp.handle = pRead->rpcMsg.handle;
rsp.code = code;
rsp.pCont = NULL;
rpcSendResponse(&rsp);
//TODO: query handle is returned by dnodeProcessQueryMsg
if (0) {
SRpcMsg rsp;
rsp.handle = pRead->rpcMsg.handle;
rsp.code = code;
rsp.pCont = NULL;
rpcSendResponse(&rsp);
}
rpcFreeCont(pRead->rpcMsg.pCont); // free the received message
}
static void dnodeProcessQueryMsg(SReadMsg *pMsg) {
static void dnodeContinueExecuteQuery(void* pVnode, void* qhandle, SReadMsg *pMsg) {
SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg));
pRead->rpcMsg = pMsg->rpcMsg;
pRead->pCont = qhandle;
pRead->contLen = 0;
pRead->pRpcContext = pMsg->pRpcContext;
pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
taos_queue queue = dnodeGetVnodeRworker(pVnode);
taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead);
}
static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg) {
SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pMsg->pCont;
SQInfo* pQInfo = NULL;
int32_t code = qCreateQueryInfo(pQueryTableMsg, &pQInfo);
if (pMsg->contLen != 0) {
void* tsdb = dnodeGetVnodeTsdb(pVnode);
int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo);
SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
pRsp->code = code;
pRsp->qhandle = htobe64((uint64_t) (pQInfo));
SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
pRsp->code = code;
pRsp->qhandle = htobe64((uint64_t) (pQInfo));
SRpcMsg rpcRsp = {
.handle = pMsg->rpcMsg.handle,
.pCont = pRsp,
.contLen = sizeof(SQueryTableRsp),
.code = code,
.msgType = 0
};
rpcSendResponse(&rpcRsp);
} else {
pQInfo = pMsg->pCont;
}
qTableQuery(pQInfo); // do execute query
}
SRpcMsg rpcRsp = {
static void dnodeProcessRetrieveMsg(void *pVnode, SReadMsg *pMsg) {
SRetrieveTableMsg *pRetrieve = pMsg->pCont;
void *pQInfo = (void*) htobe64(pRetrieve->qhandle);
dTrace("QInfo:%p vgId:%d, retrieve msg is received", pQInfo, pRetrieve->header.vgId);
int32_t contLen = 0;
SRetrieveTableRsp *pRsp = NULL;
int32_t code = qRetrieveQueryResultInfo(pQInfo);
if (code != TSDB_CODE_SUCCESS) {
contLen = sizeof(SRetrieveTableRsp);
pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen);
memset(pRsp, 0, sizeof(SRetrieveTableRsp));
} else {
// todo check code and handle error in build result set
code = qDumpRetrieveResult(pQInfo, &pRsp, &contLen);
if (qHasMoreResultsToRetrieve(pQInfo)) {
dnodeContinueExecuteQuery(pVnode, pQInfo, pMsg);
} else { // no further execution invoked, release the ref to vnode
dnodeProcessReadResult(pVnode, pMsg);
}
}
SRpcMsg rpcRsp = (SRpcMsg) {
.handle = pMsg->rpcMsg.handle,
.pCont = pRsp,
.contLen = sizeof(SQueryTableRsp),
.contLen = contLen,
.code = code,
.msgType = 0
};
// do execute query
qTableQuery(pQInfo);
rpcSendResponse(&rpcRsp);
}
static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) {
SRetrieveTableMsg *pRetrieve = pMsg->pCont;
void *pQInfo = htobe64(pRetrieve->qhandle);
dTrace("retrieve msg is disposed, qInfo:%p", pQInfo);
int32_t rowSize = 0;
int32_t numOfRows = 0;
int32_t contLen = 0;
SRpcMsg rpcRsp = {0};
int32_t code = qRetrieveQueryResultInfo(pQInfo, &numOfRows, &rowSize);
if (code != TSDB_CODE_SUCCESS) {
contLen = sizeof(SRetrieveTableRsp);
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen);
pRsp->numOfRows = 0;
pRsp->precision = 0;
pRsp->offset = 0;
pRsp->useconds = 0;
rpcRsp = (SRpcMsg) {
.handle = pMsg->rpcMsg.handle,
.pCont = pRsp,
.contLen = contLen,
.code = code,
.msgType = 0
};
//todo free qinfo
} else {
contLen = 100;
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen);
pRsp->numOfRows = 0;
pRsp->precision = 0;
pRsp->offset = 0;
pRsp->useconds = 0;
*(int64_t*) pRsp->data = 1000;
rpcRsp = (SRpcMsg) {
.handle = pMsg->rpcMsg.handle,
.pCont = pRsp,
.contLen = contLen,
.code = code,
.msgType = 0
};
}
rpcSendResponse(&rpcRsp);
}

51
src/dnode/src/dnodeWrite.c
View File

@ -53,12 +53,12 @@ typedef struct _thread_obj {
static void (*dnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(void *, SWriteMsg *);
static void *dnodeProcessWriteQueue(void *param);
static void dnodeHandleIdleWorker(SWriteWorker *pWorker);
static void dnodeProcessWriteResult(SWriteMsg *pWrite);
static void dnodeProcessSubmitMsg(SWriteMsg *pMsg);
static void dnodeProcessCreateTableMsg(SWriteMsg *pMsg);
static void dnodeProcessDropTableMsg(SWriteMsg *pMsg);
static void dnodeProcessAlterTableMsg(SWriteMsg *pMsg);
static void dnodeProcessDropStableMsg(SWriteMsg *pMsg);
static void dnodeProcessWriteResult(void *pVnode, SWriteMsg *pWrite);
static void dnodeProcessSubmitMsg(void *pVnode, SWriteMsg *pMsg);
static void dnodeProcessCreateTableMsg(void *pVnode, SWriteMsg *pMsg);
static void dnodeProcessDropTableMsg(void *pVnode, SWriteMsg *pMsg);
static void dnodeProcessAlterTableMsg(void *pVnode, SWriteMsg *pMsg);
static void dnodeProcessDropStableMsg(void *pVnode, SWriteMsg *pMsg);
SWriteWorkerPool wWorkerPool;
@ -193,20 +193,20 @@ static void *dnodeProcessWriteQueue(void *param) {
continue;
}
for (int32_t i=0; i<numOfMsgs; ++i) {
for (int32_t i = 0; i < numOfMsgs; ++i) {
// retrieve all items, and write them into WAL
taosGetQitem(qall, &type, &pWriteMsg);
taosGetQitem(qall, &type, (void **)&pWriteMsg);
// walWrite(pVnode->whandle, writeMsg.rpcMsg.msgType, writeMsg.pCont, writeMsg.contLen);
}
// flush WAL file
// walFsync(pVnode->whandle);
// browse all items, and process them one by one
taosResetQitems(qall);
for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, &type, &pWriteMsg);
taosGetQitem(qall, &type, (void **)&pWriteMsg);
terrno = 0;
if (dnodeProcessWriteMsgFp[pWriteMsg->rpcMsg.msgType]) {
@ -218,7 +218,6 @@ static void *dnodeProcessWriteQueue(void *param) {
dnodeProcessWriteResult(pVnode, pWriteMsg);
taosFreeQitem(pWriteMsg);
}
}
taosFreeQall(qall);
@ -270,7 +269,7 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) {
}
}
static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) {
static void dnodeProcessSubmitMsg(void *pVnode, SWriteMsg *pMsg) {
dTrace("submit msg is disposed");
SShellSubmitRspMsg *pRsp = rpcMallocCont(sizeof(SShellSubmitRspMsg));
@ -279,7 +278,10 @@ static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) {
pRsp->affectedRows = htonl(1);
pRsp->numOfFailedBlocks = 0;
// todo write to tsdb
void* tsdb = dnodeGetVnodeTsdb(pVnode);
assert(tsdb != NULL);
tsdbInsertData(tsdb, pMsg->pCont);
SRpcMsg rpcRsp = {
.handle = pMsg->rpcMsg.handle,
@ -288,10 +290,11 @@ static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) {
.code = 0,
.msgType = 0
};
rpcSendResponse(&rpcRsp);
}
static void dnodeProcessCreateTableMsg(SWriteMsg *pMsg) {
static void dnodeProcessCreateTableMsg(void *pVnode, SWriteMsg *pMsg) {
SMDCreateTableMsg *pTable = pMsg->rpcMsg.pCont;
SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
@ -340,16 +343,16 @@ static void dnodeProcessCreateTableMsg(SWriteMsg *pMsg) {
tsdbTableSetTagValue(&tCfg, dataRow, false);
}
void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode);
void *pTsdb = dnodeGetVnodeTsdb(pVnode);
rpcRsp.code = tsdbCreateTable(pTsdb, &tCfg);
dnodeReleaseVnode(pMsg->pVnode);
dnodeReleaseVnode(pVnode);
dTrace("table:%s, create table result:%s", pTable->tableId, tstrerror(rpcRsp.code));
rpcSendResponse(&rpcRsp);
}
static void dnodeProcessDropTableMsg(SWriteMsg *pMsg) {
static void dnodeProcessDropTableMsg(void *pVnode, SWriteMsg *pMsg) {
SMDDropTableMsg *pTable = pMsg->rpcMsg.pCont;
SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
@ -359,16 +362,16 @@ static void dnodeProcessDropTableMsg(SWriteMsg *pMsg) {
.tid = htonl(pTable->sid)
};
void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode);
void *pTsdb = dnodeGetVnodeTsdb(pVnode);
rpcRsp.code = tsdbDropTable(pTsdb, tableId);
dnodeReleaseVnode(pMsg->pVnode);
dnodeReleaseVnode(pVnode);
dTrace("table:%s, drop table result:%s", pTable->tableId, tstrerror(rpcRsp.code));
rpcSendResponse(&rpcRsp);
}
static void dnodeProcessAlterTableMsg(SWriteMsg *pMsg) {
static void dnodeProcessAlterTableMsg(void *pVnode, SWriteMsg *pMsg) {
SMDCreateTableMsg *pTable = pMsg->rpcMsg.pCont;
SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
@ -417,16 +420,16 @@ static void dnodeProcessAlterTableMsg(SWriteMsg *pMsg) {
tsdbTableSetTagValue(&tCfg, dataRow, false);
}
void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode);
void *pTsdb = dnodeGetVnodeTsdb(pVnode);
rpcRsp.code = tsdbAlterTable(pTsdb, &tCfg);
dnodeReleaseVnode(pMsg->pVnode);
dnodeReleaseVnode(pVnode);
dTrace("table:%s, alter table result:%s", pTable->tableId, tstrerror(rpcRsp.code));
rpcSendResponse(&rpcRsp);
}
static void dnodeProcessDropStableMsg(SWriteMsg *pMsg) {
static void dnodeProcessDropStableMsg(void *pVnode, SWriteMsg *pMsg) {
SMDDropSTableMsg *pTable = pMsg->rpcMsg.pCont;
SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
@ -438,7 +441,7 @@ static void dnodeProcessDropStableMsg(SWriteMsg *pMsg) {
//rpcRsp.code = tsdbDropSTable(pTsdb, pTable->uid);
rpcRsp.code = TSDB_CODE_SUCCESS;
dnodeReleaseVnode(pMsg->pVnode);
dnodeReleaseVnode(pVnode);
dTrace("stable:%s, drop stable result:%s", pTable->tableId, tstrerror(rpcRsp.code));
rpcSendResponse(&rpcRsp);

101
src/inc/mnode.h
View File

@ -25,7 +25,6 @@ extern "C" {
#include "taosdef.h"
#include "taosmsg.h"
#include "taoserror.h"
#include "sdb.h"
#include "tglobalcfg.h"
#include "thash.h"
#include "tidpool.h"
@ -54,8 +53,8 @@ typedef struct {
int8_t numOfMnodes;
int32_t numOfDnodes;
char mnodeName[TSDB_DNODE_NAME_LEN + 1];
char reserved[7];
char updateEnd[1];
int8_t reserved[15];
int8_t updateEnd[1];
int syncFd;
void *hbTimer;
void *pSync;
@ -79,8 +78,8 @@ typedef struct {
float lbScore; // calc in balance function
int32_t customScore; // config by user
char dnodeName[TSDB_DNODE_NAME_LEN + 1];
char reserved[7];
char updateEnd[1];
int8_t reserved[15];
int8_t updateEnd[1];
SVnodeLoad vload[TSDB_MAX_VNODES];
int32_t status;
uint32_t lastReboot; // time stamp for last reboot
@ -99,66 +98,46 @@ typedef struct {
} SVnodeGid;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN];
int8_t type;
int8_t dirty;
uint64_t uid;
int32_t sid;
int32_t vgId;
int64_t createdTime;
char tableId[TSDB_TABLE_ID_LEN];
int8_t type;
int8_t dirty;
} STableInfo;
struct _vg_obj;
typedef struct SSuperTableObj {
char tableId[TSDB_TABLE_ID_LEN + 1];
int8_t type;
int8_t dirty;
uint64_t uid;
int32_t sid;
int32_t vgId;
int64_t createdTime;
int32_t sversion;
int32_t numOfColumns;
int32_t numOfTags;
int8_t reserved[5];
int8_t updateEnd[1];
int32_t numOfTables;
int16_t nextColId;
SSchema *schema;
STableInfo info;
uint64_t uid;
int64_t createdTime;
int32_t sversion;
int32_t numOfColumns;
int32_t numOfTags;
int8_t reserved[15];
int8_t updateEnd[1];
int32_t numOfTables;
int16_t nextColId;
SSchema * schema;
} SSuperTableObj;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN + 1];
int8_t type;
int8_t dirty;
uint64_t uid;
int32_t sid;
int32_t vgId;
int64_t createdTime;
char superTableId[TSDB_TABLE_ID_LEN + 1];
int8_t reserved[1];
int8_t updateEnd[1];
STableInfo info;
uint64_t uid;
int64_t createdTime;
int32_t sversion; //used by normal table
int32_t numOfColumns; //used by normal table
int32_t sid;
int32_t vgId;
char superTableId[TSDB_TABLE_ID_LEN + 1];
int32_t sqlLen;
int8_t reserved[1];
int8_t updateEnd[1];
int16_t nextColId; //used by normal table
char* sql; //used by normal table
SSchema* schema; //used by normal table
SSuperTableObj *superTable;
} SChildTableObj;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN];
int8_t type;
int8_t dirty;
uint64_t uid;
int32_t sid;
int32_t vgId;
int64_t createdTime;
int32_t sversion;
int32_t numOfColumns;
int32_t sqlLen;
int8_t reserved[3];
int8_t updateEnd[1];
char* sql; //null-terminated string
int16_t nextColId;
SSchema* schema;
} SNormalTableObj;
struct _db_obj;
typedef struct _vg_obj {
uint32_t vgId;
@ -166,15 +145,16 @@ typedef struct _vg_obj {
int64_t createdTime;
SVnodeGid vnodeGid[TSDB_VNODES_SUPPORT];
int32_t numOfVnodes;
int32_t numOfTables;
int32_t lbIp;
int32_t lbTime;
int8_t lbStatus;
int8_t reserved[14];
int8_t updateEnd[1];
struct _vg_obj *prev, *next;
struct _db_obj *pDb;
int32_t numOfTables;
void * idPool;
STableInfo ** tableList;
SChildTableObj ** tableList;
} SVgObj;
typedef struct _db_obj {
@ -182,8 +162,8 @@ typedef struct _db_obj {
int8_t dirty;
int64_t createdTime;
SDbCfg cfg;
char reserved[15];
char updateEnd[1];
int8_t reserved[15];
int8_t updateEnd[1];
struct _db_obj *prev, *next;
int32_t numOfVgroups;
int32_t numOfTables;
@ -201,7 +181,7 @@ typedef struct _user_obj {
int64_t createdTime;
int8_t superAuth;
int8_t writeAuth;
int8_t reserved[16];
int8_t reserved[13];
int8_t updateEnd[1];
struct _user_obj *prev, *next;
struct _acctObj * pAcct;
@ -258,9 +238,10 @@ typedef struct {
typedef struct {
uint8_t msgType;
int8_t expected;
int8_t usePublicIp;
int8_t received;
int8_t successed;
int8_t expected;
int32_t contLen;
int32_t code;
void *ahandle;

131
src/inc/sdb.h
View File

@ -1,131 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_SDB_H
#define TDENGINE_SDB_H
#ifdef __cplusplus
extern "C" {
#endif
#include "taosmsg.h"
#include "taosdef.h"
extern uint16_t tsMgmtMgmtPort;
extern uint16_t tsMgmtSyncPort;
extern int tsMgmtPeerHBTimer; // seconds
extern char * sdbStatusStr[];
extern char * sdbRoleStr[];
extern int sdbMaster;
extern SRpcIpSet *pSdbIpList;
extern SRpcIpSet *pSdbPublicIpList;
extern void (*sdbWorkAsMasterCallback)(); // this function pointer will be set by taosd
enum _keytype {
SDB_KEYTYPE_STRING, SDB_KEYTYPE_UINT32, SDB_KEYTYPE_AUTO, SDB_KEYTYPE_RECYCLE, SDB_KEYTYPE_MAX
};
#define SDB_ROLE_UNAPPROVED 0
#define SDB_ROLE_UNDECIDED 1
#define SDB_ROLE_MASTER 2
#define SDB_ROLE_SLAVE 3
#define SDB_STATUS_OFFLINE 0
#define SDB_STATUS_UNSYNCED 1
#define SDB_STATUS_SYNCING 2
#define SDB_STATUS_SERVING 3
#define SDB_STATUS_DELETED 4
enum _sdbaction {
SDB_TYPE_INSERT,
SDB_TYPE_DELETE,
SDB_TYPE_UPDATE,
SDB_TYPE_DECODE,
SDB_TYPE_ENCODE,
SDB_TYPE_BEFORE_BATCH_UPDATE,
SDB_TYPE_BATCH_UPDATE,
SDB_TYPE_AFTER_BATCH_UPDATE,
SDB_TYPE_RESET,
SDB_TYPE_DESTROY,
SDB_MAX_ACTION_TYPES
};
#define SDB_MAX_PEERS 4
typedef struct {
uint32_t ip;
uint32_t publicIp;
char ipstr[20];
char zone[12];
char role;
int64_t createdTime;
uint64_t dbVersion;
int64_t lostTime;
char status;
char numOfMnodes;
int numOfDnodes;
char updateEnd[1];
// internal
int syncFd;
void *hbTimer;
void *pSync;
} SSdbPeer;
extern SSdbPeer *sdbPeer[];
#define sdbInited (sdbPeer[0])
#define sdbStatus (sdbPeer[0]->status)
void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory,
void *(*appTool)(char, void *, char *, int, int *));
void *sdbGetRow(void *handle, void *key);
int64_t sdbInsertRow(void *handle, void *row, int rowSize);
int sdbDeleteRow(void *handle, void *key);
int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated);
void *sdbFetchRow(void *handle, void *pNode, void **ppRow);
int sdbBatchUpdateRow(void *handle, void *row, int rowSize);
int64_t sdbGetId(void *handle);
int64_t sdbGetNumOfRows(void *handle);
void sdbSaveSnapShot(void *handle);
void sdbCloseTable(void *handle);
int sdbRemovePeerByIp(uint32_t ip);
int sdbInitPeers(char *directory);
void sdbCleanUpPeers();
int64_t sdbGetVersion();
int32_t sdbGetRunStatus();
#define TSDB_MAX_NORMAL_TABLES 10000
#define TSDB_MAX_SUPER_TABLES 1000
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_SDB_H

12
src/inc/taosdef.h
View File

@ -176,7 +176,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MAX_COLUMNS 256
#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns
#define TSDB_DNODE_NAME_LEN 63
#define TSDB_DNODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 192
#define TSDB_DB_NAME_LEN 32
#define TSDB_COL_NAME_LEN 64
@ -309,6 +309,16 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_SESSIONS_PER_VNODE (300)
#define TSDB_SESSIONS_PER_DNODE (TSDB_SESSIONS_PER_VNODE * TSDB_MAX_VNODES)
#define TSDB_MAX_MNODES 5
#define TSDB_MAX_DNODES 10
#define TSDB_MAX_ACCOUNTS 10
#define TSDB_MAX_USERS 20
#define TSDB_MAX_DBS 100
#define TSDB_MAX_VGROUPS 1000
#define TSDB_MAX_SUPER_TABLES 100
#define TSDB_MAX_NORMAL_TABLES 1000
#define TSDB_MAX_CHILD_TABLES 100000
enum {
TSDB_PRECISION_MILLI,
TSDB_PRECISION_MICRO,

476
src/inc/taosmsg.h
View File

@ -20,122 +20,122 @@
extern "C" {
#endif
#include <stdint.h>
#include <stdbool.h>
#include <stdint.h>
#include "taosdef.h"
#include "taoserror.h"
#include "trpc.h"
// message type
#define TSDB_MSG_TYPE_REG 1
#define TSDB_MSG_TYPE_REG_RSP 2
#define TSDB_MSG_TYPE_SUBMIT 3
#define TSDB_MSG_TYPE_SUBMIT_RSP 4
#define TSDB_MSG_TYPE_QUERY 5
#define TSDB_MSG_TYPE_QUERY_RSP 6
#define TSDB_MSG_TYPE_RETRIEVE 7
#define TSDB_MSG_TYPE_RETRIEVE_RSP 8
#define TSDB_MSG_TYPE_REG 1
#define TSDB_MSG_TYPE_REG_RSP 2
#define TSDB_MSG_TYPE_SUBMIT 3
#define TSDB_MSG_TYPE_SUBMIT_RSP 4
#define TSDB_MSG_TYPE_QUERY 5
#define TSDB_MSG_TYPE_QUERY_RSP 6
#define TSDB_MSG_TYPE_RETRIEVE 7
#define TSDB_MSG_TYPE_RETRIEVE_RSP 8
// message from mnode to dnode
#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9
#define TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP 10
#define TSDB_MSG_TYPE_MD_DROP_TABLE 11
#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12
#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13
#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14
#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15
#define TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP 16
#define TSDB_MSG_TYPE_MD_DROP_VNODE 17
#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18
#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19
#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20
#define TSDB_MSG_TYPE_MD_DROP_STABLE 21
#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22
#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23
#define TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP 24
#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25
#define TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP 26
#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9
#define TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP 10
#define TSDB_MSG_TYPE_MD_DROP_TABLE 11
#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12
#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13
#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14
#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15
#define TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP 16
#define TSDB_MSG_TYPE_MD_DROP_VNODE 17
#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18
#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19
#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20
#define TSDB_MSG_TYPE_MD_DROP_STABLE 21
#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22
#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23
#define TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP 24
#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25
#define TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP 26
// message from client to mnode
#define TSDB_MSG_TYPE_CM_CONNECT 31
#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32
#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33
#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34
#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35
#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36
#define TSDB_MSG_TYPE_CM_DROP_ACCT 37
#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38
#define TSDB_MSG_TYPE_CM_CREATE_USER 39
#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40
#define TSDB_MSG_TYPE_CM_ALTER_USER 41
#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42
#define TSDB_MSG_TYPE_CM_DROP_USER 43
#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44
#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45
#define TSDB_MSG_TYPE_CM_CREATE_DNODE_RSP 46
#define TSDB_MSG_TYPE_CM_DROP_DNODE 47
#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48
#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE
#define TSDB_MSG_TYPE_CM_CONFIG_DNODE_RSP TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP
#define TSDB_MSG_TYPE_CM_CREATE_DB 49
#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50
#define TSDB_MSG_TYPE_CM_DROP_DB 51
#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52
#define TSDB_MSG_TYPE_CM_USE_DB 53
#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54
#define TSDB_MSG_TYPE_CM_ALTER_DB 55
#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56
#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57
#define TSDB_MSG_TYPE_CM_CREATE_TABLE_RSP 58
#define TSDB_MSG_TYPE_CM_DROP_TABLE 59
#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60
#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61
#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62
#define TSDB_MSG_TYPE_CM_TABLE_META 63
#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64
#define TSDB_MSG_TYPE_CM_STABLE_META 65
#define TSDB_MSG_TYPE_CM_STABLE_META_RSP 66
#define TSDB_MSG_TYPE_CM_TABLES_META 67
#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68
#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69
#define TSDB_MSG_TYPE_CM_ALTER_STREAM_RSP 70
#define TSDB_MSG_TYPE_CM_SHOW 71
#define TSDB_MSG_TYPE_CM_SHOW_RSP 72
#define TSDB_MSG_TYPE_CM_KILL_QUERY 73
#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74
#define TSDB_MSG_TYPE_CM_KILL_STREAM 75
#define TSDB_MSG_TYPE_CM_KILL_STREAM_RSP 76
#define TSDB_MSG_TYPE_CM_KILL_CONN 77
#define TSDB_MSG_TYPE_CM_KILL_CONN_RSP 78
#define TSDB_MSG_TYPE_CM_HEARTBEAT 79
#define TSDB_MSG_TYPE_CM_HEARTBEAT_RSP 80
#define TSDB_MSG_TYPE_CM_CONNECT 31
#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32
#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33
#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34
#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35
#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36
#define TSDB_MSG_TYPE_CM_DROP_ACCT 37
#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38
#define TSDB_MSG_TYPE_CM_CREATE_USER 39
#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40
#define TSDB_MSG_TYPE_CM_ALTER_USER 41
#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42
#define TSDB_MSG_TYPE_CM_DROP_USER 43
#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44
#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45
#define TSDB_MSG_TYPE_CM_CREATE_DNODE_RSP 46
#define TSDB_MSG_TYPE_CM_DROP_DNODE 47
#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48
#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE
#define TSDB_MSG_TYPE_CM_CONFIG_DNODE_RSP TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP
#define TSDB_MSG_TYPE_CM_CREATE_DB 49
#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50
#define TSDB_MSG_TYPE_CM_DROP_DB 51
#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52
#define TSDB_MSG_TYPE_CM_USE_DB 53
#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54
#define TSDB_MSG_TYPE_CM_ALTER_DB 55
#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56
#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57
#define TSDB_MSG_TYPE_CM_CREATE_TABLE_RSP 58
#define TSDB_MSG_TYPE_CM_DROP_TABLE 59
#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60
#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61
#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62
#define TSDB_MSG_TYPE_CM_TABLE_META 63
#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64
#define TSDB_MSG_TYPE_CM_STABLE_VGROUP 65
#define TSDB_MSG_TYPE_CM_STABLE_VGROUP_RSP 66
#define TSDB_MSG_TYPE_CM_TABLES_META 67
#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68
#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69
#define TSDB_MSG_TYPE_CM_ALTER_STREAM_RSP 70
#define TSDB_MSG_TYPE_CM_SHOW 71
#define TSDB_MSG_TYPE_CM_SHOW_RSP 72
#define TSDB_MSG_TYPE_CM_KILL_QUERY 73
#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74
#define TSDB_MSG_TYPE_CM_KILL_STREAM 75
#define TSDB_MSG_TYPE_CM_KILL_STREAM_RSP 76
#define TSDB_MSG_TYPE_CM_KILL_CONN 77
#define TSDB_MSG_TYPE_CM_KILL_CONN_RSP 78
#define TSDB_MSG_TYPE_CM_HEARTBEAT 79
#define TSDB_MSG_TYPE_CM_HEARTBEAT_RSP 80
// message from dnode to mnode
#define TSDB_MSG_TYPE_DM_CONFIG_TABLE 91
#define TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP 92
#define TSDB_MSG_TYPE_DM_CONFIG_VNODE 93
#define TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP 94
#define TSDB_MSG_TYPE_DM_STATUS 95
#define TSDB_MSG_TYPE_DM_STATUS_RSP 96
#define TSDB_MSG_TYPE_DM_GRANT 97
#define TSDB_MSG_TYPE_DM_GRANT_RSP 98
#define TSDB_MSG_TYPE_DM_CONFIG_TABLE 91
#define TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP 92
#define TSDB_MSG_TYPE_DM_CONFIG_VNODE 93
#define TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP 94
#define TSDB_MSG_TYPE_DM_STATUS 95
#define TSDB_MSG_TYPE_DM_STATUS_RSP 96
#define TSDB_MSG_TYPE_DM_GRANT 97
#define TSDB_MSG_TYPE_DM_GRANT_RSP 98
#define TSDB_MSG_TYPE_SDB_SYNC 101
#define TSDB_MSG_TYPE_SDB_SYNC_RSP 102
#define TSDB_MSG_TYPE_SDB_FORWARD 103
#define TSDB_MSG_TYPE_SDB_FORWARD_RSP 104
#define TSDB_MSG_TYPE_SDB_SYNC 101
#define TSDB_MSG_TYPE_SDB_SYNC_RSP 102
#define TSDB_MSG_TYPE_SDB_FORWARD 103
#define TSDB_MSG_TYPE_SDB_FORWARD_RSP 104
#define TSDB_MSG_TYPE_MAX 105
#define TSDB_MSG_TYPE_MAX 105
// IE type
#define TSDB_IE_TYPE_SEC 1
#define TSDB_IE_TYPE_META 2
#define TSDB_IE_TYPE_MGMT_IP 3
#define TSDB_IE_TYPE_DNODE_CFG 4
#define TSDB_IE_TYPE_NEW_VERSION 5
#define TSDB_IE_TYPE_DNODE_EXT 6
#define TSDB_IE_TYPE_DNODE_STATE 7
#define TSDB_IE_TYPE_SEC 1
#define TSDB_IE_TYPE_META 2
#define TSDB_IE_TYPE_MGMT_IP 3
#define TSDB_IE_TYPE_DNODE_CFG 4
#define TSDB_IE_TYPE_NEW_VERSION 5
#define TSDB_IE_TYPE_DNODE_EXT 6
#define TSDB_IE_TYPE_DNODE_STATE 7
enum _mgmt_table {
TSDB_MGMT_TABLE_ACCT,
@ -157,46 +157,37 @@ enum _mgmt_table {
TSDB_MGMT_TABLE_MAX,
};
#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1
#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2
#define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3
#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1
#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2
#define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3
#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
#define TSDB_ALTER_TABLE_ADD_COLUMN 5
#define TSDB_ALTER_TABLE_DROP_COLUMN 6
#define TSDB_ALTER_TABLE_ADD_COLUMN 5
#define TSDB_ALTER_TABLE_DROP_COLUMN 6
#define TSDB_INTERPO_NONE 0
#define TSDB_INTERPO_NULL 1
#define TSDB_INTERPO_SET_VALUE 2
#define TSDB_INTERPO_LINEAR 3
#define TSDB_INTERPO_PREV 4
#define TSDB_INTERPO_NONE 0
#define TSDB_INTERPO_NULL 1
#define TSDB_INTERPO_SET_VALUE 2
#define TSDB_INTERPO_LINEAR 3
#define TSDB_INTERPO_PREV 4
#define TSDB_ALTER_USER_PASSWD 0x1
#define TSDB_ALTER_USER_PRIVILEGES 0x2
#define TSDB_ALTER_USER_PASSWD 0x1
#define TSDB_ALTER_USER_PRIVILEGES 0x2
#define TSDB_KILL_MSG_LEN 30
#define TSDB_KILL_MSG_LEN 30
#define TSDB_VN_READ_ACCCESS ((char)0x1)
#define TSDB_VN_READ_ACCCESS ((char)0x1)
#define TSDB_VN_WRITE_ACCCESS ((char)0x2)
#define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS)
#define TSDB_COL_NORMAL 0x0u
#define TSDB_COL_TAG 0x1u
#define TSDB_COL_JOIN 0x2u
#define TSDB_COL_NORMAL 0x0u
#define TSDB_COL_TAG 0x1u
#define TSDB_COL_JOIN 0x2u
extern char *taosMsg[];
#pragma pack(push, 1)
typedef struct {
int32_t vnode;
int32_t sid;
int32_t sversion;
uint64_t uid;
int16_t numOfRows;
char payLoad[];
} SShellSubmitBlock;
typedef struct {
int32_t numOfVnodes;
} SMsgDesc;
@ -206,27 +197,39 @@ typedef struct SMsgHead {
int32_t vgId;
} SMsgHead;
typedef struct {
SMsgDesc desc;
SMsgHead header;
int16_t import;
int32_t numOfTables; // total number of sid
char blks[]; // number of data blocks, each table has at least one data block
} SShellSubmitMsg;
// Submit message for one table
typedef struct SSubmitBlk {
int64_t uid; // table unique id
int32_t tid; // table id
int32_t padding; // TODO just for padding here
int32_t sversion; // data schema version
int32_t len; // data part length, not including the SSubmitBlk head
int16_t numOfRows; // total number of rows in current submit block
char data[];
} SSubmitBlk;
// Submit message for this TSDB
typedef struct SSubmitMsg {
SMsgHead header;
int32_t length;
int32_t compressed : 2;
int32_t numOfBlocks : 30;
SSubmitBlk blocks[];
} SSubmitMsg;
typedef struct {
int32_t index; // index of failed block in submit blocks
int32_t vnode; // vnode index of failed block
int32_t sid; // table index of failed block
int32_t code; // errorcode while write data to vnode, such as not created, dropped, no space, invalid table
int32_t index; // index of failed block in submit blocks
int32_t vnode; // vnode index of failed block
int32_t sid; // table index of failed block
int32_t code; // errorcode while write data to vnode, such as not created, dropped, no space, invalid table
} SShellSubmitRspBlock;
typedef struct {
int32_t code; // 0-success, > 0 error code
int32_t numOfRows; // number of records the client is trying to write
int32_t affectedRows; // number of records actually written
int32_t failedRows; // number of failed records (exclude duplicate records)
int32_t numOfFailedBlocks;
int32_t code; // 0-success, > 0 error code
int32_t numOfRows; // number of records the client is trying to write
int32_t affectedRows; // number of records actually written
int32_t failedRows; // number of failed records (exclude duplicate records)
int32_t numOfFailedBlocks;
SShellSubmitRspBlock failedBlocks[];
} SShellSubmitRspMsg;
@ -238,38 +241,38 @@ typedef struct SSchema {
} SSchema;
typedef struct {
int32_t vnode; //the index of vnode
int32_t vnode; // the index of vnode
uint32_t ip;
} SVnodeDesc;
typedef struct {
int32_t contLen;
int32_t vgId;
int8_t tableType;
int16_t numOfColumns;
int16_t numOfTags;
int32_t sid;
int32_t sversion;
int32_t tagDataLen;
int32_t sqlDataLen;
uint64_t uid;
uint64_t superTableUid;
uint64_t createdTime;
char tableId[TSDB_TABLE_ID_LEN];
char superTableId[TSDB_TABLE_ID_LEN];
char data[];
int32_t contLen;
int32_t vgId;
int8_t tableType;
int16_t numOfColumns;
int16_t numOfTags;
int32_t sid;
int32_t sversion;
int32_t tagDataLen;
int32_t sqlDataLen;
uint64_t uid;
uint64_t superTableUid;
uint64_t createdTime;
char tableId[TSDB_TABLE_ID_LEN];
char superTableId[TSDB_TABLE_ID_LEN];
char data[];
} SMDCreateTableMsg;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN];
char db[TSDB_DB_NAME_LEN];
int8_t igExists;
int16_t numOfTags;
int16_t numOfColumns;
int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string
int32_t contLen;
int8_t reserved[16];
char schema[];
char tableId[TSDB_TABLE_ID_LEN];
char db[TSDB_DB_NAME_LEN];
int8_t igExists;
int16_t numOfTags;
int16_t numOfColumns;
int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string
int32_t contLen;
int8_t reserved[16];
char schema[];
} SCMCreateTableMsg;
typedef struct {
@ -311,7 +314,7 @@ typedef struct {
int64_t maxQueryTime; // In unit of hour
int64_t maxInbound;
int64_t maxOutbound;
int8_t accessState; // Configured only by command
int8_t accessState; // Configured only by command
} SAcctCfg;
typedef struct {
@ -336,11 +339,11 @@ typedef struct {
} SMgmtHead;
typedef struct {
int32_t contLen;
int32_t vgId;
int32_t sid;
uint64_t uid;
char tableId[TSDB_TABLE_ID_LEN + 1];
int32_t contLen;
int32_t vgId;
int32_t sid;
uint64_t uid;
char tableId[TSDB_TABLE_ID_LEN + 1];
} SMDDropTableMsg;
typedef struct {
@ -351,7 +354,7 @@ typedef struct {
} SMDDropSTableMsg;
typedef struct {
int32_t vgId;
int32_t vgId;
} SMDDropVnodeMsg;
typedef struct SColIndexEx {
@ -366,7 +369,7 @@ typedef struct SColIndexEx {
*/
int16_t colIdx;
int16_t colIdxInBuf;
uint16_t flag; // denote if it is a tag or not
uint16_t flag; // denote if it is a tag or not
char name[TSDB_COL_NAME_LEN];
} SColIndexEx;
@ -438,7 +441,7 @@ typedef struct SColumnInfo {
typedef struct STableIdInfo {
int32_t sid;
int64_t uid;
TSKEY key; // last accessed ts, for subscription
TSKEY key; // last accessed ts, for subscription
} STableIdInfo;
typedef struct STimeWindow {
@ -452,47 +455,31 @@ typedef struct STimeWindow {
* the outputCols will be 3 while the numOfCols is 1.
*/
typedef struct {
int32_t contLen; // msg header
int16_t vgId;
int32_t numOfTables;
uint64_t uid;
SMsgHead head;
STimeWindow window;
int32_t numOfTables;
int16_t order;
int16_t orderColId;
int16_t numOfCols; // the number of columns will be load from vnode
int64_t intervalTime; // time interval for aggregation, in million second
int64_t intervalOffset; // start offset for interval query
int64_t slidingTime; // value for sliding window
char slidingTimeUnit; // time interval type, for revisement of interval(1d)
uint16_t tagCondLen; // tag length in current query
uint16_t nameCondLen; // table name in/like query expression string length
int16_t numOfGroupCols; // num of group by columns
int16_t orderByIdx;
int16_t orderType; // used in group by xx order by xxx
uint64_t groupbyTagIds;
int64_t limit;
int64_t offset;
uint16_t queryType; // denote another query process
int16_t numOfOutputCols; // final output columns numbers
int16_t interpoType; // interpolate type
uint64_t defaultVal; // default value array list
int16_t order;
int16_t orderColId;
int16_t numOfCols; // the number of columns will be load from vnode
char slidingTimeUnit; // time interval type, for revisement of interval(1d)
int64_t intervalTime; // time interval for aggregation, in million second
int64_t slidingTime; // value for sliding window
// tag schema, used to parse tag information in pSidExtInfo
uint64_t pTagSchema;
int16_t numOfTagsCols; // required number of tags
int16_t tagLength; // tag length in current query
int16_t numOfGroupCols; // num of group by columns
int16_t orderByIdx;
int16_t orderType; // used in group by xx order by xxx
uint64_t groupbyTagIds;
int64_t limit;
int64_t offset;
int16_t queryType; // denote another query process
int16_t numOfOutputCols; // final output columns numbers
int16_t interpoType; // interpolate type
uint64_t defaultVal; // default value array list
int32_t colNameLen;
int64_t colNameList;
int64_t pSqlFuncExprs;
int32_t colNameLen;
int64_t colNameList;
int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed
int32_t tsLen; // total length of ts comp block
int32_t tsNumOfBlocks; // ts comp block numbers
@ -506,12 +493,14 @@ typedef struct {
} SQueryTableRsp;
typedef struct {
SMsgHead header;
uint64_t qhandle;
uint16_t free;
} SRetrieveTableMsg;
typedef struct {
typedef struct SRetrieveTableRsp {
int32_t numOfRows;
int8_t completed; // all results are returned to client
int16_t precision;
int64_t offset; // updated offset value for multi-vnode projection query
int64_t useconds;
@ -593,11 +582,11 @@ typedef struct {
char dnodeName[TSDB_DNODE_NAME_LEN];
uint32_t privateIp;
uint32_t publicIp;
uint32_t lastReboot; // time stamp for last reboot
uint16_t numOfTotalVnodes; // from config file
uint32_t lastReboot; // time stamp for last reboot
uint16_t numOfTotalVnodes; // from config file
uint16_t openVnodes;
uint16_t numOfCores;
float diskAvailable; // GB
float diskAvailable; // GB
uint8_t alternativeRole;
uint8_t reserve[15];
SVnodeLoad load[];
@ -625,14 +614,14 @@ typedef struct {
char tableIds[];
} SCMMultiTableInfoMsg;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN + 1];
} SCMSuperTableInfoMsg;
typedef struct SCMSTableVgroupMsg {
char tableId[TSDB_TABLE_ID_LEN];
} SCMSTableVgroupMsg;
typedef struct {
int32_t numOfDnodes;
uint32_t dnodeIps[];
} SCMSuperTableInfoRsp;
} SCMSTableVgroupRspMsg;
typedef struct {
int16_t elemLen;
@ -678,25 +667,26 @@ typedef struct {
} SSuperTableMeta;
typedef struct STableMetaMsg {
char tableId[TSDB_TABLE_ID_LEN]; // note: This field must be at the front
int32_t contLen;
uint8_t numOfTags;
uint8_t precision;
uint8_t tableType;
int16_t numOfColumns;
int16_t sversion;
int32_t contLen;
int8_t numOfVpeers;
char tableId[TSDB_TABLE_ID_LEN]; // table id
char stableId[TSDB_TABLE_ID_LEN]; // stable name if it is created according to super table
uint8_t numOfTags;
uint8_t precision;
uint8_t tableType;
int16_t numOfColumns;
int16_t sversion;
int8_t numOfVpeers;
SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT];
int32_t sid;
int32_t vgId;
uint64_t uid;
SSchema schema[];
int32_t sid;
int32_t vgId;
uint64_t uid;
SSchema schema[];
} STableMetaMsg;
typedef struct SMultiTableMeta {
int32_t numOfTables;
int32_t contLen;
int32_t numOfTables;
int32_t contLen;
STableMetaMsg metas[];
} SMultiTableMeta;
@ -733,8 +723,8 @@ typedef struct {
} SDMConfigTableMsg;
typedef struct {
uint32_t dnode;
int32_t vnode;
uint32_t dnodeId;
int32_t vgId;
} SDMConfigVnodeMsg;
typedef struct {
@ -761,13 +751,13 @@ typedef struct {
} SStreamDesc;
typedef struct {
int32_t numOfQueries;
SQueryDesc qdesc[];
int32_t numOfQueries;
SQueryDesc qdesc[];
} SQqueryList;
typedef struct {
int32_t numOfStreams;
SStreamDesc sdesc[];
int32_t numOfStreams;
SStreamDesc sdesc[];
} SStreamList;
typedef struct {

4
src/kit/CMakeLists.txt
View File

@ -2,5 +2,5 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
ADD_SUBDIRECTORY(shell)
ADD_SUBDIRECTORY(taosdemo)
ADD_SUBDIRECTORY(taosdump)
#ADD_SUBDIRECTORY(taosdemo)
#ADD_SUBDIRECTORY(taosdump)

2
src/mnode/CMakeLists.txt
View File

@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(mnode ${SRC})
TARGET_LINK_LIBRARIES(mnode trpc tutil sdb pthread)
TARGET_LINK_LIBRARIES(mnode trpc tutil pthread)
IF (TD_CLUSTER)
TARGET_LINK_LIBRARIES(mnode)

16
src/mnode/inc/mgmtChildTable.h
View File

@ -23,22 +23,18 @@ extern "C" {
#include <stdint.h>
#include <stdbool.h>
#include "taosdef.h"
#include "mnode.h"
int32_t mgmtInitChildTables();
void mgmtCleanUpChildTables();
void * mgmtGetChildTable(char *tableId);
void *mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid);
void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pCreate, SChildTableObj *pTable);
int32_t mgmtDropChildTable(SQueuedMsg *newMsg, SChildTableObj *pTable);
int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName, char *nContent);
int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp);
void mgmtDropAllChildTables(SDbObj *pDropDb);
void mgmtCreateChildTable(SQueuedMsg *pMsg);
void mgmtDropChildTable(SQueuedMsg *pMsg, SChildTableObj *pTable);
void mgmtGetChildTableMeta(SQueuedMsg *pMsg, SChildTableObj *pTable);
void mgmtAlterChildTable(SQueuedMsg *pMsg, SChildTableObj *pTable);
void mgmtDropAllChildTables(SDbObj *pDropDb);
void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable);
#ifdef __cplusplus
}

13
src/mnode/inc/mgmtDb.h
View File

@ -22,22 +22,23 @@ extern "C" {
#include "mnode.h"
int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup);
int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup);
int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup);
int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup);
int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup);
// api
int32_t mgmtInitDbs();
void mgmtCleanUpDbs();
SDbObj *mgmtGetDb(char *db);
SDbObj *mgmtGetDbByTableId(char *db);
bool mgmtCheckIsMonitorDB(char *db, char *monitordb);
// util func
void mgmtAddSuperTableIntoDb(SDbObj *pDb);
void mgmtRemoveSuperTableFromDb(SDbObj *pDb);
void mgmtAddTableIntoDb(SDbObj *pDb);
void mgmtRemoveTableFromDb(SDbObj *pDb);
void mgmtAddVgroupIntoDb(SVgObj *pVgroup);
void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup);
void mgmtRemoveVgroupFromDb(SVgObj *pVgroup);
void mgmtMoveVgroupToTail(SVgObj *pVgroup);
void mgmtMoveVgroupToHead(SVgObj *pVgroup);
#ifdef __cplusplus
}

10
src/mnode/inc/mgmtMnode.h
View File

@ -20,9 +20,15 @@
extern "C" {
#endif
bool mgmtCheckRedirect(void *handle);
int32_t mgmtInitMnodes();
void mgmtCleanupMnodes();
void mgmtGetMnodeIpList(SRpcIpSet *ipSet);
bool mgmtInServerStatus();
bool mgmtIsMaster();
bool mgmtCheckRedirect(void *handle);
void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet);
void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet);
int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp);
int32_t mgmtRemoveMnode(uint32_t privateIp);

46
src/mnode/inc/mgmtNormalTable.h
View File

@ -1,46 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TBASE_MNODE_NORMAL_TABLE_H
#define TBASE_MNODE_NORMAL_TABLE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdbool.h>
#include "mnode.h"
int32_t mgmtInitNormalTables();
void mgmtCleanUpNormalTables();
void * mgmtGetNormalTable(char *tableId);
void * mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid);
void * mgmtBuildCreateNormalTableMsg(SNormalTableObj *pTable);
int32_t mgmtDropNormalTable(SQueuedMsg *newMsg, SNormalTableObj *pTable);
int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int32_t ncols);
int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName);
int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp);
void mgmtDropAllNormalTables(SDbObj *pDropDb);
#ifdef __cplusplus
}
#endif
#endif

8
src/mnode/inc/mgmtProfile.h
View File

@ -24,9 +24,11 @@ extern "C" {
int32_t mgmtInitProfile();
void mgmtCleanUpProfile();
bool mgmtCheckQhandle(uint64_t qhandle);
void mgmtSaveQhandle(void *qhandle);
void mgmtFreeQhandle(void *qhandle);
bool mgmtCheckQhandle(uint64_t qhandle);
void mgmtSaveQhandle(void *qhandle);
void mgmtFreeQhandle(void *qhandle);
void mgmtFreeQueuedMsg(SQueuedMsg *pMsg);
#ifdef __cplusplus
}

72
src/mnode/inc/mgmtSdb.h Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MNODE_SDB_H
#define TDENGINE_MNODE_SDB_H
#ifdef __cplusplus
extern "C" {
#endif
typedef enum {
SDB_KEY_TYPE_STRING,
SDB_KEY_TYPE_AUTO
} ESdbKeyType;
typedef enum {
SDB_OPER_TYPE_GLOBAL,
SDB_OPER_TYPE_LOCAL
} ESdbOperType;
typedef struct {
ESdbOperType type;
void * table;
void * pObj;
int64_t version;
int32_t maxRowSize;
int32_t rowSize;
void * rowData;
} SSdbOperDesc;
typedef struct {
char *tableName;
int32_t hashSessions;
int32_t maxRowSize;
ESdbKeyType keyType;
int32_t (*insertFp)(SSdbOperDesc *pOper);
int32_t (*deleteFp)(SSdbOperDesc *pOper);
int32_t (*updateFp)(SSdbOperDesc *pOper);
int32_t (*encodeFp)(SSdbOperDesc *pOper);
int32_t (*decodeFp)(SSdbOperDesc *pDesc);
int32_t (*destroyFp)(SSdbOperDesc *pDesc);
} SSdbTableDesc;
void * sdbOpenTable(SSdbTableDesc *desc);
void sdbCloseTable(void *handle);
int32_t sdbInsertRow(SSdbOperDesc *pOper);
int32_t sdbDeleteRow(SSdbOperDesc *pOper);
int32_t sdbUpdateRow(SSdbOperDesc *pOper);
void *sdbGetRow(void *handle, void *key);
void *sdbFetchRow(void *handle, void *pNode, void **ppRow);
int64_t sdbGetNumOfRows(void *handle);
uint64_t sdbGetVersion();
#ifdef __cplusplus
}
#endif
#endif

4
src/mnode/inc/mgmtShell.h
View File

@ -22,8 +22,8 @@ extern "C" {
#include "mnode.h"
int32_t mgmtInitShell();
void mgmtCleanUpShell();
void mgmtAddShellMsgHandle(uint8_t msgType, void (*fp)(SQueuedMsg *queuedMsg));
void mgmtCleanUpShell();
void mgmtAddShellMsgHandle(uint8_t msgType, void (*fp)(SQueuedMsg *queuedMsg));
typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn);

21
src/mnode/inc/mgmtSuperTable.h
View File

@ -22,31 +22,20 @@ extern "C" {
#include <stdint.h>
#include <stdbool.h>
#include "taosdef.h"
#include "mnode.h"
int32_t mgmtInitSuperTables();
void mgmtCleanUpSuperTables();
void * mgmtGetSuperTable(char *tableId);
int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate);
int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pTable);
int32_t mgmtAddSuperTableTag(SSuperTableObj *pTable, SSchema schema[], int32_t ntags);
int32_t mgmtDropSuperTableTag(SSuperTableObj *pTable, char *tagName);
int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pTable, char *oldTagName, char *newTagName);
int32_t mgmtAddSuperTableColumn(SSuperTableObj *pTable, SSchema schema[], int32_t ncols);
int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pTable, char *colName);
int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp);
void * mgmtGetSuperTableVgroup(SSuperTableObj *pStable);
int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pTable, const char *tagName);
void mgmtCreateSuperTable(SQueuedMsg *pMsg);
void mgmtDropSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pTable);
void mgmtGetSuperTableMeta(SQueuedMsg *pMsg, SSuperTableObj *pTable);
void mgmtAlterSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pTable);
void mgmtDropAllSuperTables(SDbObj *pDropDb);
int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable);
void mgmtDropAllSuperTables(SDbObj *pDropDb);
#ifdef __cplusplus
}
#endif

9
src/mnode/inc/mgmtTable.h
View File

@ -27,13 +27,8 @@ extern "C" {
int32_t mgmtInitTables();
void mgmtCleanUpTables();
STableInfo* mgmtGetTable(char *tableId);
STableInfo* mgmtGetTableByPos(uint32_t dnodeIp, int32_t vnode, int32_t sid);
int32_t mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMetaMsg *pMeta, bool usePublicIp);
void mgmtAddTableIntoSuperTable(SSuperTableObj *pStable);
void mgmtRemoveTableFromSuperTable(SSuperTableObj *pStable);
STableInfo* mgmtGetTable(char* tableId);
void mgmtExtractTableName(char* tableId, char* tableName);
#ifdef __cplusplus
}

2
src/mnode/inc/mgmtUser.h
View File

@ -24,7 +24,7 @@ extern "C" {
int32_t mgmtInitUsers();
void mgmtCleanUpUsers();
SUserObj *mgmtGetUser(char *name);
SUserObj *mgmtGetUserFromConn(void *pConn);
SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp);
#ifdef __cplusplus
}

11
src/mnode/inc/mgmtVgroup.h
View File

@ -27,18 +27,15 @@ extern "C" {
int32_t mgmtInitVgroups();
void mgmtCleanUpVgroups();
SVgObj *mgmtGetVgroup(int32_t vgId);
SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode);
void mgmtDropAllVgroups(SDbObj *pDropDb);
void mgmtCreateVgroup(SQueuedMsg *pMsg);
void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle);
void mgmtUpdateVgroup(SVgObj *pVgroup);
void mgmtUpdateVgroupIp(SDnodeObj *pDnode);
void mgmtSetVgroupIdPool();
void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle);
SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb);
void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable);
void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable);
void mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle);
void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle);

6
src/mnode/src/mgmtAcct.c
View File

@ -85,8 +85,10 @@ int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) {
pUser->prev->next = pUser->next;
}
if (pUser->next) pUser->next->prev = pUser->prev;
if (pUser->next) {
pUser->next->prev = pUser->prev;
}
if (pUser->prev == NULL) {
pAcct->pUser = pUser->next;
}

1142
src/mnode/src/mgmtChildTable.c
View File

File diff suppressed because it is too large Load Diff

2
src/mnode/src/mgmtDClient.c
View File

@ -42,7 +42,7 @@ int32_t mgmtInitDClient() {
rpcInit.label = "MND-DC";
rpcInit.numOfThreads = 1;
rpcInit.cfp = mgmtProcessRspFromDnode;
rpcInit.sessions = tsMaxDnodes * 5;
rpcInit.sessions = 100;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.user = "mgmtDClient";

2
src/mnode/src/mgmtDServer.c
View File

@ -45,7 +45,7 @@ int32_t mgmtInitDServer() {
rpcInit.label = "MND-DS";
rpcInit.numOfThreads = 1;
rpcInit.cfp = mgmtProcessMsgFromDnode;
rpcInit.sessions = tsMaxDnodes * 5;
rpcInit.sessions = 100;
rpcInit.connType = TAOS_CONN_SERVER;
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.afp = mgmtDServerRetrieveAuth;

512
src/mnode/src/mgmtDb.c
View File

@ -15,101 +15,126 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "mgmtDb.h"
#include "mgmtAcct.h"
#include "mgmtBalance.h"
#include "mgmtChildTable.h"
#include "mgmtDnode.h"
#include "mgmtGrant.h"
#include "mgmtMnode.h"
#include "mgmtNormalTable.h"
#include "mgmtShell.h"
#include "mgmtSuperTable.h"
#include "mgmtTable.h"
#include "mgmtUser.h"
#include "mgmtVgroup.h"
#include "mnode.h"
#include "taoserror.h"
#include "tstatus.h"
#include "tutil.h"
#include "name.h"
#include "mnode.h"
#include "mgmtAcct.h"
#include "mgmtBalance.h"
#include "mgmtChildTable.h"
#include "mgmtDb.h"
#include "mgmtDnode.h"
#include "mgmtGrant.h"
#include "mgmtShell.h"
#include "mgmtMnode.h"
#include "mgmtChildTable.h"
#include "mgmtSdb.h"
#include "mgmtSuperTable.h"
#include "mgmtTable.h"
#include "mgmtUser.h"
#include "mgmtVgroup.h"
static void *tsDbSdb = NULL;
static int32_t tsDbUpdateSize;
static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate);
static void mgmtDropDb(void *handle, void *tmrId);
static void mgmtSetDbDirty(SDbObj *pDb);
static int32_t mgmtSetDbDirty(SDbObj *pDb);
static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg);
static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg);
static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg);
static void *(*mgmtDbActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize);
static void mgmtDbActionInit() {
mgmtDbActionFp[SDB_TYPE_INSERT] = mgmtDbActionInsert;
mgmtDbActionFp[SDB_TYPE_DELETE] = mgmtDbActionDelete;
mgmtDbActionFp[SDB_TYPE_UPDATE] = mgmtDbActionUpdate;
mgmtDbActionFp[SDB_TYPE_ENCODE] = mgmtDbActionEncode;
mgmtDbActionFp[SDB_TYPE_DECODE] = mgmtDbActionDecode;
mgmtDbActionFp[SDB_TYPE_RESET] = mgmtDbActionReset;
mgmtDbActionFp[SDB_TYPE_DESTROY] = mgmtDbActionDestroy;
static int32_t mgmtDbActionDestroy(SSdbOperDesc *pOper) {
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
static void *mgmtDbAction(char action, void *row, char *str, int32_t size, int32_t *ssize) {
if (mgmtDbActionFp[(uint8_t)action] != NULL) {
return (*(mgmtDbActionFp[(uint8_t)action]))(row, str, size, ssize);
static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
SDbObj *pDb = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
pDb->pHead = NULL;
pDb->pTail = NULL;
pDb->prev = NULL;
pDb->next = NULL;
pDb->numOfVgroups = 0;
pDb->numOfTables = 0;
pDb->numOfSuperTables = 0;
if (pAcct != NULL) {
mgmtAddDbIntoAcct(pAcct, pDb);
}
return NULL;
else {
mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct);
return TSDB_CODE_INVALID_ACCT;
}
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) {
SDbObj *pDb = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
mgmtRemoveDbFromAcct(pAcct, pDb);
mgmtDropAllChildTables(pDb);
mgmtDropAllSuperTables(pDb);
mgmtDropAllVgroups(pDb);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) {
SDbObj *pDb = pOper->pObj;
if (pOper->maxRowSize < tsDbUpdateSize) {
return -1;
} else {
memcpy(pOper->rowData, pDb, tsDbUpdateSize);
pOper->rowSize = tsDbUpdateSize;
return TSDB_CODE_SUCCESS;
}
}
static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) {
SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj));
if (pDb == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
memcpy(pDb, pOper->rowData, tsDbUpdateSize);
pOper->pObj = pDb;
return TSDB_CODE_SUCCESS;
}
int32_t mgmtInitDbs() {
void * pNode = NULL;
SDbObj * pDb = NULL;
SAcctObj *pAcct = NULL;
mgmtDbActionInit();
SDbObj tObj;
tsDbUpdateSize = tObj.updateEnd - (char *)&tObj;
tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
tsDbSdb = sdbOpenTable(tsMaxDbs, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtDbAction);
SSdbTableDesc tableDesc = {
.tableName = "dbs",
.hashSessions = TSDB_MAX_DBS,
.maxRowSize = tsDbUpdateSize,
.keyType = SDB_KEY_TYPE_STRING,
.insertFp = mgmtDbActionInsert,
.deleteFp = mgmtDbActionDelete,
.updateFp = mgmtDbActionUpdate,
.encodeFp = mgmtDbActionEncode,
.decodeFp = mgmtDbActionDecode,
.destroyFp = mgmtDbActionDestroy,
};
tsDbSdb = sdbOpenTable(&tableDesc);
if (tsDbSdb == NULL) {
mError("failed to init db data");
return -1;
}
while (1) {
pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb);
if (pDb == NULL) break;
pDb->pHead = NULL;
pDb->pTail = NULL;
pDb->prev = NULL;
pDb->next = NULL;
pDb->numOfTables = 0;
pDb->numOfVgroups = 0;
pDb->numOfSuperTables = 0;
pAcct = mgmtGetAcct(pDb->cfg.acct);
if (pAcct != NULL)
mgmtAddDbIntoAcct(pAcct, pDb);
else {
mError("db:%s acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct);
}
}
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_DB, mgmtProcessCreateDbMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_DB, mgmtProcessAlterDbMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mgmtProcessDropDbMsg);
@ -210,14 +235,14 @@ static int32_t mgmtCheckDBParams(SCMCreateDbMsg *pCreate) {
static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) {
// assign default parameters
if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; //
if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; //
if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; //
if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; //
if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; //
if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; //
if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; //
if (pCreate->compression < 0) pCreate->compression = tsCompression; //
if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode;
if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize;
if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile;
if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep;
if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep;
if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep;
if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime;
if (pCreate->compression < 0) pCreate->compression = tsCompression;
if (pCreate->commitLog < 0) pCreate->commitLog = tsCommitLog;
if (pCreate->replications < 0) pCreate->replications = tsReplications; //
if (pCreate->rowsInFileBlock < 0) pCreate->rowsInFileBlock = tsRowsInFileBlock; //
@ -248,18 +273,10 @@ static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) {
pCreate->blocksPerTable = TSDB_MIN_AVG_BLOCKS;
}
pCreate->maxSessions++;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
int32_t numOfDbs = sdbGetNumOfRows(tsDbSdb);
if (numOfDbs >= tsMaxDbs) {
mWarn("numOfDbs:%d, exceed tsMaxDbs:%d", numOfDbs, tsMaxDbs);
return TSDB_CODE_TOO_MANY_DATABASES;
}
int32_t code = mgmtCheckDbLimit(pAcct);
if (code != 0) {
return code;
@ -287,9 +304,17 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
pDb->createdTime = taosGetTimestampMs();
pDb->cfg = *pCreate;
if (sdbInsertRow(tsDbSdb, pDb, 0) < 0) {
code = TSDB_CODE_SDB_ERROR;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
.rowSize = sizeof(SDbObj)
};
code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
tfree(pDb);
code = TSDB_CODE_SDB_ERROR;
}
return code;
@ -303,73 +328,8 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) {
return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb));
}
static int32_t mgmtAlterDb(SAcctObj *pAcct, SCMAlterDbMsg *pAlter) {
return 0;
// int32_t code = TSDB_CODE_SUCCESS;
//
// SDbObj *pDb = (SDbObj *) sdbGetRow(tsDbSdb, pAlter->db);
// if (pDb == NULL) {
// mTrace("db:%s is not exist", pAlter->db);
// return TSDB_CODE_INVALID_DB;
// }
//
// int32_t oldReplicaNum = pDb->cfg.replications;
// if (pAlter->daysToKeep > 0) {
// mTrace("db:%s daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, pAlter->daysToKeep);
// pDb->cfg.daysToKeep = pAlter->daysToKeep;
// } else if (pAlter->replications > 0) {
// mTrace("db:%s replica:%d change to %d", pDb->name, pDb->cfg.replications, pAlter->replications);
// if (pAlter->replications < TSDB_REPLICA_MIN_NUM || pAlter->replications > TSDB_REPLICA_MAX_NUM) {
// mError("invalid db option replica: %d valid range: %d--%d", pAlter->replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM);
// return TSDB_CODE_INVALID_OPTION;
// }
// pDb->cfg.replications = pAlter->replications;
// } else if (pAlter->maxSessions > 0) {
// mTrace("db:%s tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, pAlter->maxSessions);
// if (pAlter->maxSessions < TSDB_MIN_TABLES_PER_VNODE || pAlter->maxSessions > TSDB_MAX_TABLES_PER_VNODE) {
// mError("invalid db option tables: %d valid range: %d--%d", pAlter->maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE);
// return TSDB_CODE_INVALID_OPTION;
// }
// if (pAlter->maxSessions < pDb->cfg.maxSessions) {
// mError("invalid db option tables: %d should larger than original:%d", pAlter->maxSessions, pDb->cfg.maxSessions);
// return TSDB_CODE_INVALID_OPTION;
// }
// return TSDB_CODE_INVALID_OPTION;
// //The modification of tables needs to rewrite the head file, so disable this option
// //pDb->cfg.maxSessions = pAlter->maxSessions;
// } else {
// mError("db:%s alter msg, replica:%d, keep:%d, tables:%d, origin replica:%d keep:%d", pDb->name,
// pAlter->replications, pAlter->maxSessions, pAlter->daysToKeep,
// pDb->cfg.replications, pDb->cfg.daysToKeep);
// return TSDB_CODE_INVALID_OPTION;
// }
//
// if (sdbUpdateRow(tsDbSdb, pDb, tsDbUpdateSize, 1) < 0) {
// return TSDB_CODE_SDB_ERROR;
// }
//
// SVgObj *pVgroup = pDb->pHead;
// while (pVgroup != NULL) {
// balanceUpdateVgroupState(pVgroup, TSDB_VG_LB_STATUS_UPDATE, 0);
// if (oldReplicaNum < pDb->cfg.replications) {
// if (!balanceAddVnode(pVgroup, NULL, NULL)) {
// mWarn("db:%s vgroup:%d not enough dnode to add vnode", pAlter->db, pVgroup->vgId);
// code = TSDB_CODE_NO_ENOUGH_DNODES;
// }
// }
// if (pAlter->maxSessions > 0) {
// //rebuild meterList in mgmtVgroup.c
// mgmtUpdateVgroup(pVgroup);
// }
//// mgmtSendCreateVnodeMsg(pVgroup);
// pVgroup = pVgroup->next;
// }
// mgmtStartBalanceTimer(10);
//
// return code;
}
int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) {
void mgmtAddVgroupIntoDb(SVgObj *pVgroup) {
SDbObj *pDb = pVgroup->pDb;
pVgroup->next = pDb->pHead;
pVgroup->prev = NULL;
@ -378,11 +338,10 @@ int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) {
pDb->pHead = pVgroup;
pDb->numOfVgroups++;
return 0;
}
int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) {
void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup) {
SDbObj *pDb = pVgroup->pDb;
pVgroup->next = NULL;
pVgroup->prev = pDb->pTail;
@ -391,32 +350,25 @@ int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) {
pDb->pTail = pVgroup;
pDb->numOfVgroups++;
return 0;
}
int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup) {
void mgmtRemoveVgroupFromDb(SVgObj *pVgroup) {
SDbObj *pDb = pVgroup->pDb;
if (pVgroup->prev) pVgroup->prev->next = pVgroup->next;
if (pVgroup->next) pVgroup->next->prev = pVgroup->prev;
if (pVgroup->prev == NULL) pDb->pHead = pVgroup->next;
if (pVgroup->next == NULL) pDb->pTail = pVgroup->prev;
pDb->numOfVgroups--;
return 0;
}
int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup) {
mgmtRemoveVgroupFromDb(pDb, pVgroup);
mgmtAddVgroupIntoDbTail(pDb, pVgroup);
return 0;
void mgmtMoveVgroupToTail(SVgObj *pVgroup) {
mgmtRemoveVgroupFromDb(pVgroup);
mgmtAddVgroupIntoDbTail(pVgroup);
}
int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup) {
mgmtRemoveVgroupFromDb(pDb, pVgroup);
mgmtAddVgroupIntoDb(pDb, pVgroup);
return 0;
void mgmtMoveVgroupToHead(SVgObj *pVgroup) {
mgmtRemoveVgroupFromDb(pVgroup);
mgmtAddVgroupIntoDb(pVgroup);
}
void mgmtCleanUpDbs() {
@ -427,7 +379,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
SUserObj *pUser = mgmtGetUserFromConn(pConn);
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
pShow->bytes[cols] = TSDB_DB_NAME_LEN;
@ -438,7 +390,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "created_time");
strcpy(pSchema[cols].name, "create time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -566,7 +518,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
return 0;
}
char *mgmtGetDbStr(char *src) {
static char *mgmtGetDbStr(char *src) {
char *pos = strstr(src, TS_PATH_DELIMITER);
return ++pos;
@ -577,7 +529,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
SDbObj *pDb = NULL;
char * pWrite;
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn);
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
while (numOfRows < rows) {
@ -636,7 +588,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
if (strcmp(pUser->user, "root") == 0) {
#endif
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.maxSessions - 1; // table num can be created should minus 1
*(int32_t *)pWrite = pDb->cfg.maxSessions; // table num can be created should minus 1
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -649,9 +601,9 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
#ifdef _TD_ARM_32_
*(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1));
*(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions));
#else
*(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1));
*(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions));
#endif
cols++;
@ -691,68 +643,6 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
return numOfRows;
}
void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize) {
SDbObj *pDb = (SDbObj *) row;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
pDb->pHead = NULL;
pDb->pTail = NULL;
pDb->numOfVgroups = 0;
pDb->numOfTables = 0;
mgmtAddDbIntoAcct(pAcct, pDb);
return NULL;
}
void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize) {
SDbObj *pDb = (SDbObj *) row;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
mgmtRemoveDbFromAcct(pAcct, pDb);
mgmtDropAllNormalTables(pDb);
mgmtDropAllChildTables(pDb);
mgmtDropAllSuperTables(pDb);
return NULL;
}
void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) {
return mgmtDbActionReset(row, str, size, ssize);
}
void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize) {
SDbObj *pDb = (SDbObj *) row;
if (size < tsDbUpdateSize) {
*ssize = -1;
} else {
memcpy(str, pDb, tsDbUpdateSize);
*ssize = tsDbUpdateSize;
}
return NULL;
}
void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize) {
SDbObj *pDb = (SDbObj *) malloc(sizeof(SDbObj));
if (pDb == NULL) return NULL;
memset(pDb, 0, sizeof(SDbObj));
memcpy(pDb, str, tsDbUpdateSize);
return (void *)pDb;
}
void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize) {
SDbObj *pDb = (SDbObj *) row;
memcpy(pDb, str, tsDbUpdateSize);
return NULL;
}
void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) {
tfree(row);
return NULL;
}
void mgmtAddSuperTableIntoDb(SDbObj *pDb) {
atomic_add_fetch_32(&pDb->numOfSuperTables, 1);
}
@ -768,8 +658,23 @@ void mgmtRemoveTableFromDb(SDbObj *pDb) {
atomic_add_fetch_32(&pDb->numOfTables, -1);
}
static void mgmtSetDbDirty(SDbObj *pDb) {
static int32_t mgmtSetDbDirty(SDbObj *pDb) {
if (pDb->dirty) return TSDB_CODE_SUCCESS;
pDb->dirty = true;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
.rowSize = tsDbUpdateSize
};
int32_t code = sdbUpdateRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
return TSDB_CODE_SDB_ERROR;
}
return code;
}
static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
@ -794,32 +699,121 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
} else {
code = mgmtCreateDb(pMsg->pUser->pAcct, pCreate);
if (code == TSDB_CODE_SUCCESS) {
mLPrint("DB:%s is created by %s", pCreate->db, pMsg->pUser->user);
mLPrint("db:%s, is created by %s", pCreate->db, pMsg->pUser->user);
}
}
mgmtSendSimpleResp(pMsg->thandle, code);
}
static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
SDbCfg newCfg = pDb->cfg;
int32_t daysToKeep = htonl(pAlter->daysToKeep);
int32_t maxSessions = htonl(pAlter->maxSessions);
int8_t replications = pAlter->replications;
terrno = TSDB_CODE_SUCCESS;
if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) {
mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep);
newCfg.daysToKeep = daysToKeep;
} else if (replications > 0 && replications != pDb->cfg.replications) {
mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications);
if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) {
mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM);
terrno = TSDB_CODE_INVALID_OPTION;
}
newCfg.replications = replications;
} else if (maxSessions > 0 && maxSessions != pDb->cfg.maxSessions) {
mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, maxSessions);
if (maxSessions < TSDB_MIN_TABLES_PER_VNODE || maxSessions > TSDB_MAX_TABLES_PER_VNODE) {
mError("invalid db option tables: %d valid range: %d--%d", maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE);
terrno = TSDB_CODE_INVALID_OPTION;
}
if (maxSessions < pDb->cfg.maxSessions) {
mError("invalid db option tables: %d should larger than original:%d", maxSessions, pDb->cfg.maxSessions);
terrno = TSDB_CODE_INVALID_OPTION;
}
newCfg.maxSessions = maxSessions;
} else {
}
return newCfg;
}
static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
SDbCfg newCfg = mgmtGetAlterDbOption(pDb, pAlter);
if (terrno != TSDB_CODE_SUCCESS) {
return terrno;
}
if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) {
pDb->cfg = newCfg;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
.rowSize = tsDbUpdateSize
};
int32_t code = sdbUpdateRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
return TSDB_CODE_SDB_ERROR;
}
}
return TSDB_CODE_SUCCESS;
}
static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) return;
SCMAlterDbMsg *pAlter = pMsg->pCont;
pAlter->daysPerFile = htonl(pAlter->daysPerFile);
pAlter->daysToKeep = htonl(pAlter->daysToKeep);
pAlter->maxSessions = htonl(pAlter->maxSessions) + 1;
mTrace("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->thandle);
int32_t code;
if (!pMsg->pUser->writeAuth) {
code = TSDB_CODE_NO_RIGHTS;
} else {
code = mgmtAlterDb(pMsg->pUser->pAcct, pAlter);
if (code == TSDB_CODE_SUCCESS) {
mLPrint("DB:%s is altered by %s", pAlter->db, pMsg->pUser->user);
}
if (mgmtCheckExpired()) {
mError("db:%s, failed to alter, grant expired", pAlter->db);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED);
return;
}
mgmtSendSimpleResp(pMsg->thandle, code);
if (!pMsg->pUser->writeAuth) {
mError("db:%s, failed to alter, no rights", pAlter->db);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
return;
}
SDbObj *pDb = mgmtGetDb(pAlter->db);
if (pDb == NULL) {
mError("db:%s, failed to alter, invalid db", pAlter->db);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB);
return;
}
int32_t code = mgmtAlterDb(pDb, pAlter);
if (code != TSDB_CODE_SUCCESS) {
mError("db:%s, failed to alter, invalid db option", pAlter->db);
mgmtSendSimpleResp(pMsg->thandle, code);
}
SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg));
memcpy(newMsg, pMsg, sizeof(SQueuedMsg));
pMsg->pCont = NULL;
SVgObj *pVgroup = pDb->pHead;
if (pVgroup != NULL) {
mPrint("vgroup:%d, will be altered", pVgroup->vgId);
newMsg->ahandle = pVgroup;
newMsg->expected = pVgroup->numOfVnodes;
mgmtAlterVgroup(pVgroup, newMsg);
return;
}
mTrace("db:%s, all vgroups is altered", pDb->name);
mgmtSendSimpleResp(newMsg->thandle, TSDB_CODE_SUCCESS);
rpcFreeCont(newMsg->pCont);
free(newMsg);
}
static void mgmtDropDb(void *handle, void *tmrId) {
@ -827,7 +821,12 @@ static void mgmtDropDb(void *handle, void *tmrId) {
SDbObj *pDb = newMsg->ahandle;
mPrint("db:%s, drop db from sdb", pDb->name);
int32_t code = sdbDeleteRow(tsDbSdb, pDb);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsDbSdb,
.pObj = pDb
};
int32_t code = sdbDeleteRow(&oper);
if (code != 0) {
code = TSDB_CODE_SDB_ERROR;
}
@ -874,7 +873,12 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
return;
}
mgmtSetDbDirty(pDb);
int32_t code = mgmtSetDbDirty(pDb);
if (code != TSDB_CODE_SUCCESS) {
mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code));
mgmtSendSimpleResp(pMsg->thandle, code);
return;
}
SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg));
memcpy(newMsg, pMsg, sizeof(SQueuedMsg));

16
src/mnode/src/mgmtDnode.c
View File

@ -76,7 +76,7 @@ bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) {
int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn);
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS;
@ -169,7 +169,7 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo
static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn);
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS;
@ -256,7 +256,7 @@ static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, vo
static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn);
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS;
@ -547,9 +547,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
return ;
}
uint32_t lastPrivateIp = pDnode->privateIp;
uint32_t lastPublicIp = pDnode->publicIp;
pDnode->privateIp = htonl(pStatus->privateIp);
pDnode->publicIp = htonl(pStatus->publicIp);
pDnode->lastReboot = htonl(pStatus->lastReboot);
@ -566,11 +563,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
mgmtSetDnodeMaxVnodes(pDnode);
}
if (lastPrivateIp != pDnode->privateIp || lastPublicIp != pDnode->publicIp) {
mgmtUpdateVgroupIp(pDnode);
//mgmtUpdateMnodeIp();
}
int32_t openVnodes = htons(pStatus->openVnodes);
for (int32_t j = 0; j < openVnodes; ++j) {
pDnode->vload[j].vgId = htonl(pStatus->load[j].vgId);
@ -599,7 +591,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
return;
}
mgmtGetMnodeIpList(&pRsp->ipList);
mgmtGetMnodePrivateIpList(&pRsp->ipList);
pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId);
pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus);

12
src/mnode/src/mgmtMain.c
View File

@ -25,6 +25,8 @@
#include "mgmtDClient.h"
#include "mgmtDnode.h"
#include "mgmtDServer.h"
#include "mgmtMnode.h"
#include "mgmtSdb.h"
#include "mgmtVgroup.h"
#include "mgmtUser.h"
#include "mgmtTable.h"
@ -65,7 +67,7 @@ int32_t mgmtStartSystem() {
return 0;
}
tsMgmtTmr = taosTmrInit((tsMaxDnodes + tsMaxShellConns) * 3, 200, 3600000, "MND");
tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
if (tsMgmtTmr == NULL) {
mError("failed to init timer");
return -1;
@ -109,8 +111,8 @@ int32_t mgmtStartSystem() {
return -1;
}
if (sdbInitPeers(tsMnodeDir) < 0) {
mError("failed to init peers");
if (mgmtInitMnodes() < 0) {
mError("failed to init mnodes");
return -1;
}
@ -125,7 +127,7 @@ int32_t mgmtStartSystem() {
void mgmtStopSystem() {
if (sdbMaster) {
if (mgmtIsMaster()) {
mTrace("it is a master mgmt node, it could not be stopped");
return;
}
@ -136,7 +138,7 @@ void mgmtStopSystem() {
void mgmtCleanUpSystem() {
mPrint("starting to clean up mgmt");
sdbCleanUpPeers();
mgmtCleanupMnodes();
mgmtCleanupBalance();
mgmtCleanUpShell();
mgmtCleanupDClient();

84
src/mnode/src/mgmtMnode.c
View File

@ -17,48 +17,93 @@
#include "os.h"
#include "trpc.h"
#include "mgmtMnode.h"
#include "mgmtSdb.h"
#include "mgmtUser.h"
int32_t (*mgmtAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL;
int32_t (*mgmtRemoveMnodeFp)(uint32_t privateIp) = NULL;
int32_t (*mgmtGetMnodesNumFp)() = NULL;
void * (*mgmtGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL;
int32_t (*mpeerAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL;
int32_t (*mpeerRemoveMnodeFp)(uint32_t privateIp) = NULL;
int32_t (*mpeerGetMnodesNumFp)() = NULL;
void * (*mpeerGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL;
int32_t (*mpeerInitMnodesFp)() = NULL;
void (*mpeerCleanUpMnodesFp)() = NULL;
static SMnodeObj tsMnodeObj = {0};
static bool tsMnodeIsMaster = false;
static bool tsMnodeIsServing = false;
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static char *mgmtMnodeStatusStr[] = {
"offline",
"unsynced",
"syncing",
"serving",
"null"
};
static char *mgmtMnodeRoleStr[] = {
"unauthed",
"undecided",
"master",
"slave",
"null"
};
int32_t mgmtInitMnodes() {
if (mpeerInitMnodesFp) {
return (*mpeerInitMnodesFp)();
} else {
tsMnodeIsServing = true;
tsMnodeIsMaster = true;
return 0;
}
}
void mgmtCleanupMnodes() {
if (mpeerCleanUpMnodesFp) {
(*mpeerCleanUpMnodesFp)();
}
}
bool mgmtInServerStatus() {
return tsMnodeIsServing;
}
bool mgmtIsMaster() {
return tsMnodeIsMaster;
}
bool mgmtCheckRedirect(void *handle) {
return false;
}
int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp) {
if (mgmtAddMnodeFp) {
return (*mgmtAddMnodeFp)(privateIp, publicIp);
if (mpeerAddMnodeFp) {
return (*mpeerAddMnodeFp)(privateIp, publicIp);
} else {
return 0;
}
}
int32_t mgmtRemoveMnode(uint32_t privateIp) {
if (mgmtRemoveMnodeFp) {
return (*mgmtRemoveMnodeFp)(privateIp);
if (mpeerRemoveMnodeFp) {
return (*mpeerRemoveMnodeFp)(privateIp);
} else {
return 0;
}
}
static int32_t mgmtGetMnodesNum() {
if (mgmtGetMnodesNumFp) {
return (*mgmtGetMnodesNumFp)();
if (mpeerGetMnodesNumFp) {
return (*mpeerGetMnodesNumFp)();
} else {
return 1;
}
}
static void *mgmtGetNextMnode(SShowObj *pShow, SMnodeObj **pMnode) {
if (mgmtGetNextMnodeFp) {
return (*mgmtGetNextMnodeFp)(pShow, pMnode);
if (mpeerGetNextMnodeFp) {
return (*mpeerGetNextMnodeFp)(pShow, pMnode);
} else {
if (*pMnode == NULL) {
*pMnode = &tsMnodeObj;
@ -73,7 +118,7 @@ static void *mgmtGetNextMnode(SShowObj *pShow, SMnodeObj **pMnode) {
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn);
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS;
@ -148,11 +193,11 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, sdbStatusStr[(uint8_t)pMnode->status]);
strcpy(pWrite, mgmtMnodeStatusStr[pMnode->status]);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, sdbRoleStr[(uint8_t)pMnode->role]);
strcpy(pWrite, mgmtMnodeRoleStr[pMnode->role]);
cols++;
tinet_ntoa(ipstr, pMnode->publicIp);
@ -167,7 +212,14 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
return numOfRows;
}
void mgmtGetMnodeIpList(SRpcIpSet *ipSet) {
void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet) {
ipSet->inUse = 0;
ipSet->port = htons(tsMnodeDnodePort);
ipSet->numOfIps = 1;
ipSet->ip[0] = htonl(inet_addr(tsMasterIp));
}
void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet) {
ipSet->inUse = 0;
ipSet->port = htons(tsMnodeDnodePort);
ipSet->numOfIps = 1;

575
src/mnode/src/mgmtNormalTable.c
View File

@ -1,575 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosmsg.h"
#include "tscompression.h"
#include "tskiplist.h"
#include "ttime.h"
#include "tstatus.h"
#include "tutil.h"
#include "mnode.h"
#include "mgmtAcct.h"
#include "mgmtDb.h"
#include "mgmtDClient.h"
#include "mgmtGrant.h"
#include "mgmtNormalTable.h"
#include "mgmtSuperTable.h"
#include "mgmtTable.h"
#include "mgmtVgroup.h"
void *tsNormalTableSdb;
int32_t tsNormalTableUpdateSize;
void *(*mgmtNormalTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize);
void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize);
void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize);
void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize);
void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize);
void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize);
void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize);
void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize);
static void mgmtDestroyNormalTable(SNormalTableObj *pTable) {
free(pTable->schema);
free(pTable->sql);
free(pTable);
}
static void mgmtNormalTableActionInit() {
mgmtNormalTableActionFp[SDB_TYPE_INSERT] = mgmtNormalTableActionInsert;
mgmtNormalTableActionFp[SDB_TYPE_DELETE] = mgmtNormalTableActionDelete;
mgmtNormalTableActionFp[SDB_TYPE_UPDATE] = mgmtNormalTableActionUpdate;
mgmtNormalTableActionFp[SDB_TYPE_ENCODE] = mgmtNormalTableActionEncode;
mgmtNormalTableActionFp[SDB_TYPE_DECODE] = mgmtNormalTableActionDecode;
mgmtNormalTableActionFp[SDB_TYPE_RESET] = mgmtNormalTableActionReset;
mgmtNormalTableActionFp[SDB_TYPE_DESTROY] = mgmtNormalTableActionDestroy;
}
void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) {
SNormalTableObj *pTable = (SNormalTableObj *) row;
memcpy(pTable, str, tsNormalTableUpdateSize);
int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns) + pTable->sqlLen;
pTable->schema = realloc(pTable->schema, schemaSize);
pTable->sql = (char*)pTable->schema + sizeof(SSchema) * (pTable->numOfColumns);
memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize);
return NULL;
}
void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) {
SNormalTableObj *pTable = (SNormalTableObj *)row;
mgmtDestroyNormalTable(pTable);
return NULL;
}
void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) {
SNormalTableObj *pTable = (SNormalTableObj *) row;
SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
mError("id:%s not in vgroup:%d", pTable->tableId, pTable->vgId);
return NULL;
}
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
if (pDb == NULL) {
mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName);
return NULL;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("account not exists");
return NULL;
}
if (!sdbMaster) {
int32_t sid = taosAllocateId(pVgroup->idPool);
if (sid != pTable->sid) {
mError("sid:%d is not matched from the master:%d", sid, pTable->sid);
return NULL;
}
}
mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1);
mgmtAddTableIntoDb(pDb);
mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable);
if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) {
mgmtMoveVgroupToTail(pDb, pVgroup);
}
return NULL;
}
void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) {
SNormalTableObj *pTable = (SNormalTableObj *) row;
if (pTable->vgId == 0) {
return NULL;
}
SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
return NULL;
}
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
if (pDb == NULL) {
mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName);
return NULL;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("account not exists");
return NULL;
}
mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1);
mgmtRemoveTableFromDb(pDb);
mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable);
if (pVgroup->numOfTables > 0) {
mgmtMoveVgroupToHead(pDb, pVgroup);
}
return NULL;
}
void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) {
return mgmtNormalTableActionReset(row, str, size, NULL);
}
void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) {
SNormalTableObj *pTable = (SNormalTableObj *) row;
assert(row != NULL && str != NULL);
int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema);
if (size < tsNormalTableUpdateSize + schemaSize + 1) {
*ssize = -1;
return NULL;
}
memcpy(str, pTable, tsNormalTableUpdateSize);
memcpy(str + tsNormalTableUpdateSize, pTable->schema, schemaSize);
memcpy(str + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen);
*ssize = tsNormalTableUpdateSize + schemaSize + pTable->sqlLen;
return NULL;
}
void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) {
assert(str != NULL);
SNormalTableObj *pTable = (SNormalTableObj *)malloc(sizeof(SNormalTableObj));
if (pTable == NULL) {
return NULL;
}
memset(pTable, 0, sizeof(SNormalTableObj));
if (size < tsNormalTableUpdateSize) {
mgmtDestroyNormalTable(pTable);
return NULL;
}
memcpy(pTable, str, tsNormalTableUpdateSize);
int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema);
pTable->schema = (SSchema *)malloc(schemaSize);
if (pTable->schema == NULL) {
mgmtDestroyNormalTable(pTable);
return NULL;
}
memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize);
pTable->sql = (char *)malloc(pTable->sqlLen);
if (pTable->sql == NULL) {
mgmtDestroyNormalTable(pTable);
return NULL;
}
memcpy(pTable->sql, str + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen);
return (void *)pTable;
}
void *mgmtNormalTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) {
if (mgmtNormalTableActionFp[(uint8_t)action] != NULL) {
return (*(mgmtNormalTableActionFp[(uint8_t)action]))(row, str, size, ssize);
}
return NULL;
}
int32_t mgmtInitNormalTables() {
void *pNode = NULL;
void *pLastNode = NULL;
SNormalTableObj *pTable = NULL;
mgmtNormalTableActionInit();
SNormalTableObj tObj;
tsNormalTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj;
tsNormalTableSdb = sdbOpenTable(tsMaxTables, sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS,
"ntables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtNormalTableAction);
if (tsNormalTableSdb == NULL) {
mError("failed to init ntables data");
return -1;
}
while (1) {
pLastNode = pNode;
pNode = sdbFetchRow(tsNormalTableSdb, pNode, (void **)&pTable);
if (pTable == NULL) break;
SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId);
if (pDb == NULL) {
mError("ntable:%s, failed to get db, discard it", pTable->tableId);
sdbDeleteRow(tsNormalTableSdb, pTable);
pNode = pLastNode;
continue;
}
SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
mError("ntable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid);
pTable->vgId = 0;
sdbDeleteRow(tsNormalTableSdb, pTable);
pNode = pLastNode;
continue;
}
if (strcmp(pVgroup->dbName, pDb->name) != 0) {
mError("ntable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it",
pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid);
pTable->vgId = 0;
sdbDeleteRow(tsNormalTableSdb, pTable);
pNode = pLastNode;
continue;
}
if (pVgroup->tableList == NULL) {
mError("ntable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId);
pTable->vgId = 0;
sdbDeleteRow(tsNormalTableSdb, pTable);
pNode = pLastNode;
continue;
}
mgmtAddTableIntoVgroup(pVgroup, (STableInfo *)pTable);
//pVgroup->tableList[pTable->sid] = (STableInfo*)pTable;
taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid, 1);
pTable->sql = (char *)pTable->schema + sizeof(SSchema) * pTable->numOfColumns;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1);
}
mTrace("ntables is initialized");
return 0;
}
void mgmtCleanUpNormalTables() {
sdbCloseTable(tsNormalTableSdb);
}
void *mgmtBuildCreateNormalTableMsg(SNormalTableObj *pTable) {
int32_t totalCols = pTable->numOfColumns;
int32_t contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + pTable->sqlLen;
SMDCreateTableMsg *pCreate = rpcMallocCont(contLen);
if (pCreate == NULL) {
terrno = TSDB_CODE_SERV_OUT_OF_MEMORY;
return NULL;
}
memcpy(pCreate->tableId, pTable->tableId, TSDB_TABLE_ID_LEN + 1);
pCreate->contLen = htonl(contLen);
pCreate->vgId = htonl(pTable->vgId);
pCreate->tableType = pTable->type;
pCreate->numOfColumns = htons(pTable->numOfColumns);
pCreate->numOfTags = 0;
pCreate->sid = htonl(pTable->sid);
pCreate->sversion = htonl(pTable->sversion);
pCreate->tagDataLen = 0;
pCreate->sqlDataLen = htonl(pTable->sqlLen);
pCreate->uid = htobe64(pTable->uid);
pCreate->superTableUid = 0;
pCreate->createdTime = htobe64(pTable->createdTime);
SSchema *pSchema = (SSchema *) pCreate->data;
memcpy(pSchema, pTable->schema, totalCols * sizeof(SSchema));
for (int32_t col = 0; col < totalCols; ++col) {
pSchema->bytes = htons(pSchema->bytes);
pSchema->colId = htons(pSchema->colId);
pSchema++;
}
memcpy(pCreate + sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema), pTable->sql, pTable->sqlLen);
return pCreate;
}
void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid) {
int32_t numOfTables = sdbGetNumOfRows(tsNormalTableSdb);
if (numOfTables >= TSDB_MAX_NORMAL_TABLES) {
mError("table:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_NORMAL_TABLES);
terrno = TSDB_CODE_TOO_MANY_TABLES;
return NULL;
}
SNormalTableObj *pTable = (SNormalTableObj *) calloc(sizeof(SNormalTableObj), 1);
if (pTable == NULL) {
mError("table:%s, failed to alloc memory", pCreate->tableId);
terrno = TSDB_CODE_SERV_OUT_OF_MEMORY;
return NULL;
}
strcpy(pTable->tableId, pCreate->tableId);
pTable->type = TSDB_NORMAL_TABLE;
pTable->vgId = pVgroup->vgId;
pTable->createdTime = taosGetTimestampMs();
pTable->uid = (((uint64_t) pTable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul));
pTable->sid = sid;
pTable->sversion = 0;
pTable->numOfColumns = htons(pCreate->numOfColumns);
pTable->sqlLen = htons(pCreate->sqlLen);
int32_t numOfCols = pTable->numOfColumns;
int32_t schemaSize = numOfCols * sizeof(SSchema);
pTable->schema = (SSchema *) calloc(1, schemaSize);
if (pTable->schema == NULL) {
free(pTable);
terrno = TSDB_CODE_SERV_OUT_OF_MEMORY;
return NULL;
}
memcpy(pTable->schema, pCreate->schema, numOfCols * sizeof(SSchema));
pTable->nextColId = 0;
for (int32_t col = 0; col < numOfCols; col++) {
SSchema *tschema = pTable->schema;
tschema[col].colId = pTable->nextColId++;
tschema[col].bytes = htons(tschema[col].bytes);
}
if (pTable->sqlLen != 0) {
pTable->type = TSDB_STREAM_TABLE;
pTable->sql = calloc(1, pTable->sqlLen);
if (pTable->sql == NULL) {
free(pTable);
terrno = TSDB_CODE_SERV_OUT_OF_MEMORY;
return NULL;
}
memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen);
pTable->sql[pTable->sqlLen - 1] = 0;
mTrace("table:%s, stream sql len:%d sql:%s", pTable->tableId, pTable->sqlLen, pTable->sql);
}
if (sdbInsertRow(tsNormalTableSdb, pTable, 0) < 0) {
mError("table:%s, update sdb error", pTable->tableId);
free(pTable);
terrno = TSDB_CODE_SDB_ERROR;
return NULL;
}
mTrace("table:%s, create ntable in vgroup, uid:%" PRIu64 , pTable->tableId, pTable->uid);
return pTable;
}
int32_t mgmtDropNormalTable(SQueuedMsg *newMsg, SNormalTableObj *pTable) {
SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
mError("table:%s, failed to drop normal table, vgroup not exist", pTable->tableId);
return TSDB_CODE_OTHERS;
}
SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg));
if (pDrop == NULL) {
mError("table:%s, failed to drop normal table, no enough memory", pTable->tableId);
return TSDB_CODE_SERV_OUT_OF_MEMORY;
}
strcpy(pDrop->tableId, pTable->tableId);
pDrop->contLen = htonl(sizeof(SMDDropTableMsg));
pDrop->vgId = htonl(pVgroup->vgId);
pDrop->sid = htonl(pTable->sid);
pDrop->uid = htobe64(pTable->uid);
SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
mTrace("table:%s, send drop table msg", pDrop->tableId);
SRpcMsg rpcMsg = {
.handle = newMsg,
.pCont = pDrop,
.contLen = sizeof(SMDDropTableMsg),
.code = 0,
.msgType = TSDB_MSG_TYPE_MD_DROP_TABLE
};
newMsg->ahandle = pTable;
mgmtSendMsgToDnode(&ipSet, &rpcMsg);
return TSDB_CODE_SUCCESS;
}
void* mgmtGetNormalTable(char *tableId) {
return sdbGetRow(tsNormalTableSdb, tableId);
}
static int32_t mgmtFindNormalTableColumnIndex(SNormalTableObj *pTable, char *colName) {
SSchema *schema = (SSchema *) pTable->schema;
for (int32_t i = 0; i < pTable->numOfColumns; i++) {
if (strcasecmp(schema[i].name, colName) == 0) {
return i;
}
}
return -1;
}
int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int32_t ncols) {
if (ncols <= 0) {
return TSDB_CODE_APP_ERROR;
}
for (int32_t i = 0; i < ncols; i++) {
if (mgmtFindNormalTableColumnIndex(pTable, schema[i].name) > 0) {
return TSDB_CODE_APP_ERROR;
}
}
SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId);
if (pDb == NULL) {
mError("table: %s not belongs to any database", pTable->tableId);
return TSDB_CODE_APP_ERROR;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("DB: %s not belongs to andy account", pDb->name);
return TSDB_CODE_APP_ERROR;
}
int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema);
pTable->schema = realloc(pTable->schema, schemaSize + sizeof(SSchema) * ncols);
memcpy(pTable->schema + schemaSize, schema, sizeof(SSchema) * ncols);
SSchema *tschema = (SSchema *) (pTable->schema + sizeof(SSchema) * pTable->numOfColumns);
for (int32_t i = 0; i < ncols; i++) {
tschema[i].colId = pTable->nextColId++;
}
pTable->numOfColumns += ncols;
pTable->sversion++;
pAcct->acctInfo.numOfTimeSeries += ncols;
sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1);
return TSDB_CODE_SUCCESS;
}
int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName) {
int32_t col = mgmtFindNormalTableColumnIndex(pTable, colName);
if (col < 0) {
return TSDB_CODE_APP_ERROR;
}
SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId);
if (pDb == NULL) {
mError("table: %s not belongs to any database", pTable->tableId);
return TSDB_CODE_APP_ERROR;
}
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
if (pAcct == NULL) {
mError("DB: %s not belongs to any account", pDb->name);
return TSDB_CODE_APP_ERROR;
}
memmove(pTable->schema + sizeof(SSchema) * col, pTable->schema + sizeof(SSchema) * (col + 1),
sizeof(SSchema) * (pTable->numOfColumns - col - 1));
pTable->numOfColumns--;
pTable->sversion++;
pAcct->acctInfo.numOfTimeSeries--;
sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SNormalTableObj *pTable) {
int32_t numOfCols = pTable->numOfColumns;
for (int32_t i = 0; i < numOfCols; ++i) {
strcpy(pSchema->name, pTable->schema[i].name);
pSchema->type = pTable->schema[i].type;
pSchema->bytes = htons(pTable->schema[i].bytes);
pSchema->colId = htons(pTable->schema[i].colId);
pSchema++;
}
return numOfCols * sizeof(SSchema);
}
int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) {
pMeta->uid = htobe64(pTable->uid);
pMeta->sid = htonl(pTable->sid);
pMeta->vgId = htonl(pTable->vgId);
pMeta->sversion = htons(pTable->sversion);
pMeta->precision = pDb->cfg.precision;
pMeta->numOfTags = 0;
pMeta->numOfColumns = htons(pTable->numOfColumns);
pMeta->tableType = pTable->type;
pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromNormalTable(pMeta->schema, pTable);
strncpy(pMeta->tableId, pTable->tableId, tListLen(pTable->tableId));
SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
return TSDB_CODE_INVALID_TABLE;
}
for (int32_t i = 0; i < TSDB_VNODES_SUPPORT; ++i) {
if (usePublicIp) {
pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp;
} else {
pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].privateIp;
}
pMeta->vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode);
}
pMeta->numOfVpeers = pVgroup->numOfVnodes;
return TSDB_CODE_SUCCESS;
}
void mgmtDropAllNormalTables(SDbObj *pDropDb) {
void *pNode = NULL;
void *pLastNode = NULL;
int32_t numOfTables = 0;
int32_t dbNameLen = strlen(pDropDb->name);
SNormalTableObj *pTable = NULL;
while (1) {
pNode = sdbFetchRow(tsNormalTableSdb, pNode, (void **)&pTable);
if (pTable == NULL) break;
if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) {
sdbDeleteRow(tsNormalTableSdb, pTable);
pNode = pLastNode;
numOfTables ++;
continue;
}
}
mTrace("db:%s, all normal tables:%d is dropped", pDropDb->name, numOfTables);
}

16
src/mnode/src/mgmtProfile.c
View File

@ -678,7 +678,7 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
if (mgmtCheckRedirect(pMsg->thandle)) return;
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
if (pUser == NULL) {
rpcRsp.code = TSDB_CODE_INVALID_USER;
rpcSendResponse(&rpcRsp);
@ -702,7 +702,7 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
if (mgmtCheckRedirect(pMsg->thandle)) return;
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
if (pUser == NULL) {
rpcRsp.code = TSDB_CODE_INVALID_USER;
rpcSendResponse(&rpcRsp);
@ -726,7 +726,7 @@ void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
if (mgmtCheckRedirect(pMsg->thandle)) return;
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
if (pUser == NULL) {
rpcRsp.code = TSDB_CODE_INVALID_USER;
rpcSendResponse(&rpcRsp);
@ -762,3 +762,13 @@ int32_t mgmtInitProfile() {
void mgmtCleanUpProfile() {
}
void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) {
if (pMsg != NULL) {
if (pMsg->pCont != NULL) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
free(pMsg);
}
}

767
src/mnode/src/mgmtSdb.c Normal file
View File

@ -0,0 +1,767 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tchecksum.h"
#include "tglobalcfg.h"
#include "tlog.h"
#include "trpc.h"
#include "tutil.h"
#include "hashint.h"
#include "hashstr.h"
#include "mgmtSdb.h"
#define abs(x) (((x) < 0) ? -(x) : (x))
#define SDB_MAX_PEERS 4
#define SDB_DELIMITER 0xFFF00F00
#define SDB_ENDCOMMIT 0xAFFFAAAF
typedef struct {
uint64_t swVersion;
int16_t sdbFileVersion;
char reserved[2];
TSCKSUM checkSum;
} SSdbHeader;
typedef struct _SSdbTable {
SSdbHeader header;
char tableName[TSDB_DB_NAME_LEN];
char fileName[TSDB_FILENAME_LEN];
ESdbKeyType keyType;
int32_t tableId;
int32_t hashSessions;
int32_t maxRowSize;
int32_t autoIndex;
int32_t fd;
int64_t numOfRows;
int64_t version;
int64_t fileSize;
void * iHandle;
int32_t (*insertFp)(SSdbOperDesc *pDesc);
int32_t (*deleteFp)(SSdbOperDesc *pOper);
int32_t (*updateFp)(SSdbOperDesc *pOper);
int32_t (*decodeFp)(SSdbOperDesc *pOper);
int32_t (*encodeFp)(SSdbOperDesc *pOper);
int32_t (*destroyFp)(SSdbOperDesc *pOper);
pthread_mutex_t mutex;
} SSdbTable;
typedef struct {
int64_t version;
int64_t offset;
int32_t rowSize;
void * row;
} SRowMeta;
typedef struct {
int32_t delimiter;
int32_t rowSize;
int64_t version;
char data[];
} SRowHead;
typedef enum {
SDB_FORWARD_TYPE_INSERT,
SDB_FORWARD_TYPE_DELETE,
SDB_FORWARD_TYPE_UPDATE
} ESdbForwardType;
typedef struct {
ESdbForwardType type;
int32_t tableId;
int64_t version;
int32_t rowSize;
void * rowData;
} SForwardMsg;
extern char version[];
const int16_t sdbFileVersion = 2;
int32_t (*mpeerForwardRequestFp)(SForwardMsg *forwardMsg) = NULL;
static SSdbTable *sdbTableList[10] = {0};
static int32_t sdbNumOfTables = 0;
static uint64_t sdbVersion = 0;
static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash};
static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash};
static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash};
static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData};
static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash};
static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData};
uint64_t sdbGetVersion() { return sdbVersion; }
int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->version; }
int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; }
static char *sdbGetkeyStr(SSdbTable *pTable, void *row) {
static char str[16];
switch (pTable->keyType) {
case SDB_KEY_TYPE_STRING:
return (char *)row;
case SDB_KEY_TYPE_AUTO:
sprintf(str, "%d", *(int32_t *)row);
return str;
default:
return "unknown";
}
}
static int32_t sdbForwardDbReqToPeer(SForwardMsg *forwardMsg) {
if (mpeerForwardRequestFp) {
return mpeerForwardRequestFp(forwardMsg);
} else {
return 0;
}
}
static void sdbFinishCommit(SSdbTable *pTable) {
uint32_t sdbEcommit = SDB_ENDCOMMIT;
off_t offset = lseek(pTable->fd, 0, SEEK_END);
assert(offset == pTable->fileSize);
twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit));
pTable->fileSize += sizeof(sdbEcommit);
}
static int32_t sdbOpenSdbFile(SSdbTable *pTable) {
struct stat fstat, ofstat;
uint64_t size;
char * dirc = NULL;
char * basec = NULL;
union {
char cversion[64];
uint64_t iversion;
} swVersion;
memcpy(swVersion.cversion, version, sizeof(uint64_t));
// check sdb.db and .sdb.db status
char fn[TSDB_FILENAME_LEN] = "\0";
dirc = strdup(pTable->fileName);
basec = strdup(pTable->fileName);
sprintf(fn, "%s/.%s", dirname(dirc), basename(basec));
tfree(dirc);
tfree(basec);
if (stat(fn, &ofstat) == 0) { // .sdb.db file exists
if (stat(pTable->fileName, &fstat) == 0) {
remove(fn);
} else {
remove(pTable->fileName);
rename(fn, pTable->fileName);
}
}
pTable->fd = open(pTable->fileName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (pTable->fd < 0) {
sdbError("table:%s, failed to open file:%s", pTable->tableName, pTable->fileName);
return -1;
}
pTable->fileSize = 0;
stat(pTable->fileName, &fstat);
size = sizeof(pTable->header);
if (fstat.st_size == 0) {
pTable->header.swVersion = swVersion.iversion;
pTable->header.sdbFileVersion = sdbFileVersion;
if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) {
sdbError("table:%s, failed to get file header checksum, file:%s", pTable->tableName, pTable->fileName);
tclose(pTable->fd);
return -1;
}
twrite(pTable->fd, &(pTable->header), size);
pTable->fileSize += size;
sdbFinishCommit(pTable);
} else {
uint32_t sdbEcommit = 0;
off_t offset = lseek(pTable->fd, -(sizeof(sdbEcommit)), SEEK_END);
while (offset > 0) {
read(pTable->fd, &sdbEcommit, sizeof(sdbEcommit));
if (sdbEcommit == SDB_ENDCOMMIT) {
ftruncate(pTable->fd, offset + sizeof(sdbEcommit));
break;
}
offset = lseek(pTable->fd, -(sizeof(sdbEcommit) + 1), SEEK_CUR);
}
lseek(pTable->fd, 0, SEEK_SET);
ssize_t tsize = read(pTable->fd, &(pTable->header), size);
if (tsize < size) {
sdbError("table:%s, failed to read sdb file header, file:%s", pTable->tableName, pTable->fileName);
tclose(pTable->fd);
return -1;
}
if (pTable->header.swVersion != swVersion.iversion) {
sdbWarn("table:%s, sdb file:%s version not match software version", pTable->tableName, pTable->fileName);
}
if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) {
sdbError("table:%s, sdb file header is broken since checksum mismatch, file:%s", pTable->tableName, pTable->fileName);
tclose(pTable->fd);
return -1;
}
pTable->fileSize += size;
// skip end commit symbol
lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR);
pTable->fileSize += sizeof(sdbEcommit);
}
pTable->numOfRows = 0;
return pTable->fd;
}
static int32_t sdbInitTableByFile(SSdbTable *pTable) {
sdbTrace("table:%s, open sdb file:%s for read", pTable->tableName, pTable->fileName);
if (sdbOpenSdbFile(pTable) < 0) {
sdbError("table:%s, failed to open sdb file:%s for read", pTable->tableName, pTable->fileName);
return -1;
}
int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM);
SRowHead *rowHead = (SRowHead *)malloc(total_size);
if (rowHead == NULL) {
sdbError("table:%s, failed to allocate row head memory, sdb:%s", pTable->tableName, pTable->tableName);
return -1;
}
int32_t numOfChanged = 0;
int32_t maxAutoIndex = 0;
while (1) {
memset(rowHead, 0, total_size);
int32_t bytes = read(pTable->fd, rowHead, sizeof(SRowHead));
if (bytes < 0) {
sdbError("table:%s, failed to read sdb file:%s", pTable->tableName, pTable->fileName);
tfree(rowHead);
return -1;
}
if (bytes == 0) break;
if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) {
pTable->fileSize++;
lseek(pTable->fd, -(bytes - 1), SEEK_CUR);
continue;
}
if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) {
sdbError("table:%s, error row size in sdb filesize:%d, version:%d rowSize:%d maxRowSize:%d", pTable->tableName,
pTable->fileSize, rowHead->version, rowHead->rowSize, pTable->maxRowSize);
pTable->fileSize += sizeof(SRowHead);
continue;
}
bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM));
if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) {
// TODO: Here may cause pTable->fileSize not end of the file
sdbError("table:%s, failed to read sdb file, version:%d rowSize:%d", pTable->tableName, rowHead->version,
rowHead->rowSize);
break;
}
int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM);
if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) {
sdbError("table:%s, error sdb checksum, version:%d, skip", pTable->tableName, rowHead->version);
pTable->fileSize += real_size;
continue;
}
if (pTable->keyType == SDB_KEY_TYPE_AUTO) {
maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data);
}
pTable->version = MAX(pTable->version, abs(rowHead->version));
void *pMetaRow = sdbGetRow(pTable, rowHead->data);
if (pMetaRow == NULL) {
if (rowHead->version < 0) {
sdbError("table:%s, error sdb negative version:%d, record:%s, skip", pTable->tableName, rowHead->version,
sdbGetkeyStr(pTable, rowHead->data));
} else {
SRowMeta rowMeta;
rowMeta.version = rowHead->version;
rowMeta.offset = pTable->fileSize;
rowMeta.rowSize = rowHead->rowSize;
SSdbOperDesc oper = {
.table = pTable,
.rowData = rowHead->data,
.rowSize = rowHead->rowSize
};
int32_t code = (*pTable->decodeFp)(&oper);
if (code == TSDB_CODE_SUCCESS) {
rowMeta.row = oper.pObj;
(*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta);
pTable->numOfRows++;
sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read new record:%s",
pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data));
} else {
sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s",
pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data));
}
}
} else {
if (rowHead->version < 0) {
SSdbOperDesc oper = {
.table = pTable,
.pObj = pMetaRow
};
(*pTable->destroyFp)(&oper);
(*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data);
pTable->numOfRows--;
sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read deleted record:%s",
pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data));
} else {
SRowMeta rowMeta;
rowMeta.version = rowHead->version;
rowMeta.offset = pTable->fileSize;
rowMeta.rowSize = rowHead->rowSize;
SSdbOperDesc oper = {
.table = pTable,
.rowData = rowHead->data,
.rowSize = rowHead->rowSize,
.pObj = pMetaRow
};
(*pTable->destroyFp)(&oper);
(*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data);
int32_t code = (*pTable->decodeFp)(&oper);
if (code == TSDB_CODE_SUCCESS) {
rowMeta.row = oper.pObj;
(*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta);
sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read updated record:%s",
pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data));
} else {
sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s",
pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data));
}
}
numOfChanged++;
}
pTable->fileSize += real_size;
pTable->fileSize += 4;
lseek(pTable->fd, 4, SEEK_CUR);
}
void *pNode = NULL;
while (1) {
SRowMeta * pMeta;
pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
if (pMeta == NULL) break;
SSdbOperDesc oper = {
.pObj = pMeta->row,
.table = pTable,
.version = pMeta->version,
};
int32_t code = (*pTable->insertFp)(&oper);
if (code != TSDB_CODE_SUCCESS) {
sdbError("table:%s, failed to insert record:%s", pTable->tableName, sdbGetkeyStr(pTable, rowHead->data));
}
}
sdbVersion += pTable->version;
if (pTable->keyType == SDB_KEY_TYPE_AUTO) {
pTable->autoIndex = maxAutoIndex;
}
tfree(rowHead);
return 0;
}
void *sdbOpenTable(SSdbTableDesc *pDesc) {
SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable));
if (pTable == NULL) return NULL;
pTable->keyType = pDesc->keyType;
pTable->hashSessions = pDesc->hashSessions;
pTable->maxRowSize = pDesc->maxRowSize;
pTable->insertFp = pDesc->insertFp;
pTable->deleteFp = pDesc->deleteFp;
pTable->updateFp = pDesc->updateFp;
pTable->encodeFp = pDesc->encodeFp;
pTable->decodeFp = pDesc->decodeFp;
pTable->destroyFp = pDesc->destroyFp;
strcpy(pTable->tableName, pDesc->tableName);
sprintf(pTable->fileName, "%s/%s.db", tsMnodeDir, pTable->tableName);
if (sdbInitIndexFp[pTable->keyType] != NULL) {
pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta));
}
pthread_mutex_init(&pTable->mutex, NULL);
if (sdbInitTableByFile(pTable) < 0) return NULL;
pTable->tableId = sdbNumOfTables++;
sdbTableList[pTable->tableId] = pTable;
sdbTrace("table:%s, is initialized, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d numOfTables:%d",
pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbNumOfTables);
return pTable;
}
static SRowMeta *sdbGetRowMeta(void *handle, void *key) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
if (handle == NULL) return NULL;
pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key);
return pMeta;
}
void *sdbGetRow(void *handle, void *key) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
if (handle == NULL) return NULL;
pthread_mutex_lock(&pTable->mutex);
pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key);
pthread_mutex_unlock(&pTable->mutex);
if (pMeta == NULL) {
return NULL;
}
return pMeta->row;
}
int32_t sdbInsertRow(SSdbOperDesc *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) {
sdbError("sdb tables is null");
return TSDB_CODE_OTHERS;
}
if (sdbGetRow(pTable, pOper->pObj)) {
sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj));
return TSDB_CODE_ALREADY_THERE;
}
pOper->maxRowSize = pTable->maxRowSize;
pthread_mutex_lock(&pTable->mutex);
if (pOper->type == SDB_OPER_TYPE_GLOBAL) {
SForwardMsg forward = {
.type = SDB_FORWARD_TYPE_INSERT,
.tableId = pTable->tableId,
.version = pTable->version + 1,
.rowSize = pOper->rowSize,
.rowData = pOper->rowData,
};
if (sdbForwardDbReqToPeer(&forward) != 0) {
sdbError("table:%s, failed to forward record:%s version:%" PRId64 " sdbversion:%" PRId64,
pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion);
pthread_mutex_unlock(&pTable->mutex);
return TSDB_CODE_OTHERS;
}
}
int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM);
SRowHead *rowHead = (SRowHead *)calloc(1, total_size);
if (rowHead == NULL) {
pthread_mutex_unlock(&pTable->mutex);
sdbError("table:%s, failed to allocate row head memory for record:%s version:%" PRId64 " sdbversion:%" PRId64,
pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion);
return -1;
}
if (pTable->keyType == SDB_KEY_TYPE_AUTO) {
*((uint32_t *)pOper->pObj) = ++pTable->autoIndex;
}
pTable->version++;
sdbVersion++;
pOper->rowData = rowHead->data;
(*pTable->encodeFp)(pOper);
rowHead->rowSize = pOper->rowSize;
rowHead->delimiter = SDB_DELIMITER;
rowHead->version = pTable->version;
assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize);
int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM);
if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) {
sdbError("table:%s, failed to get checksum while inserting", pTable->tableName);
pTable->version--;
sdbVersion--;
pthread_mutex_unlock(&pTable->mutex);
tfree(rowHead);
return -1;
}
twrite(pTable->fd, rowHead, real_size);
pTable->fileSize += real_size;
sdbFinishCommit(pTable);
tfree(rowHead);
// update in SDB layer
SRowMeta rowMeta;
rowMeta.version = pTable->version;
rowMeta.offset = pTable->fileSize;
rowMeta.rowSize = pOper->rowSize;
rowMeta.row = pOper->pObj;
(*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta);
pTable->numOfRows++;
pthread_mutex_unlock(&pTable->mutex);
sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, insert record:%s, rowSize:%d fileSize:%" PRId64,
pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize);
(*pTable->insertFp)(pOper);
return 0;
}
// row here can be object or null-terminated string
int32_t sdbDeleteRow(SSdbOperDesc *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) return -1;
SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
if (pMeta == NULL) {
sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
return -1;
}
void * pMetaRow = pMeta->row;
assert(pMetaRow != NULL);
pthread_mutex_lock(&pTable->mutex);
if (pOper->type == SDB_OPER_TYPE_GLOBAL) {
SForwardMsg forward = {
.type = SDB_FORWARD_TYPE_DELETE,
.tableId = pTable->tableId,
.version = pTable->version + 1,
.rowSize = pMeta->rowSize,
.rowData = pMeta->row,
};
if (sdbForwardDbReqToPeer(&forward) != 0) {
sdbError("table:%s, failed to delete record", pTable->tableName);
pthread_mutex_unlock(&pTable->mutex);
return -1;
}
}
int32_t total_size = sizeof(SRowHead) + pMeta->rowSize + sizeof(TSCKSUM);
SRowHead *rowHead = (SRowHead *)calloc(1, total_size);
if (rowHead == NULL) {
sdbError("failed to allocate row head memory, sdb:%s", pTable->tableName);
pthread_mutex_unlock(&pTable->mutex);
return -1;
}
pTable->version++;
sdbVersion++;
int32_t rowSize = 0;
switch (pTable->keyType) {
case SDB_KEY_TYPE_STRING:
rowSize = strlen((char *)pOper->pObj) + 1;
break;
case SDB_KEY_TYPE_AUTO:
rowSize = sizeof(uint64_t);
break;
default:
return -1;
}
rowHead->delimiter = SDB_DELIMITER;
rowHead->rowSize = rowSize;
rowHead->version = -(pTable->version);
memcpy(rowHead->data, pOper->pObj, rowSize);
int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM);
if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) {
sdbError("failed to get checksum while inserting, sdb:%s", pTable->tableName);
pTable->version--;
sdbVersion--;
pthread_mutex_unlock(&pTable->mutex);
tfree(rowHead);
return -1;
}
twrite(pTable->fd, rowHead, real_size);
pTable->fileSize += real_size;
sdbFinishCommit(pTable);
tfree(rowHead);
sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, delete record:%s, rowSize:%d fileSize:%" PRId64,
pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize);
// Delete from current layer
(*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj);
pTable->numOfRows--;
pthread_mutex_unlock(&pTable->mutex);
(*pTable->deleteFp)(pOper);
(*pTable->destroyFp)(pOper);
return 0;
}
// row here can be the object or the string info (encoded string)
int32_t sdbUpdateRow(SSdbOperDesc *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) return -1;
SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
if (pMeta == NULL) {
sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " version:%" PRId64,
pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version);
return -1;
}
void *pMetaRow = pMeta->row;
assert(pMetaRow != NULL);
pthread_mutex_lock(&pTable->mutex);
if (pOper->type == SDB_OPER_TYPE_GLOBAL) {
SForwardMsg forward = {
.type = SDB_FORWARD_TYPE_UPDATE,
.tableId = pTable->tableId,
.version = pTable->version + 1,
.rowSize = pOper->rowSize,
.rowData = pOper->rowData,
};
if (sdbForwardDbReqToPeer(&forward) != 0) {
sdbError("table:%s, failed to update record", pTable->tableName);
pthread_mutex_unlock(&pTable->mutex);
return -1;
}
}
int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM);
SRowHead *rowHead = (SRowHead *)calloc(1, total_size);
if (rowHead == NULL) {
sdbError("table:%s, failed to allocate row head memory", pTable->tableName);
return -1;
}
if (pMetaRow != pOper->pObj) {
memcpy(rowHead->data, pOper->rowData, pOper->rowSize);
rowHead->rowSize = pOper->rowSize;
} else {
SSdbOperDesc oper = {
.table = pTable,
.rowData = rowHead->data,
.maxRowSize = pTable->maxRowSize,
.pObj = pOper->pObj
};
(*pTable->encodeFp)(&oper);
rowHead->rowSize = oper.rowSize;
}
pTable->version++;
sdbVersion++;
int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM);
rowHead->delimiter = SDB_DELIMITER;
rowHead->version = pTable->version;
if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) {
sdbError("table:%s, failed to get checksum, version:%d", pTable->tableName, rowHead->version);
pTable->version--;
sdbVersion--;
pthread_mutex_unlock(&pTable->mutex);
tfree(rowHead);
return -1;
}
twrite(pTable->fd, rowHead, real_size);
pTable->fileSize += real_size;
sdbFinishCommit(pTable);
sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, update record:%s, rowSize:%d fileSize:%" PRId64,
pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize);
pMeta->version = pTable->version;
pMeta->offset = pTable->fileSize;
pMeta->rowSize = rowHead->rowSize;
pthread_mutex_unlock(&pTable->mutex);
(*pTable->updateFp)(pOper); // update in upper layer
tfree(rowHead);
return 0;
}
void sdbCloseTable(void *handle) {
SSdbTable *pTable = (SSdbTable *)handle;
void * pNode = NULL;
if (pTable == NULL) return;
while (1) {
SRowMeta * pMeta;
pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
if (pMeta == NULL) break;
SSdbOperDesc oper = {
.pObj = pMeta->row,
.table = pTable,
.version = pMeta->version,
};
(*pTable->destroyFp)(&oper);
}
if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle);
if (pTable->fd) tclose(pTable->fd);
pthread_mutex_destroy(&pTable->mutex);
sdbNumOfTables--;
sdbTrace("table:%s, is closed, version:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables);
tfree(pTable);
}
void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
*ppRow = NULL;
if (pTable == NULL) return NULL;
pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
if (pMeta == NULL) return NULL;
*ppRow = pMeta->row;
return pNode;
}

55
src/mnode/src/mgmtShell.c
View File

@ -30,8 +30,8 @@
#include "mgmtDnode.h"
#include "mgmtGrant.h"
#include "mgmtMnode.h"
#include "mgmtNormalTable.h"
#include "mgmtProfile.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
#include "mgmtSuperTable.h"
#include "mgmtTable.h"
@ -63,7 +63,7 @@ int32_t mgmtInitShell() {
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_HEARTBEAT, mgmtProcessHeartBeatMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg);
tsMgmtTranQhandle = taosInitScheduler(tsMaxDnodes + tsMaxShellConns, 1, "mnodeT");
tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT");
int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0;
if (numOfThreads < 1) {
@ -131,19 +131,25 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) {
}
static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
if (sdbGetRunStatus() != SDB_STATUS_SERVING) {
if (!mgmtInServerStatus()) {
mgmtProcessMsgWhileNotReady(rpcMsg);
rpcFreeCont(rpcMsg->pCont);
return;
}
if (mgmtCheckExpired()) {
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_GRANT_EXPIRED);
return;
}
if (tsMgmtProcessShellMsgFp[rpcMsg->msgType] == NULL) {
mgmtProcessUnSupportMsg(rpcMsg);
rpcFreeCont(rpcMsg->pCont);
return;
}
SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
bool usePublicIp = false;
SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle, &usePublicIp);
if (pUser == NULL) {
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_INVALID_USER);
rpcFreeCont(rpcMsg->pCont);
@ -157,6 +163,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
queuedMsg.contLen = rpcMsg->contLen;
queuedMsg.pCont = rpcMsg->pCont;
queuedMsg.pUser = pUser;
queuedMsg.usePublicIp = usePublicIp;
(*tsMgmtProcessShellMsgFp[rpcMsg->msgType])(&queuedMsg);
rpcFreeCont(rpcMsg->pCont);
} else {
@ -166,6 +173,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
queuedMsg->contLen = rpcMsg->contLen;
queuedMsg->pCont = rpcMsg->pCont;
queuedMsg->pUser = pUser;
queuedMsg->usePublicIp = usePublicIp;
mgmtAddToShellQueue(queuedMsg);
}
}
@ -309,20 +317,10 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
return;
}
pHBRsp->ipList.inUse = 0;
pHBRsp->ipList.port = htons(tsMnodeShellPort);
pHBRsp->ipList.numOfIps = 0;
if (pSdbPublicIpList != NULL && pSdbIpList != NULL) {
pHBRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps);
if (connInfo.serverIp == tsPublicIpInt) {
for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) {
pHBRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]);
}
} else {
for (int i = 0; i < pSdbIpList->numOfIps; ++i) {
pHBRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]);
}
}
if (connInfo.serverIp == tsPublicIpInt) {
mgmtGetMnodePublicIpList(&pHBRsp->ipList);
} else {
mgmtGetMnodePrivateIpList(&pHBRsp->ipList);
}
/*
@ -411,20 +409,11 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
strcpy(pConnectRsp->serverVersion, version);
pConnectRsp->writeAuth = pUser->writeAuth;
pConnectRsp->superAuth = pUser->superAuth;
pConnectRsp->ipList.inUse = 0;
pConnectRsp->ipList.port = htons(tsMnodeShellPort);
pConnectRsp->ipList.numOfIps = 0;
if (pSdbPublicIpList != NULL && pSdbIpList != NULL) {
pConnectRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps);
if (connInfo.serverIp == tsPublicIpInt) {
for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) {
pConnectRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]);
}
} else {
for (int i = 0; i < pSdbIpList->numOfIps; ++i) {
pConnectRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]);
}
}
if (connInfo.serverIp == tsPublicIpInt) {
mgmtGetMnodePublicIpList(&pConnectRsp->ipList);
} else {
mgmtGetMnodePrivateIpList(&pConnectRsp->ipList);
}
connect_over:
@ -458,7 +447,7 @@ static bool mgmtCheckMeterMetaMsgType(void *pMsg) {
static bool mgmtCheckMsgReadOnly(int8_t type, void *pCont) {
if ((type == TSDB_MSG_TYPE_CM_TABLE_META && (!mgmtCheckMeterMetaMsgType(pCont))) ||
type == TSDB_MSG_TYPE_CM_STABLE_META || type == TSDB_MSG_TYPE_RETRIEVE ||
type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_RETRIEVE ||
type == TSDB_MSG_TYPE_CM_SHOW || type == TSDB_MSG_TYPE_CM_TABLES_META ||
type == TSDB_MSG_TYPE_CM_CONNECT) {
return true;

402
src/mnode/src/mgmtSuperTable.c
View File

@ -15,178 +15,126 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "name.h"
#include "tsqlfunction.h"
#include "mgmtAcct.h"
#include "mgmtChildTable.h"
#include "mgmtDb.h"
#include "mgmtDClient.h"
#include "mgmtDnode.h"
#include "mgmtGrant.h"
#include "mgmtShell.h"
#include "mgmtSuperTable.h"
#include "mgmtSdb.h"
#include "mgmtTable.h"
#include "mgmtUser.h"
#include "mgmtVgroup.h"
#include "mnode.h"
#include "name.h"
#include "tsqlfunction.h"
static void *tsSuperTableSdb;
static void *tsSuperTableSdb;
static int32_t tsSuperTableUpdateSize;
static void *(*mgmtSuperTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize);
static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *queueMsg);
static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg);
static int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static void mgmtDestroySuperTable(SSuperTableObj *pTable) {
free(pTable->schema);
free(pTable);
static void mgmtDestroySuperTable(SSuperTableObj *pStable) {
tfree(pStable->schema);
tfree(pStable);
}
static void mgmtSuperTableActionInit() {
SSuperTableObj tObj;
tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj;
mgmtSuperTableActionFp[SDB_TYPE_INSERT] = mgmtSuperTableActionInsert;
mgmtSuperTableActionFp[SDB_TYPE_DELETE] = mgmtSuperTableActionDelete;
mgmtSuperTableActionFp[SDB_TYPE_UPDATE] = mgmtSuperTableActionUpdate;
mgmtSuperTableActionFp[SDB_TYPE_ENCODE] = mgmtSuperTableActionEncode;
mgmtSuperTableActionFp[SDB_TYPE_DECODE] = mgmtSuperTableActionDecode;
mgmtSuperTableActionFp[SDB_TYPE_RESET] = mgmtSuperTableActionReset;
mgmtSuperTableActionFp[SDB_TYPE_DESTROY] = mgmtSuperTableActionDestroy;
static int32_t mgmtSuperTableActionDestroy(SSdbOperDesc *pOper) {
mgmtDestroySuperTable(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) {
SSuperTableObj *pTable = (SSuperTableObj *) row;
memcpy(pTable, str, tsSuperTableUpdateSize);
int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags);
pTable->schema = realloc(pTable->schema, schemaSize);
memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize);
return NULL;
}
void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) {
SSuperTableObj *pTable = (SSuperTableObj *) row;
mgmtDestroySuperTable(pTable);
return NULL;
}
void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) {
STableInfo *pTable = (STableInfo *) row;
SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId);
if (pDb) {
static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) {
SSuperTableObj *pStable = pOper->pObj;
SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId);
if (pDb != NULL) {
mgmtAddSuperTableIntoDb(pDb);
}
return NULL;
return TSDB_CODE_SUCCESS;
}
void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) {
STableInfo *pTable = (STableInfo *) row;
SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId);
if (pDb) {
static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) {
SSuperTableObj *pStable = pOper->pObj;
SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId);
if (pDb != NULL) {
mgmtRemoveSuperTableFromDb(pDb);
mgmtDropAllChildTablesInStable((SSuperTableObj *)pStable);
}
return NULL;
return TSDB_CODE_SUCCESS;
}
void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) {
return mgmtSuperTableActionReset(row, str, size, NULL);
static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) {
SSuperTableObj *pTable = (SSuperTableObj *) row;
assert(row != NULL && str != NULL);
static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) {
SSuperTableObj *pStable = pOper->pObj;
assert(pOper->pObj != NULL && pOper->rowData != NULL);
int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags);
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags);
if (size < tsSuperTableUpdateSize + schemaSize + 1) {
*ssize = -1;
return NULL;
if (pOper->maxRowSize < tsSuperTableUpdateSize + schemaSize) {
return TSDB_CODE_INVALID_MSG_LEN;
}
memcpy(str, pTable, tsSuperTableUpdateSize);
memcpy(str + tsSuperTableUpdateSize, pTable->schema, schemaSize);
*ssize = tsSuperTableUpdateSize + schemaSize;
memcpy(pOper->rowData, pStable, tsSuperTableUpdateSize);
memcpy(pOper->rowData + tsSuperTableUpdateSize, pStable->schema, schemaSize);
pOper->rowSize = tsSuperTableUpdateSize + schemaSize;
return NULL;
return TSDB_CODE_SUCCESS;
}
void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) {
assert(str != NULL);
static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) {
assert(pOper->rowData != NULL);
SSuperTableObj *pTable = (SSuperTableObj *) malloc(sizeof(SSuperTableObj));
if (pTable == NULL) {
return NULL;
}
memset(pTable, 0, sizeof(SSuperTableObj));
SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj));
if (pStable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
if (size < tsSuperTableUpdateSize) {
mgmtDestroySuperTable(pTable);
return NULL;
}
memcpy(pTable, str, tsSuperTableUpdateSize);
memcpy(pStable, pOper->rowData, tsSuperTableUpdateSize);
int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags);
pTable->schema = malloc(schemaSize);
if (pTable->schema == NULL) {
mgmtDestroySuperTable(pTable);
return NULL;
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags);
pStable->schema = malloc(schemaSize);
if (pStable->schema == NULL) {
mgmtDestroySuperTable(pStable);
return -1;
}
memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize);
return (void *) pTable;
}
memcpy(pStable->schema, pOper->rowData + tsSuperTableUpdateSize, schemaSize);
pOper->pObj = pStable;
void *mgmtSuperTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) {
if (mgmtSuperTableActionFp[(uint8_t) action] != NULL) {
return (*(mgmtSuperTableActionFp[(uint8_t) action]))(row, str, size, ssize);
}
return NULL;
return TSDB_CODE_SUCCESS;
}
int32_t mgmtInitSuperTables() {
void *pNode = NULL;
void *pLastNode = NULL;
SSuperTableObj *pTable = NULL;
SSuperTableObj tObj;
tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
mgmtSuperTableActionInit();
SSdbTableDesc tableDesc = {
.tableName = "stables",
.hashSessions = TSDB_MAX_SUPER_TABLES,
.maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS,
.keyType = SDB_KEY_TYPE_STRING,
.insertFp = mgmtSuperTableActionInsert,
.deleteFp = mgmtSuperTableActionDelete,
.updateFp = mgmtSuperTableActionUpdate,
.encodeFp = mgmtSuperTableActionEncode,
.decodeFp = mgmtSuperTableActionDecode,
.destroyFp = mgmtSuperTableActionDestroy,
};
tsSuperTableSdb = sdbOpenTable(tsMaxTables, tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS,
"stables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtSuperTableAction);
tsSuperTableSdb = sdbOpenTable(&tableDesc);
if (tsSuperTableSdb == NULL) {
mError("failed to init stables data");
return -1;
}
pNode = NULL;
while (1) {
pNode = sdbFetchRow(tsSuperTableSdb, pNode, (void **) &pTable);
if (pTable == NULL) {
break;
}
SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId);
if (pDb == NULL) {
mError("super table:%s, failed to get db, discard it", pTable->tableId);
sdbDeleteRow(tsSuperTableSdb, pTable);
pNode = pLastNode;
continue;
}
mgmtAddSuperTableIntoDb(pDb);
}
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_STABLE_VGROUP, mgmtProcessSuperTableVgroupMsg);
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mgmtGetShowSuperTableMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mgmtRetrieveShowSuperTables);
mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropStableRsp);
mTrace("stables is initialized");
return 0;
@ -196,24 +144,18 @@ void mgmtCleanUpSuperTables() {
sdbCloseTable(tsSuperTableSdb);
}
int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) {
int32_t numOfTables = sdbGetNumOfRows(tsSuperTableSdb);
if (numOfTables >= TSDB_MAX_SUPER_TABLES) {
mError("stable:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_SUPER_TABLES);
return TSDB_CODE_TOO_MANY_TABLES;
}
SSuperTableObj *pStable = (SSuperTableObj *)calloc(sizeof(SSuperTableObj), 1);
void mgmtCreateSuperTable(SQueuedMsg *pMsg) {
SCMCreateTableMsg *pCreate = pMsg->pCont;
SSuperTableObj *pStable = (SSuperTableObj *)calloc(1, sizeof(SSuperTableObj));
if (pStable == NULL) {
return TSDB_CODE_SERV_OUT_OF_MEMORY;
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
return;
}
strcpy(pStable->tableId, pCreate->tableId);
pStable->type = TSDB_SUPER_TABLE;
strcpy(pStable->info.tableId, pCreate->tableId);
pStable->info.type = TSDB_SUPER_TABLE;
pStable->createdTime = taosGetTimestampMs();
pStable->vgId = 0;
pStable->sid = 0;
pStable->uid = (((uint64_t) pStable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul));
pStable->uid = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul));
pStable->sversion = 0;
pStable->numOfColumns = htons(pCreate->numOfColumns);
pStable->numOfTags = htons(pCreate->numOfTags);
@ -224,7 +166,8 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) {
if (pStable->schema == NULL) {
free(pStable);
mError("stable:%s, no schema input", pCreate->tableId);
return TSDB_CODE_INVALID_TABLE;
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
return;
}
memcpy(pStable->schema, pCreate->schema, numOfCols * sizeof(SSchema));
@ -235,24 +178,36 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) {
tschema[col].bytes = htons(tschema[col].bytes);
}
if (sdbInsertRow(tsSuperTableSdb, pStable, 0) < 0) {
mError("stable:%s, update sdb error", pStable->tableId);
return TSDB_CODE_SDB_ERROR;
}
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = sizeof(SSuperTableObj) + schemaSize
};
mPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns);
return TSDB_CODE_SUCCESS;
int32_t code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
mgmtDestroySuperTable(pStable);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SDB_ERROR);
} else {
mLPrint("stable:%s, is created, tags:%d cols:%d", pStable->info.tableId, pStable->numOfTags, pStable->numOfColumns);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
}
}
int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pStable) {
void mgmtDropSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pStable) {
if (pStable->numOfTables != 0) {
mError("stable:%s, numOfTables:%d not 0", pStable->tableId, pStable->numOfTables);
return TSDB_CODE_OTHERS;
mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS);
} else {
//TODO: drop child tables
mError("stable:%s, is dropped from sdb", pStable->tableId);
mgmtRemoveSuperTableFromDb(pDb);
return TSDB_CODE_OTHERS;
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable
};
int32_t code = sdbDeleteRow(&oper);
mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code));
mgmtSendSimpleResp(pMsg->thandle, code);
}
}
@ -260,15 +215,14 @@ void* mgmtGetSuperTable(char *tableId) {
return sdbGetRow(tsSuperTableSdb, tableId);
}
void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) {
//TODO get vgroup of dnodes
SCMSuperTableInfoRsp *rsp = rpcMallocCont(sizeof(SCMSuperTableInfoRsp) + sizeof(uint32_t) * mgmtGetDnodesNum());
rsp->numOfDnodes = 1;
rsp->dnodeIps[0] = 0;
static void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) {
SCMSTableVgroupRspMsg *rsp = rpcMallocCont(sizeof(SCMSTableVgroupRspMsg) + sizeof(uint32_t) * mgmtGetDnodesNum());
rsp->numOfDnodes = htonl(1);
rsp->dnodeIps[0] = htonl(inet_addr(tsPrivateIp));
return rsp;
}
int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) {
static int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) {
for (int32_t i = 0; i < pStable->numOfTags; i++) {
SSchema *schema = (SSchema *)(pStable->schema + (pStable->numOfColumns + i) * sizeof(SSchema));
if (strcasecmp(tagName, schema->name) == 0) {
@ -279,7 +233,7 @@ int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName)
return -1;
}
int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) {
static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) {
if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) {
return TSDB_CODE_APP_ERROR;
}
@ -293,9 +247,9 @@ int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t
}
}
SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId);
SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId);
if (pDb == NULL) {
mError("meter: %s not belongs to any database", pStable->tableId);
mError("meter: %s not belongs to any database", pStable->info.tableId);
return TSDB_CODE_APP_ERROR;
}
@ -321,21 +275,21 @@ int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t
pStable->sversion++;
pAcct->acctInfo.numOfTimeSeries += (ntags * pStable->numOfTables);
sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1);
// sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL);
mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->tableId);
mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->info.tableId);
return TSDB_CODE_SUCCESS;
}
int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
int32_t col = mgmtFindSuperTableTagIndex(pStable, tagName);
if (col <= 0 || col >= pStable->numOfTags) {
return TSDB_CODE_APP_ERROR;
}
SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId);
SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId);
if (pDb == NULL) {
mError("table: %s not belongs to any database", pStable->tableId);
mError("table: %s not belongs to any database", pStable->info.tableId);
return TSDB_CODE_APP_ERROR;
}
@ -354,21 +308,21 @@ int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns);
pStable->schema = realloc(pStable->schema, schemaSize);
sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1);
// sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL);
return TSDB_CODE_SUCCESS;
}
int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) {
static int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) {
int32_t col = mgmtFindSuperTableTagIndex(pStable, oldTagName);
if (col < 0) {
// Tag name does not exist
mError("Failed to modify table %s tag column, oname: %s, nname: %s", pStable->tableId, oldTagName, newTagName);
mError("Failed to modify table %s tag column, oname: %s, nname: %s", pStable->info.tableId, oldTagName, newTagName);
return TSDB_CODE_INVALID_MSG_TYPE;
}
int32_t rowSize = 0;
uint32_t len = strlen(newTagName);
// int32_t rowSize = 0;
uint32_t len = strlen(newTagName);
if (col >= pStable->numOfTags || len >= TSDB_COL_NAME_LEN || mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) {
return TSDB_CODE_APP_ERROR;
@ -384,17 +338,18 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN
if (msg == NULL) return TSDB_CODE_APP_ERROR;
memset(msg, 0, size);
mgmtSuperTableActionEncode(pStable, msg, size, &rowSize);
// mgmtSuperTableActionEncode(pStable, msg, size, &rowSize);
int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, rowSize, 1);
int32_t ret = 0;
// int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL);
tfree(msg);
if (ret < 0) {
mError("Failed to modify table %s tag column", pStable->tableId);
mError("Failed to modify table %s tag column", pStable->info.tableId);
return TSDB_CODE_APP_ERROR;
}
mTrace("Succeed to modify table %s tag column", pStable->tableId);
mTrace("Succeed to modify table %s tag column", pStable->info.tableId);
return TSDB_CODE_SUCCESS;
}
@ -409,7 +364,7 @@ static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colN
return -1;
}
int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32_t ncols) {
static int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32_t ncols) {
if (ncols <= 0) {
return TSDB_CODE_APP_ERROR;
}
@ -420,9 +375,9 @@ int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32
}
}
SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId);
SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId);
if (pDb == NULL) {
mError("meter: %s not belongs to any database", pStable->tableId);
mError("meter: %s not belongs to any database", pStable->info.tableId);
return TSDB_CODE_APP_ERROR;
}
@ -448,20 +403,20 @@ int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32
pStable->sversion++;
pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables);
sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1);
// sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL);
return TSDB_CODE_SUCCESS;
}
int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) {
static int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) {
int32_t col = mgmtFindSuperTableColumnIndex(pStable, colName);
if (col < 0) {
return TSDB_CODE_APP_ERROR;
}
SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId);
SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId);
if (pDb == NULL) {
mError("table: %s not belongs to any database", pStable->tableId);
mError("table: %s not belongs to any database", pStable->info.tableId);
return TSDB_CODE_APP_ERROR;
}
@ -481,7 +436,7 @@ int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) {
pStable->schema = realloc(pStable->schema, schemaSize);
pAcct->acctInfo.numOfTimeSeries -= (pStable->numOfTables);
sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1);
// sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL);
return TSDB_CODE_SUCCESS;
}
@ -548,7 +503,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
SDbObj *pDb = mgmtGetDb(pShow->db);
if (pDb == NULL) return 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn);
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) {
if (strcmp(pUser->user, "root") != 0 && strcmp(pUser->user, "_root") != 0 && strcmp(pUser->user, "monitor") != 0 ) {
@ -566,12 +521,12 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
while (numOfRows < rows) {
pShow->pNode = sdbFetchRow(tsSuperTableSdb, pShow->pNode, (void **) &pTable);
if (pTable == NULL) break;
if (strncmp(pTable->tableId, prefix, prefixLen)) {
if (strncmp(pTable->info.tableId, prefix, prefixLen)) {
continue;
}
memset(stableName, 0, tListLen(stableName));
extractTableName(pTable->tableId, stableName);
mgmtExtractTableName(pTable->info.tableId, stableName);
if (pShow->payloadLen > 0 &&
patternMatch(pShow->payload, stableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH)
@ -619,23 +574,20 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
break;
}
if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) {
sdbDeleteRow(tsSuperTableSdb, pTable);
if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.table = tsSuperTableSdb,
.pObj = pTable,
};
sdbDeleteRow(&oper);
pNode = pLastNode;
numOfTables ++;
continue;
}
}
mTrace("db:%s, all super tables:%d is dropped", pDropDb->name, numOfTables);
}
void mgmtAddTableIntoSuperTable(SSuperTableObj *pStable) {
pStable->numOfTables++;
}
void mgmtRemoveTableFromSuperTable(SSuperTableObj *pStable) {
pStable->numOfTables--;
mTrace("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables);
}
int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) {
@ -651,18 +603,70 @@ int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) {
return (pTable->numOfColumns + pTable->numOfTags) * sizeof(SSchema);
}
int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) {
void mgmtGetSuperTableMeta(SQueuedMsg *pMsg, SSuperTableObj *pTable) {
SDbObj *pDb = pMsg->pDb;
STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS);
pMeta->uid = htobe64(pTable->uid);
pMeta->sid = htonl(pTable->sid);
pMeta->vgId = htonl(pTable->vgId);
pMeta->sversion = htons(pTable->sversion);
pMeta->precision = pDb->cfg.precision;
pMeta->numOfTags = pTable->numOfTags;
pMeta->numOfColumns = htons(pTable->numOfColumns);
pMeta->tableType = pTable->type;
pMeta->numOfTags = (uint8_t)pTable->numOfTags;
pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns);
pMeta->tableType = pTable->info.type;
pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable);
strcpy(pMeta->tableId, pTable->tableId);
strcpy(pMeta->tableId, pTable->info.tableId);
return TSDB_CODE_SUCCESS;
SRpcMsg rpcRsp = {
.handle = pMsg->thandle,
.pCont = pMeta,
.contLen = pMeta->contLen,
};
pMeta->contLen = htons(pMeta->contLen);
rpcSendResponse(&rpcRsp);
mTrace("stable:%%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid);
}
static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
SCMSTableVgroupMsg *pInfo = pMsg->pCont;
STableInfo *pTable = mgmtGetSuperTable(pInfo->tableId);
if (pTable == NULL) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
return;
}
SCMSTableVgroupRspMsg *pRsp = mgmtGetSuperTableVgroup((SSuperTableObj *) pTable);
if (pRsp != NULL) {
int32_t msgLen = sizeof(SSuperTableObj) + htonl(pRsp->numOfDnodes) * sizeof(int32_t);
SRpcMsg rpcRsp = {0};
rpcRsp.handle = pMsg->thandle;
rpcRsp.pCont = pRsp;
rpcRsp.contLen = msgLen;
rpcSendResponse(&rpcRsp);
} else {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
}
}
void mgmtAlterSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pTable) {
int32_t code = TSDB_CODE_OPS_NOT_SUPPORT;
SCMAlterTableMsg *pAlter = pMsg->pCont;
if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) {
code = mgmtAddSuperTableTag((SSuperTableObj *) pTable, pAlter->schema, 1);
} else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) {
code = mgmtDropSuperTableTag((SSuperTableObj *) pTable, pAlter->schema[0].name);
} else if (pAlter->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) {
code = mgmtModifySuperTableTagNameByName((SSuperTableObj *) pTable, pAlter->schema[0].name, pAlter->schema[1].name);
} else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) {
code = mgmtAddSuperTableColumn((SSuperTableObj *) pTable, pAlter->schema, 1);
} else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) {
code = mgmtDropSuperTableColumnByName((SSuperTableObj *) pTable, pAlter->schema[0].name);
} else {}
mgmtSendSimpleResp(pMsg->thandle, code);
}
static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) {
mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code);
}

803
src/mnode/src/mgmtTable.c
View File

@ -15,51 +15,25 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "mgmtTable.h"
#include "mgmtAcct.h"
#include "mgmtChildTable.h"
#include "mgmtDClient.h"
#include "mgmtDb.h"
#include "mgmtDnode.h"
#include "mgmtDServer.h"
#include "mgmtGrant.h"
#include "mgmtMnode.h"
#include "mgmtNormalTable.h"
#include "mgmtProfile.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
#include "mgmtSuperTable.h"
#include "mgmtTable.h"
#include "mgmtUser.h"
#include "mgmtVgroup.h"
#include "mnode.h"
#include "qast.h"
#include "qextbuffer.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tscompression.h"
#include "tskiplist.h"
#include "tsqlfunction.h"
#include "tstatus.h"
#include "ttime.h"
#include "name.h"
extern void *tsNormalTableSdb;
extern void *tsChildTableSdb;
static void mgmtProcessCreateTableMsg(SQueuedMsg *queueMsg);
static void mgmtProcessDropTableMsg(SQueuedMsg *queueMsg);
static void mgmtProcessAlterTableMsg(SQueuedMsg *queueMsg);
static void mgmtProcessTableMetaMsg(SQueuedMsg *queueMsg);
static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *queueMsg);
static void mgmtProcessSuperTableMetaMsg(SQueuedMsg *queueMsg);
static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg);
static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg);
static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg);
static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg);
static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle);
int32_t mgmtInitTables() {
int32_t code = mgmtInitSuperTables();
@ -67,46 +41,26 @@ int32_t mgmtInitTables() {
return code;
}
code = mgmtInitNormalTables();
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = mgmtInitChildTables();
if (code != TSDB_CODE_SUCCESS) {
return code;
}
mgmtSetVgroupIdPool();
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_TABLE, mgmtProcessCreateTableMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_TABLE, mgmtProcessDropTableMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_TABLE, mgmtProcessAlterTableMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_TABLE_META, mgmtProcessTableMetaMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_TABLES_META, mgmtProcessMultiTableMetaMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_STABLE_META, mgmtProcessSuperTableMetaMsg);
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mgmtGetShowTableMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mgmtRetrieveShowTables);
mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP, mgmtProcessCreateTableRsp);
mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mgmtProcessDropTableRsp);
mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP, mgmtProcessAlterTableRsp);
mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropStableRsp);
return TSDB_CODE_SUCCESS;
}
STableInfo* mgmtGetTable(char *tableId) {
STableInfo *tableInfo = (STableInfo *) mgmtGetSuperTable(tableId);
STableInfo *mgmtGetTable(char *tableId) {
STableInfo *tableInfo = mgmtGetSuperTable(tableId);
if (tableInfo != NULL) {
return tableInfo;
}
tableInfo = (STableInfo *) mgmtGetNormalTable(tableId);
if (tableInfo != NULL) {
return tableInfo;
}
tableInfo = (STableInfo *) mgmtGetChildTable(tableId);
tableInfo = mgmtGetChildTable(tableId);
if (tableInfo != NULL) {
return tableInfo;
}
@ -114,252 +68,29 @@ STableInfo* mgmtGetTable(char *tableId) {
return NULL;
}
STableInfo* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) {
SDnodeObj *pObj = mgmtGetDnode(dnodeId);
SVgObj *pVgroup = mgmtGetVgroup(vnode);
if (pObj == NULL || pVgroup == NULL) {
return NULL;
}
return pVgroup->tableList[sid];
}
int32_t mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMetaMsg *pMeta, bool usePublicIp) {
if (pTable->type == TSDB_CHILD_TABLE) {
mgmtGetChildTableMeta(pDb, (SChildTableObj *) pTable, pMeta, usePublicIp);
} else if (pTable->type == TSDB_NORMAL_TABLE) {
mgmtGetNormalTableMeta(pDb, (SNormalTableObj *) pTable, pMeta, usePublicIp);
} else if (pTable->type == TSDB_SUPER_TABLE) {
mgmtGetSuperTableMeta(pDb, (SSuperTableObj *) pTable, pMeta, usePublicIp);
} else {
mTrace("%s, uid:%" PRIu64 " table meta retrieve failed, invalid type", pTable->tableId, pTable->uid);
return TSDB_CODE_INVALID_TABLE;
}
mTrace("%s, uid:%" PRIu64 " table meta is retrieved", pTable->tableId, pTable->uid);
return TSDB_CODE_SUCCESS;
}
int32_t mgmtAlterTable(SDbObj *pDb, SCMAlterTableMsg *pAlter) {
STableInfo *pTable = mgmtGetTable(pAlter->tableId);
if (pTable == NULL) {
return TSDB_CODE_INVALID_TABLE;
}
if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) {
return TSDB_CODE_MONITOR_DB_FORBIDDEN;
}
if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) {
if (pTable->type == TSDB_SUPER_TABLE) {
return mgmtAddSuperTableTag((SSuperTableObj *) pTable, pAlter->schema, 1);
}
} else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) {
if (pTable->type == TSDB_SUPER_TABLE) {
return mgmtDropSuperTableTag((SSuperTableObj *) pTable, pAlter->schema[0].name);
}
} else if (pAlter->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) {
if (pTable->type == TSDB_SUPER_TABLE) {
return mgmtModifySuperTableTagNameByName((SSuperTableObj *) pTable, pAlter->schema[0].name, pAlter->schema[1].name);
}
} else if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) {
if (pTable->type == TSDB_CHILD_TABLE) {
return mgmtModifyChildTableTagValueByName((SChildTableObj *) pTable, pAlter->schema[0].name, pAlter->tagVal);
}
} else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) {
if (pTable->type == TSDB_NORMAL_TABLE) {
return mgmtAddNormalTableColumn((SNormalTableObj *) pTable, pAlter->schema, 1);
} else if (pTable->type == TSDB_SUPER_TABLE) {
return mgmtAddSuperTableColumn((SSuperTableObj *) pTable, pAlter->schema, 1);
} else {}
} else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) {
if (pTable->type == TSDB_NORMAL_TABLE) {
return mgmtDropNormalTableColumnByName((SNormalTableObj *) pTable, pAlter->schema[0].name);
} else if (pTable->type == TSDB_SUPER_TABLE) {
return mgmtDropSuperTableColumnByName((SSuperTableObj *) pTable, pAlter->schema[0].name);
} else {}
} else {}
return TSDB_CODE_OPS_NOT_SUPPORT;
}
void mgmtCleanUpTables() {
mgmtCleanUpNormalTables();
mgmtCleanUpChildTables();
mgmtCleanUpSuperTables();
}
int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SDbObj *pDb = mgmtGetDb(pShow->db);
if (pDb == NULL) {
return TSDB_CODE_DB_NOT_SELECTED;
void mgmtExtractTableName(char* tableId, char* name) {
int pos = -1;
int num = 0;
for (pos = 0; tableId[pos] != 0; ++pos) {
if (tableId[pos] == '.') num++;
if (num == 2) break;
}
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "table_name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "created_time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "columns");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "stable_name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) {
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = pDb->numOfTables;
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
return 0;
}
/*
* remove the hole in result set
*/
static void mgmtVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) {
if (rows < capacity) {
for (int32_t i = 0; i < numOfCols; ++i) {
memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows);
}
if (num == 2) {
strcpy(name, tableId + pos + 1);
}
}
int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
SDbObj *pDb = mgmtGetDb(pShow->db);
if (pDb == NULL) return 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn);
if (pUser == NULL) return 0;
if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) {
if (strcmp(pUser->user, "root") != 0 && strcmp(pUser->user, "_root") != 0 &&
strcmp(pUser->user, "monitor") != 0) {
return 0;
}
}
int32_t numOfRows = 0;
int32_t numOfRead = 0;
int32_t cols = 0;
void *pTable = NULL;
char *pWrite = NULL;
char prefix[20] = {0};
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
strcpy(prefix, pDb->name);
strcat(prefix, TS_PATH_DELIMITER);
int32_t prefixLen = strlen(prefix);
while (numOfRows < rows) {
int16_t numOfColumns = 0;
int64_t createdTime = 0;
char *tableId = NULL;
char *superTableId = NULL;
void *pNormalTableNode = sdbFetchRow(tsNormalTableSdb, pShow->pNode, (void **) &pTable);
if (pTable != NULL) {
SNormalTableObj *pNormalTable = (SNormalTableObj *) pTable;
pShow->pNode = pNormalTableNode;
tableId = pNormalTable->tableId;
superTableId = NULL;
createdTime = pNormalTable->createdTime;
numOfColumns = pNormalTable->numOfColumns;
} else {
void *pChildTableNode = sdbFetchRow(tsChildTableSdb, pShow->pNode, (void **) &pTable);
if (pTable != NULL) {
SChildTableObj *pChildTable = (SChildTableObj *) pTable;
pShow->pNode = pChildTableNode;
tableId = pChildTable->tableId;
superTableId = pChildTable->superTableId;
createdTime = pChildTable->createdTime;
numOfColumns = pChildTable->superTable->numOfColumns;
} else {
break;
}
}
// not belong to current db
if (strncmp(tableId, prefix, prefixLen)) {
continue;
}
char tableName[TSDB_TABLE_NAME_LEN] = {0};
memset(tableName, 0, tListLen(tableName));
numOfRead++;
// pattern compare for meter name
extractTableName(tableId, tableName);
if (pShow->payloadLen > 0 &&
patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) {
continue;
}
cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strncpy(pWrite, tableName, TSDB_TABLE_NAME_LEN);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *) pWrite = createdTime;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *) pWrite = numOfColumns;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
if (superTableId != NULL) {
extractTableName(superTableId, pWrite);
}
cols++;
numOfRows++;
}
pShow->numOfReads += numOfRead;
const int32_t NUM_OF_COLUMNS = 4;
mgmtVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow);
return numOfRows;
}
void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) return;
static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
SCMCreateTableMsg *pCreate = pMsg->pCont;
mTrace("table:%s, create msg is received from thandle:%p", pCreate->tableId, pMsg->thandle);
if (mgmtCheckExpired()) {
mError("table:%s, failed to create, grant expired", pCreate->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED);
return;
}
if (mgmtCheckRedirect(pMsg->thandle)) return;
if (!pMsg->pUser->writeAuth) {
mError("table:%s, failed to create, no rights", pCreate->tableId);
@ -367,16 +98,8 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
return;
}
SAcctObj *pAcct = pMsg->pUser->pAcct;
int32_t code = mgmtCheckTableLimit(pAcct, htons(pCreate->numOfColumns));
if (code != TSDB_CODE_SUCCESS) {
mError("table:%s, failed to create, exceed the limit", pCreate->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
return;
}
pMsg->pDb = mgmtGetDb(pCreate->db);
if (pMsg->pDb == NULL) {
if (pMsg->pDb == NULL || pMsg->pDb->dirty) {
mError("table:%s, failed to create, db not selected", pCreate->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
return;
@ -396,88 +119,19 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
}
if (pCreate->numOfTags != 0) {
mTrace("table:%s, is a super table", pCreate->tableId);
code = mgmtCreateSuperTable(pCreate);
mgmtSendSimpleResp(pMsg->thandle, code);
return;
}
code = mgmtCheckTimeSeries(pCreate->numOfColumns);
if (code != TSDB_CODE_SUCCESS) {
mError("table:%s, failed to create, timeseries exceed the limit", pCreate->tableId);
mgmtSendSimpleResp(pMsg->thandle, code);
return;
}
SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg));
memcpy(newMsg, pMsg, sizeof(SQueuedMsg));
pMsg->pCont = NULL;
SVgObj *pVgroup = mgmtGetAvailableVgroup(pMsg->pDb);
if (pVgroup == NULL) {
mTrace("table:%s, start to create a new vgroup", pCreate->tableId);
mgmtCreateVgroup(newMsg);
return;
}
int32_t sid = taosAllocateId(pVgroup->idPool);
if (sid < 0) {
mTrace("tables:%s, no enough sid in vgroup:%d", pVgroup->vgId);
mgmtCreateVgroup(newMsg);
return;
}
SMDCreateTableMsg *pMDCreate = NULL;
if (pCreate->numOfColumns == 0) {
mTrace("table:%s, is a child table, vgroup:%d sid:%d ahandle:%p", pCreate->tableId, pVgroup->vgId, sid, pMsg);
pTable = mgmtCreateChildTable(pCreate, pVgroup, sid);
if (pTable == NULL) {
mgmtSendSimpleResp(pMsg->thandle, terrno);
return;
}
pMDCreate = mgmtBuildCreateChildTableMsg(pCreate, (SChildTableObj *) pTable);
if (pMDCreate == NULL) {
mgmtSendSimpleResp(pMsg->thandle, terrno);
return;
}
mTrace("table:%s, is a stable", pCreate->tableId);
mgmtCreateSuperTable(pMsg);
} else {
mTrace("table:%s, is a normal table, vgroup:%d sid:%d ahandle:%p", pCreate->tableId, pVgroup->vgId, sid, pMsg);
pTable = mgmtCreateNormalTable(pCreate, pVgroup, sid);
if (pTable == NULL) {
mgmtSendSimpleResp(pMsg->thandle, terrno);
return;
}
pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable);
if (pMDCreate == NULL) {
mgmtSendSimpleResp(pMsg->thandle, terrno);
return;
}
mTrace("table:%s, is a ctable", pCreate->tableId);
mgmtCreateChildTable(pMsg);
}
SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
SRpcMsg rpcMsg = {
.handle = newMsg,
.pCont = pMDCreate,
.contLen = htonl(pMDCreate->contLen),
.code = 0,
.msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE
};
newMsg->ahandle = pTable;
mgmtSendMsgToDnode(&ipSet, &rpcMsg);
}
void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) return;
static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
SCMDropTableMsg *pDrop = pMsg->pCont;
mTrace("table:%s, drop table msg is received from thandle:%p", pDrop->tableId, pMsg->thandle);
if (mgmtCheckExpired()) {
mError("table:%s, failed to drop, grant expired", pDrop->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED);
return;
}
if (mgmtCheckRedirect(pMsg->thandle)) return;
if (!pMsg->pUser->writeAuth) {
mError("table:%s, failed to drop, no rights", pDrop->tableId);
@ -485,13 +139,19 @@ void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
return;
}
SDbObj *pDb = mgmtGetDbByTableId(pDrop->tableId);
if (pDb == NULL) {
pMsg->pDb = mgmtGetDbByTableId(pDrop->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->dirty) {
mError("table:%s, failed to drop table, db not selected", pDrop->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
return;
}
if (mgmtCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) {
mError("table:%s, failed to drop table, in monitor database", pDrop->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN);
return;
}
STableInfo *pTable = mgmtGetTable(pDrop->tableId);
if (pTable == NULL) {
if (pDrop->igNotExists) {
@ -505,371 +165,82 @@ void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
}
}
if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) {
mError("table:%s, failed to drop table, in monitor database", pDrop->tableId);
if (pTable->type == TSDB_SUPER_TABLE) {
mTrace("table:%s, start to drop stable", pDrop->tableId);
mgmtDropSuperTable(pMsg, (SSuperTableObj *)pTable);
} else {
mTrace("table:%s, start to drop ctable", pDrop->tableId);
mgmtDropChildTable(pMsg, (SChildTableObj *)pTable);
}
}
static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
SCMAlterTableMsg *pAlter = pMsg->pCont;
mTrace("table:%s, alter table msg is received from thandle:%p", pAlter->tableId, pMsg->thandle);
if (mgmtCheckRedirect(pMsg->thandle)) return;
if (!pMsg->pUser->writeAuth) {
mTrace("table:%s, failed to alter table, no rights", pAlter->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
return;
}
pMsg->pDb = mgmtGetDbByTableId(pAlter->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->dirty) {
mError("table:%s, failed to alter table, db not selected", pAlter->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
return;
}
if (mgmtCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) {
mError("table:%s, failed to alter table, its log db", pAlter->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN);
return;
}
SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg));
memcpy(newMsg, pMsg, sizeof(SQueuedMsg));
pMsg->pCont = NULL;
int32_t code;
switch (pTable->type) {
case TSDB_SUPER_TABLE:
mTrace("table:%s, start to drop super table", pDrop->tableId);
code = mgmtDropSuperTable(newMsg, pDb, (SSuperTableObj *) pTable);
break;
case TSDB_CHILD_TABLE:
mTrace("table:%s, start to drop child table", pDrop->tableId);
code = mgmtDropChildTable(newMsg, (SChildTableObj *) pTable);
break;
case TSDB_NORMAL_TABLE:
mTrace("table:%s, start to drop normal table", pDrop->tableId);
code = mgmtDropNormalTable(newMsg, (SNormalTableObj *) pTable);
break;
case TSDB_STREAM_TABLE:
mTrace("table:%s, start to drop stream table", pDrop->tableId);
code = mgmtDropNormalTable(newMsg, (SNormalTableObj *) pTable);
break;
default:
code = TSDB_CODE_INVALID_TABLE_TYPE;
mError("table:%s, invalid table type:%d", pDrop->tableId, pTable->type);
}
if (code != TSDB_CODE_SUCCESS) {
free(newMsg);
mgmtSendSimpleResp(pMsg->thandle, code);
}
}
void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
if (mgmtCheckRedirect(pMsg->thandle)) {
STableInfo *pTable = mgmtGetTable(pAlter->tableId);
if (pTable == NULL) {
mError("table:%s, failed to alter table, table not exist", pTable->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
return;
}
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
if (pUser == NULL) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER);
pAlter->numOfCols = htons(pAlter->numOfCols);
if (pAlter->numOfCols > 2) {
mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_APP_ERROR);
return;
}
SCMAlterTableMsg *pAlter = pMsg->pCont;
for (int32_t i = 0; i < pAlter->numOfCols; ++i) {
pAlter->schema[i].bytes = htons(pAlter->schema[i].bytes);
}
int32_t code;
if (!pUser->writeAuth) {
code = TSDB_CODE_NO_RIGHTS;
if (pTable->type == TSDB_SUPER_TABLE) {
mTrace("table:%s, start to alter stable", pAlter->tableId);
mgmtAlterSuperTable(pMsg, (SSuperTableObj *)pTable);
} else {
pAlter->type = htons(pAlter->type);
pAlter->numOfCols = htons(pAlter->numOfCols);
if (pAlter->numOfCols > 2) {
mError("table:%s error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols);
code = TSDB_CODE_APP_ERROR;
} else {
SDbObj *pDb = mgmtGetDb(pAlter->db);
if (pDb) {
for (int32_t i = 0; i < pAlter->numOfCols; ++i) {
pAlter->schema[i].bytes = htons(pAlter->schema[i].bytes);
}
code = mgmtAlterTable(pDb, pAlter);
if (code == 0) {
mLPrint("table:%s is altered by %s", pAlter->tableId, pUser->user);
}
} else {
code = TSDB_CODE_DB_NOT_SELECTED;
}
}
mTrace("table:%s, start to alter ctable", pAlter->tableId);
mgmtAlterChildTable(pMsg, (SChildTableObj *)pTable);
}
mgmtSendSimpleResp(pMsg->thandle, code);
}
void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle) {
SRpcMsg rpcRsp = {.handle = thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
SDbObj* pDb = mgmtGetDbByTableId(pTable->tableId);
if (pDb == NULL || pDb->dirty) {
mError("table:%s, failed to get table meta, db not selected", pTable->tableId);
rpcRsp.code = TSDB_CODE_DB_NOT_SELECTED;
rpcSendResponse(&rpcRsp);
return;
}
SRpcConnInfo connInfo;
if (rpcGetConnInfo(thandle, &connInfo) != 0) {
mError("conn:%p is already released while get table meta", thandle);
return;
}
bool usePublicIp = (connInfo.serverIp == tsPublicIpInt);
STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS);
rpcRsp.code = mgmtGetTableMeta(pDb, pTable, pMeta, usePublicIp);
if (rpcRsp.code != TSDB_CODE_SUCCESS) {
rpcFreeCont(pMeta);
} else {
rpcRsp.pCont = pMeta;
rpcRsp.contLen = pMeta->contLen;
pMeta->contLen = htons(pMeta->contLen);
}
rpcSendResponse(&rpcRsp);
}
void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) {
static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) {
SCMTableInfoMsg *pInfo = pMsg->pCont;
pInfo->createFlag = htons(pInfo->createFlag);
mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle);
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
if (pUser == NULL) {
mError("table:%s, failed to get table meta, invalid user", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER);
pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->dirty) {
mError("table:%s, failed to get table meta, db not selected", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
return;
}
STableInfo *pTable = mgmtGetTable(pInfo->tableId);
if (pTable == NULL) {
if (pInfo->createFlag != 1) {
mError("table:%s, failed to get table meta, table not exist", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
return;
} else {
// on demand create table from super table if table does not exists
if (mgmtCheckRedirect(pMsg->thandle)) {
mError("table:%s, failed to create table while get meta info, need redirect message", pInfo->tableId);
return;
}
int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData);
SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
if (pCreateMsg == NULL) {
mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
return;
}
memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData));
strcpy(pCreateMsg->tableId, pInfo->tableId);
mError("table:%s, start to create table while get meta info", pInfo->tableId);
// mgmtCreateTable(pCreateMsg, contLen, pMsg->thandle, true);
}
if (pTable == NULL || pTable->type != TSDB_SUPER_TABLE) {
mgmtGetChildTableMeta(pMsg, (SChildTableObj *)pTable);
} else {
mgmtProcessGetTableMeta(pTable, pMsg->thandle);
mgmtGetSuperTableMeta(pMsg, (SSuperTableObj *)pTable);
}
}
void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) {
SRpcConnInfo connInfo;
if (rpcGetConnInfo(pMsg->thandle, &connInfo) != 0) {
mError("conn:%p is already released while get mulit table meta", pMsg->thandle);
return;
}
bool usePublicIp = (connInfo.serverIp == tsPublicIpInt);
SUserObj *pUser = mgmtGetUser(connInfo.user);
if (pUser == NULL) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER);
return;
}
SCMMultiTableInfoMsg *pInfo = pMsg->pCont;
pInfo->numOfTables = htonl(pInfo->numOfTables);
int32_t totalMallocLen = 4*1024*1024; // first malloc 4 MB, subsequent reallocation as twice
SMultiTableMeta *pMultiMeta = rpcMallocCont(totalMallocLen);
if (pMultiMeta == NULL) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
return;
}
pMultiMeta->contLen = sizeof(SMultiTableMeta);
pMultiMeta->numOfTables = 0;
for (int t = 0; t < pInfo->numOfTables; ++t) {
char *tableId = (char*)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN);
STableInfo *pTable = mgmtGetTable(tableId);
if (pTable == NULL) continue;
SDbObj *pDb = mgmtGetDbByTableId(tableId);
if (pDb == NULL) continue;
int availLen = totalMallocLen - pMultiMeta->contLen;
if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS) {
//TODO realloc
//totalMallocLen *= 2;
//pMultiMeta = rpcReMalloc(pMultiMeta, totalMallocLen);
//if (pMultiMeta == NULL) {
/// rpcSendResponse(ahandle, TSDB_CODE_SERV_OUT_OF_MEMORY, NULL, 0);
// return TSDB_CODE_SERV_OUT_OF_MEMORY;
//} else {
// t--;
// continue;
//}
}
STableMetaMsg *pMeta = (STableMetaMsg *)(pMultiMeta->metas + pMultiMeta->contLen);
int32_t code = mgmtGetTableMeta(pDb, pTable, pMeta, usePublicIp);
if (code == TSDB_CODE_SUCCESS) {
pMultiMeta->numOfTables ++;
pMultiMeta->contLen += pMeta->contLen;
}
}
SRpcMsg rpcRsp = {0};
rpcRsp.handle = pMsg->thandle;
rpcRsp.pCont = pMultiMeta;
rpcRsp.contLen = pMultiMeta->contLen;
rpcSendResponse(&rpcRsp);
}
void mgmtProcessSuperTableMetaMsg(SQueuedMsg *pMsg) {
SCMSuperTableInfoMsg *pInfo = pMsg->pCont;
STableInfo *pTable = mgmtGetSuperTable(pInfo->tableId);
if (pTable == NULL) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
return;
}
SCMSuperTableInfoRsp *pRsp = mgmtGetSuperTableVgroup((SSuperTableObj *) pTable);
if (pRsp != NULL) {
int32_t msgLen = sizeof(SSuperTableObj) + htonl(pRsp->numOfDnodes) * sizeof(int32_t);
SRpcMsg rpcRsp = {0};
rpcRsp.handle = pMsg->thandle;
rpcRsp.pCont = pRsp;
rpcRsp.contLen = msgLen;
rpcSendResponse(&rpcRsp);
} else {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
}
}
static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) {
if (rpcMsg->handle == NULL) return;
SQueuedMsg *queueMsg = rpcMsg->handle;
queueMsg->received++;
STableInfo *pTable = queueMsg->ahandle;
mTrace("table:%s, create table rsp received, thandle:%p ahandle:%p result:%s", pTable->tableId, queueMsg->thandle,
rpcMsg->handle, tstrerror(rpcMsg->code));
if (rpcMsg->code != TSDB_CODE_SUCCESS) {
if (pTable->type == TSDB_CHILD_TABLE) {
sdbDeleteRow(tsChildTableSdb, pTable);
} else if (pTable->type == TSDB_NORMAL_TABLE){
sdbDeleteRow(tsNormalTableSdb, pTable);
} else {}
mError("table:%s, failed to create in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code));
mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
} else {
mTrace("table:%s, created in dnode", pTable->tableId);
if (queueMsg->msgType != TSDB_MSG_TYPE_CM_CREATE_TABLE) {
SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg));
newMsg->msgType = queueMsg->msgType;
newMsg->thandle = queueMsg->thandle;
newMsg->pDb = queueMsg->pDb;
newMsg->pUser = queueMsg->pUser;
newMsg->contLen = queueMsg->contLen;
newMsg->pCont = rpcMallocCont(newMsg->contLen);
memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen);
mTrace("table:%s, start to get meta", pTable->tableId);
mgmtAddToShellQueue(newMsg);
} else {
mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
}
}
free(queueMsg);
}
static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg) {
mTrace("alter table rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code);
}
static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) {
if (rpcMsg->handle == NULL) return;
SQueuedMsg *queueMsg = rpcMsg->handle;
queueMsg->received++;
STableInfo *pTable = queueMsg->ahandle;
mTrace("table:%s, drop table rsp received, thandle:%p result:%s", pTable->tableId, queueMsg->thandle, tstrerror(rpcMsg->code));
if (rpcMsg->code != TSDB_CODE_SUCCESS) {
mError("table:%s, failed to drop in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code));
mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
free(queueMsg);
return;
}
SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
mError("table:%s, failed to get vgroup", pTable->tableId);
mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_INVALID_VGROUP_ID);
free(queueMsg);
return;
}
if (pTable->type == TSDB_CHILD_TABLE) {
if (sdbDeleteRow(tsChildTableSdb, pTable) < 0) {
mError("table:%s, update ctables sdb error", pTable->tableId);
mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR);
free(queueMsg);
return;
}
} else if (pTable->type == TSDB_NORMAL_TABLE){
if (sdbDeleteRow(tsNormalTableSdb, pTable) < 0) {
mError("table:%s, update ntables sdb error", pTable->tableId);
mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR);
free(queueMsg);
return;
}
}
if (pVgroup->numOfTables <= 0) {
mPrint("vgroup:%d, all tables is dropped, drop vgroup", pVgroup->vgId);
mgmtDropVgroup(pVgroup, NULL);
}
mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SUCCESS);
free(queueMsg);
}
static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) {
mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code);
}
//
//
//static void mgmtProcessTableCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *thandle) {
// SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) pCont;
// pCfg->dnode = htonl(pCfg->dnode);
// pCfg->vnode = htonl(pCfg->vnode);
// pCfg->sid = htonl(pCfg->sid);
// mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid);
//
// if (!sdbMaster) {
// mError("dnode:%s, vnode:%d, sid:%d, not master, redirect it", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid);
// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0);
// return;
// }
//
// STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid);
// if (pTable == NULL) {
// mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid);
// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_INVALID_TABLE, NULL, 0);
// return;
// }
//
// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0);
//
// //TODO
// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode);
// mgmtSendCreateTableMsg(NULL, &ipSet, NULL);
//}
//
}

258
src/mnode/src/mgmtUser.c
View File

@ -17,67 +17,105 @@
#include "os.h"
#include "trpc.h"
#include "ttime.h"
#include "tutil.h"
#include "mgmtAcct.h"
#include "mgmtGrant.h"
#include "mgmtMnode.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
#include "mgmtUser.h"
void *tsUserSdb = NULL;
static void *tsUserSdb = NULL;
static int32_t tsUserUpdateSize = 0;
static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass);
static int32_t mgmtDropUser(SAcctObj *pAcct, char *name);
static int32_t mgmtUpdateUser(SUserObj *pUser);
static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass);
static int32_t mgmtDropUser(SAcctObj *pAcct, char *name);
static int32_t mgmtUpdateUser(SUserObj *pUser);
static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg);
static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg);
static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg);
static void *(*mgmtUserActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize);
static void mgmtUserActionInit();
static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) {
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) {
SUserObj *pUser = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
if (pAcct != NULL) {
mgmtAddUserIntoAcct(pAcct, pUser);
}
else {
mError("user:%s, acct:%s info not exist in sdb", pUser->user, pUser->acct);
return TSDB_CODE_INVALID_ACCT;
}
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) {
SUserObj *pUser = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
mgmtRemoveUserFromAcct(pAcct, pUser);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) {
SUserObj *pUser = pOper->pObj;
if (pOper->maxRowSize < tsUserUpdateSize) {
return -1;
} else {
memcpy(pOper->rowData, pUser, tsUserUpdateSize);
pOper->rowSize = tsUserUpdateSize;
return TSDB_CODE_SUCCESS;
}
}
static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) {
SUserObj *pUser = (SUserObj *) calloc(1, sizeof(SUserObj));
if (pUser == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
memcpy(pUser, pOper->rowData, tsUserUpdateSize);
pOper->pObj = pUser;
return TSDB_CODE_SUCCESS;
}
int32_t mgmtInitUsers() {
void *pNode = NULL;
SUserObj *pUser = NULL;
SAcctObj *pAcct = NULL;
int32_t numOfUsers = 0;
mgmtUserActionInit();
SUserObj tObj;
tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj;
tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
tsUserSdb = sdbOpenTable(tsMaxUsers, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtUserAction);
SSdbTableDesc tableDesc = {
.tableName = "users",
.hashSessions = TSDB_MAX_USERS,
.maxRowSize = tsUserUpdateSize,
.keyType = SDB_KEY_TYPE_STRING,
.insertFp = mgmtUserActionInsert,
.deleteFp = mgmtUserActionDelete,
.updateFp = mgmtUserActionUpdate,
.encodeFp = mgmtUserActionEncode,
.decodeFp = mgmtUserActionDecode,
.destroyFp = mgmtUserActionDestroy,
};
tsUserSdb = sdbOpenTable(&tableDesc);
if (tsUserSdb == NULL) {
mError("failed to init user data");
return -1;
}
while (1) {
pNode = sdbFetchRow(tsUserSdb, pNode, (void **)&pUser);
if (pUser == NULL) break;
pUser->prev = NULL;
pUser->next = NULL;
pAcct = mgmtGetAcct(pUser->acct);
mgmtAddUserIntoAcct(pAcct, pUser);
numOfUsers++;
}
pAcct = mgmtGetAcct("root");
SAcctObj *pAcct = mgmtGetAcct("root");
mgmtCreateUser(pAcct, "root", "taosdata");
mgmtCreateUser(pAcct, "monitor", tsInternalPass);
mgmtCreateUser(pAcct, "_root", tsInternalPass);
@ -101,16 +139,23 @@ SUserObj *mgmtGetUser(char *name) {
}
static int32_t mgmtUpdateUser(SUserObj *pUser) {
return sdbUpdateRow(tsUserSdb, pUser, 0, 1);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsUserSdb,
.pObj = pUser,
.rowSize = tsUserUpdateSize
};
int32_t code = sdbUpdateRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
tfree(pUser);
code = TSDB_CODE_SDB_ERROR;
}
return code;
}
static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
int32_t numOfUsers = sdbGetNumOfRows(tsUserSdb);
if (numOfUsers >= tsMaxUsers) {
mWarn("numOfUsers:%d, exceed tsMaxUsers:%d", numOfUsers, tsMaxUsers);
return TSDB_CODE_TOO_MANY_USERS;
}
int32_t code = mgmtCheckUserLimit(pAcct);
if (code != 0) {
return code;
@ -131,8 +176,7 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
return code;
}
pUser = malloc(sizeof(SUserObj));
memset(pUser, 0, sizeof(SUserObj));
pUser = calloc(1, sizeof(SUserObj));
strcpy(pUser->user, name);
taosEncryptPass((uint8_t*) pass, strlen(pass), pUser->pass);
strcpy(pUser->acct, pAcct->user);
@ -143,8 +187,15 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
pUser->superAuth = 1;
}
code = TSDB_CODE_SUCCESS;
if (sdbInsertRow(tsUserSdb, pUser, 0) < 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsUserSdb,
.pObj = pUser,
.rowSize = sizeof(SUserObj)
};
code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
tfree(pUser);
code = TSDB_CODE_SDB_ERROR;
}
@ -165,13 +216,22 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) {
return TSDB_CODE_NO_RIGHTS;
}
sdbDeleteRow(tsUserSdb, pUser);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsUserSdb,
.pObj = pUser
};
return 0;
int32_t code = sdbDeleteRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_SDB_ERROR;
}
return code;
}
static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SUserObj *pUser = mgmtGetUserFromConn(pConn);
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) {
return TSDB_CODE_INVALID_USER;
}
@ -250,86 +310,12 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
return numOfRows;
}
static void mgmtUserActionInit() {
mgmtUserActionFp[SDB_TYPE_INSERT] = mgmtUserActionInsert;
mgmtUserActionFp[SDB_TYPE_DELETE] = mgmtUserActionDelete;
mgmtUserActionFp[SDB_TYPE_UPDATE] = mgmtUserActionUpdate;
mgmtUserActionFp[SDB_TYPE_ENCODE] = mgmtUserActionEncode;
mgmtUserActionFp[SDB_TYPE_DECODE] = mgmtUserActionDecode;
mgmtUserActionFp[SDB_TYPE_RESET] = mgmtUserActionReset;
mgmtUserActionFp[SDB_TYPE_DESTROY] = mgmtUserActionDestroy;
}
static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize) {
if (mgmtUserActionFp[(uint8_t) action] != NULL) {
return (*(mgmtUserActionFp[(uint8_t) action]))(row, str, size, ssize);
}
return NULL;
}
static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize) {
SUserObj *pUser = (SUserObj *) row;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
pUser->pAcct = pAcct;
mgmtAddUserIntoAcct(pAcct, pUser);
return NULL;
}
static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize) {
SUserObj *pUser = (SUserObj *) row;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
mgmtRemoveUserFromAcct(pAcct, pUser);
return NULL;
}
static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) {
return mgmtUserActionReset(row, str, size, ssize);
}
static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize) {
SUserObj *pUser = (SUserObj *) row;
if (size < tsUserUpdateSize) {
*ssize = -1;
} else {
memcpy(str, pUser, tsUserUpdateSize);
*ssize = tsUserUpdateSize;
}
return NULL;
}
static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize) {
SUserObj *pUser = (SUserObj *) malloc(sizeof(SUserObj));
if (pUser == NULL) return NULL;
memset(pUser, 0, sizeof(SUserObj));
memcpy(pUser, str, tsUserUpdateSize);
return (void *)pUser;
}
static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize) {
SUserObj *pUser = (SUserObj *)row;
memcpy(pUser, str, tsUserUpdateSize);
return NULL;
}
static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) {
tfree(row);
return NULL;
}
SUserObj *mgmtGetUserFromConn(void *pConn) {
SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) {
SRpcConnInfo connInfo;
if (rpcGetConnInfo(pConn, &connInfo) == 0) {
if (usePublicIp) {
*usePublicIp = (connInfo.serverIp == tsPublicIpInt);
}
return mgmtGetUser(connInfo.user);
}
@ -393,7 +379,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
memset(pUser->pass, 0, sizeof(pUser->pass));
taosEncryptPass((uint8_t*)pAlter->pass, strlen(pAlter->pass), pUser->pass);
code = mgmtUpdateUser(pUser);
mLPrint("user:%s password is altered by %s, code:%d", pAlter->user, pUser->user, code);
mLPrint("user:%s password is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code));
} else {
code = TSDB_CODE_NO_RIGHTS;
}
@ -428,10 +414,6 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
}
if (hasRight) {
//if (pAlter->privilege == 1) { // super
// pUser->superAuth = 1;
// pUser->writeAuth = 1;
//}
if (pAlter->privilege == 2) { // read
pUser->superAuth = 0;
pUser->writeAuth = 0;
@ -442,7 +424,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
}
code = mgmtUpdateUser(pUser);
mLPrint("user:%s privilege is altered by %s, code:%d", pAlter->user, pUser->user, code);
mLPrint("user:%s privilege is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code));
} else {
code = TSDB_CODE_NO_RIGHTS;
}
@ -492,7 +474,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) {
if (hasRight) {
code = mgmtDropUser(pUser->pAcct, pDrop->user);
if (code == TSDB_CODE_SUCCESS) {
mLPrint("user:%s is dropped by %s", pDrop->user, pUser->user);
mLPrint("user:%s is dropped by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code));
}
} else {
code = TSDB_CODE_NO_RIGHTS;

485
src/mnode/src/mgmtVgroup.c
View File

@ -20,107 +20,156 @@
#include "tstatus.h"
#include "mnode.h"
#include "mgmtBalance.h"
#include "mgmtChildTable.h"
#include "mgmtDb.h"
#include "mgmtDClient.h"
#include "mgmtDnode.h"
#include "mgmtDServer.h"
#include "mgmtMnode.h"
#include "mgmtProfile.h"
#include "mgmtSdb.h"
#include "mgmtShell.h"
#include "mgmtTable.h"
#include "mgmtVgroup.h"
static void *tsVgroupSdb = NULL;
static void *tsVgroupSdb = NULL;
static int32_t tsVgUpdateSize = 0;
static void *(*mgmtVgroupActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize);
static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize);
static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg);
static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg);
static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) ;
static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle);
static void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
static void mgmtVgroupActionInit() {
SVgObj tObj;
tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj;
mgmtVgroupActionFp[SDB_TYPE_INSERT] = mgmtVgroupActionInsert;
mgmtVgroupActionFp[SDB_TYPE_DELETE] = mgmtVgroupActionDelete;
mgmtVgroupActionFp[SDB_TYPE_UPDATE] = mgmtVgroupActionUpdate;
mgmtVgroupActionFp[SDB_TYPE_ENCODE] = mgmtVgroupActionEncode;
mgmtVgroupActionFp[SDB_TYPE_DECODE] = mgmtVgroupActionDecode;
mgmtVgroupActionFp[SDB_TYPE_RESET] = mgmtVgroupActionReset;
mgmtVgroupActionFp[SDB_TYPE_DESTROY] = mgmtVgroupActionDestroy;
}
static void *mgmtVgroupAction(char action, void *row, char *str, int32_t size, int32_t *ssize) {
if (mgmtVgroupActionFp[(uint8_t) action] != NULL) {
return (*(mgmtVgroupActionFp[(uint8_t) action]))(row, str, size, ssize);
static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) {
SVgObj *pVgroup = pOper->pObj;
if (pVgroup->idPool) {
taosIdPoolCleanUp(pVgroup->idPool);
pVgroup->idPool = NULL;
}
return NULL;
if (pVgroup->tableList) {
tfree(pVgroup->tableList);
}
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
int32_t mgmtInitVgroups() {
void *pNode = NULL;
SVgObj *pVgroup = NULL;
static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) {
SVgObj *pVgroup = pOper->pObj;
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
if (pDb == NULL) {
return TSDB_CODE_INVALID_DB;
}
mgmtVgroupActionInit();
pVgroup->pDb = pDb;
pVgroup->prev = NULL;
pVgroup->next = NULL;
tsVgroupSdb = sdbOpenTable(tsMaxVGroups, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMnodeDir, mgmtVgroupAction);
if (tsVgroupSdb == NULL) {
mError("failed to init vgroups data");
int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxSessions;
pVgroup->tableList = calloc(pDb->cfg.maxSessions, sizeof(SChildTableObj *));
if (pVgroup->tableList == NULL) {
mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size);
return -1;
}
while (1) {
pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
if (pVgroup == NULL) break;
pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions);
if (pVgroup->idPool == NULL) {
mError("vgroup:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId);
tfree(pVgroup->tableList);
return -1;
}
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
if (pDb == NULL) continue;
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId);
pVgroup->vnodeGid[i].privateIp = pDnode->privateIp;
pVgroup->vnodeGid[i].publicIp = pDnode->publicIp;
pVgroup->vnodeGid[i].vnode = pVgroup->vgId;
}
pVgroup->prev = NULL;
pVgroup->next = NULL;
mgmtAddVgroupIntoDb(pVgroup);
int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions;
pVgroup->tableList = (STableInfo **)malloc(size);
if (pVgroup->tableList == NULL) {
mError("failed to malloc(size:%d) for the tableList of vgroups", size);
return -1;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) {
SVgObj *pVgroup = pOper->pObj;
if (pVgroup->pDb != NULL) {
mgmtRemoveVgroupFromDb(pVgroup);
}
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) {
SVgObj *pVgroup = pOper->pObj;
int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool);
SDbObj *pDb = pVgroup->pDb;
if (pDb != NULL) {
if (pDb->cfg.maxSessions != oldTables) {
mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions);
taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions);
int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxSessions;
pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
}
memset(pVgroup->tableList, 0, size);
}
pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions);
if (pVgroup->idPool == NULL) {
mError("failed to taosInitIdPool for vgroups");
free(pVgroup->tableList);
return -1;
}
taosIdPoolReinit(pVgroup->idPool);
mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxSessions, pVgroup->numOfVnodes);
return TSDB_CODE_SUCCESS;
}
if (tsIsCluster && pVgroup->vnodeGid[0].publicIp == 0) {
pVgroup->vnodeGid[0].publicIp = inet_addr(tsPublicIp);
pVgroup->vnodeGid[0].privateIp = inet_addr(tsPrivateIp);
sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1);
}
static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) {
SVgObj *pVgroup = pOper->pObj;
if (pOper->maxRowSize < tsVgUpdateSize) {
return -1;
} else {
memcpy(pOper->rowData, pVgroup, tsVgUpdateSize);
pOper->rowSize = tsVgUpdateSize;
return TSDB_CODE_SUCCESS;
}
}
// mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId);
static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) {
SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj));
if (pVgroup == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
memcpy(pVgroup, pOper->rowData, tsVgUpdateSize);
pOper->pObj = pVgroup;
return TSDB_CODE_SUCCESS;
}
int32_t mgmtInitVgroups() {
SVgObj tObj;
tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableName = "vgroups",
.hashSessions = TSDB_MAX_VGROUPS,
.maxRowSize = tsVgUpdateSize,
.keyType = SDB_KEY_TYPE_AUTO,
.insertFp = mgmtVgroupActionInsert,
.deleteFp = mgmtVgroupActionDelete,
.updateFp = mgmtVgroupActionUpdate,
.encodeFp = mgmtVgroupActionEncode,
.decodeFp = mgmtVgroupActionDecode,
.destroyFp = mgmtVgroupActionDestroy,
};
tsVgroupSdb = sdbOpenTable(&tableDesc);
if (tsVgroupSdb == NULL) {
mError("failed to init vgroups data");
return -1;
}
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mgmtGetVgroupMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mgmtRetrieveVgroups);
mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mgmtProcessCreateVnodeRsp);
mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mgmtProcessDropVnodeRsp);
mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg);
mTrace("vgroup is initialized");
return 0;
@ -139,32 +188,41 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg) {
if (pDb == NULL) {
mError("failed to create vgroup, db not found");
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB);
mgmtFreeQueuedMsg(pMsg);
return;
}
SVgObj *pVgroup = (SVgObj *)calloc(sizeof(SVgObj), 1);
SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj));
strcpy(pVgroup->dbName, pDb->name);
pVgroup->numOfVnodes = pDb->cfg.replications;
pVgroup->createdTime = taosGetTimestampMs();
if (mgmtAllocVnodes(pVgroup) != 0) {
mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes);
free(pVgroup);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES);
mgmtFreeQueuedMsg(pMsg);
return;
}
pVgroup->createdTime = taosGetTimestampMs();
pVgroup->tableList = (STableInfo **) calloc(sizeof(STableInfo *), pDb->cfg.maxSessions);
pVgroup->numOfTables = 0;
pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
.rowSize = sizeof(SVgObj)
};
mgmtAddVgroupIntoDb(pDb, pVgroup);
// mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId);
sdbInsertRow(tsVgroupSdb, pVgroup, 0);
int32_t code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
tfree(pVgroup);
code = TSDB_CODE_SDB_ERROR;
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SDB_ERROR);
mgmtFreeQueuedMsg(pMsg);
return;
}
mPrint("vgroup:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
mPrint("vgroup:%d, dnode:%d vnode:%d", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode);
mPrint("vgroup:%d, index:%d, dnode:%d vnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode);
}
pMsg->ahandle = pVgroup;
@ -178,28 +236,12 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) {
} else {
mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
mgmtSendDropVgroupMsg(pVgroup, NULL);
sdbDeleteRow(tsVgroupSdb, pVgroup);
}
}
void mgmtSetVgroupIdPool() {
void * pNode = NULL;
SVgObj *pVgroup = NULL;
SDbObj *pDb;
while (1) {
pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
if (pVgroup == NULL || pVgroup->idPool == 0) break;
taosIdPoolSetFreeList(pVgroup->idPool);
pVgroup->numOfTables = taosIdPoolNumOfUsed(pVgroup->idPool);
pDb = mgmtGetDb(pVgroup->dbName);
pDb->numOfTables += pVgroup->numOfTables;
if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1)
mgmtAddVgroupIntoDbTail(pDb, pVgroup);
else
mgmtAddVgroupIntoDb(pDb, pVgroup);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
};
sdbDeleteRow(&oper);
}
}
@ -230,15 +272,15 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
pShow->bytes[cols] = 9;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "vgroup_status");
strcpy(pSchema[cols].name, "vgroup status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
int32_t maxReplica = 0;
SVgObj *pVgroup = NULL;
STableInfo *pTable = NULL;
SChildTableObj *pTable = NULL;
if (pShow->payloadLen > 0 ) {
pTable = mgmtGetTable(pShow->payload);
pTable = mgmtGetChildTable(pShow->payload);
if (NULL == pTable) {
return TSDB_CODE_INVALID_TABLE_ID;
}
@ -270,13 +312,13 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
pShow->bytes[cols] = 9;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "vnode_status");
strcpy(pSchema[cols].name, "vnode status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "public_ip");
strcpy(pSchema[cols].name, "public ip");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
}
@ -388,109 +430,30 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
return numOfRows;
}
static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize) {
SVgObj *pVgroup = row;
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
pVgroup->vnodeGid[i].vnode = pVgroup->vgId;
}
return NULL;
}
static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize) {
SVgObj *pVgroup = row;
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
if (pDb != NULL) {
mgmtRemoveVgroupFromDb(pDb, pVgroup);
}
// mgmtUnSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes);
tfree(pVgroup->tableList);
return NULL;
}
static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) {
mgmtVgroupActionReset(row, str, size, ssize);
SVgObj *pVgroup = (SVgObj *) row;
int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool);
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
if (pDb != NULL) {
if (pDb->cfg.maxSessions != oldTables) {
mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions);
taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions);
int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions;
pVgroup->tableList = (STableInfo **)realloc(pVgroup->tableList, size);
}
}
mTrace("vgroup:%d update, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes);
return NULL;
}
static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize) {
SVgObj *pVgroup = (SVgObj *) row;
if (size < tsVgUpdateSize) {
*ssize = -1;
} else {
memcpy(str, pVgroup, tsVgUpdateSize);
*ssize = tsVgUpdateSize;
}
return NULL;
}
static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize) {
SVgObj *pVgroup = (SVgObj *) malloc(sizeof(SVgObj));
if (pVgroup == NULL) return NULL;
memset(pVgroup, 0, sizeof(SVgObj));
int32_t tsVgUpdateSize = pVgroup->updateEnd - (int8_t *) pVgroup;
memcpy(pVgroup, str, tsVgUpdateSize);
return (void *) pVgroup;
}
static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize) {
SVgObj *pVgroup = (SVgObj *) row;
memcpy(pVgroup, str, tsVgUpdateSize);
return NULL;
}
static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) {
SVgObj *pVgroup = (SVgObj *) row;
if (pVgroup->idPool) {
taosIdPoolCleanUp(pVgroup->idPool);
pVgroup->idPool = NULL;
}
if (pVgroup->tableList) tfree(pVgroup->tableList);
tfree(row);
return NULL;
}
void mgmtUpdateVgroup(SVgObj *pVgroup) {
sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 0);
}
void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) {
pVgroup->numOfTables++;
if (pTable->sid >= 0)
void mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] == NULL) {
pVgroup->tableList[pTable->sid] = pTable;
taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid);
pVgroup->numOfTables++;
}
if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions)
mgmtAddVgroupIntoDbTail(pVgroup);
}
void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) {
pVgroup->numOfTables--;
if (pTable->sid >= 0)
void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] != NULL) {
pVgroup->tableList[pTable->sid] = NULL;
taosFreeId(pVgroup->idPool, pTable->sid);
taosFreeId(pVgroup->idPool, pTable->sid);
pVgroup->numOfTables--;
}
if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions)
mgmtAddVgroupIntoDbTail(pVgroup);
}
SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) {
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
SDbObj *pDb = pVgroup->pDb;
if (pDb == NULL) return NULL;
SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg));
@ -514,26 +477,13 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) {
SVnodeDesc *vpeerDesc = pVnode->vpeerDesc;
for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {
vpeerDesc[j].vnode = htonl(pVgroup->vnodeGid[j].vnode);
vpeerDesc[j].ip = htonl(pVgroup->vnodeGid[j].privateIp);
}
return pVnode;
}
SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode) {
if (vnode < 0 || vnode >= TSDB_MAX_VNODES) {
return NULL;
}
SDnodeObj *pDnode = mgmtGetDnode(dnode);
if (pDnode == NULL) {
return NULL;
}
int32_t vgId = pDnode->vload[vnode].vgId;
return mgmtGetVgroup(vgId);
}
SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup) {
SRpcIpSet ipSet = {
.numOfIps = pVgroup->numOfVnodes,
@ -603,13 +553,23 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
newMsg->contLen = queueMsg->contLen;
newMsg->pCont = rpcMallocCont(newMsg->contLen);
memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen);
queueMsg->pCont = NULL;
mgmtAddToShellQueue(newMsg);
} else {
sdbDeleteRow(tsVgroupSdb, pVgroup);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
};
int32_t code = sdbDeleteRow(&oper);
if (code != 0) {
code = TSDB_CODE_SDB_ERROR;
}
mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
}
free(queueMsg);
mgmtFreeQueuedMsg(queueMsg);
}
static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) {
@ -659,7 +619,15 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
if (queueMsg->received != queueMsg->expected) return;
sdbDeleteRow(tsVgroupSdb, pVgroup);
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
};
int32_t code = sdbDeleteRow(&oper);
if (code != 0) {
code = TSDB_CODE_SDB_ERROR;
}
SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg));
newMsg->msgType = queueMsg->msgType;
@ -671,50 +639,71 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen);
mgmtAddToShellQueue(newMsg);
free(queueMsg);
queueMsg->pCont = NULL;
mgmtFreeQueuedMsg(queueMsg);
}
void mgmtUpdateVgroupIp(SDnodeObj *pDnode) {
void * pNode = NULL;
static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
if (mgmtCheckRedirect(rpcMsg->handle)) return;
SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
pCfg->vgId = htonl(pCfg->vgId);
SDnodeObj *pDnode = mgmtGetDnode(pCfg->dnodeId);
if (pDnode == NULL) {
mTrace("dnode:%s, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId);
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE);
return;
}
SVgObj *pVgroup = mgmtGetVgroup(pCfg->vgId);
if (pVgroup == NULL) {
mTrace("dnode:%s, vgId:%d, no vgroup info", taosIpStr(pCfg->dnodeId), pCfg->vgId);
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE);
return;
}
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp);
mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL);
}
void mgmtDropAllVgroups(SDbObj *pDropDb) {
void *pNode = NULL;
void *pLastNode = NULL;
int32_t numOfTables = 0;
int32_t dbNameLen = strlen(pDropDb->name);
SVgObj *pVgroup = NULL;
while (1) {
pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
if (pVgroup == NULL) break;
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SVnodeGid *vnodeGid = pVgroup->vnodeGid + i;
if (vnodeGid->dnodeId == pDnode->dnodeId) {
mPrint("vgroup:%d, dnode:%d, privateIp:%s change to %s, publicIp:%s change to %s",
pVgroup->vgId, vnodeGid->dnodeId, pDnode->privateIp, taosIpStr(vnodeGid->privateIp),
pDnode->publicIp, taosIpStr(vnodeGid->publicIp));
vnodeGid->publicIp = pDnode->publicIp;
vnodeGid->privateIp = pDnode->privateIp;
sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1);
}
if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) {
SSdbOperDesc oper = {
.type = SDB_OPER_TYPE_LOCAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
};
sdbDeleteRow(&oper);
pNode = pLastNode;
numOfTables++;
continue;
}
}
mTrace("db:%s, all vgroups is dropped from sdb", pDropDb->name, numOfTables);
}
//static void mgmtProcessVnodeCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *pConn) {
// if (!sdbMaster) {
// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0);
// return;
// }
//
// SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) pCont;
// pCfg->dnode = htonl(pCfg->dnode);
// pCfg->vnode = htonl(pCfg->vnode);
//
// SVgObj *pVgroup = mgmtGetVgroupByVnode(pCfg->dnode, pCfg->vnode);
// if (pVgroup == NULL) {
// mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode);
// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_NOT_ACTIVE_VNODE, NULL, 0);
// return;
// }
//
// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0);
//
// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode);
// mgmtSendCreateVnodeMsg(pVgroup, pCfg->vnode, &ipSet, NULL);
//}
//
void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) {
assert(ahandle != NULL);
if (pVgroup->numOfVnodes != pVgroup->pDb->cfg.replications) {
// TODO:
// mgmtSendAlterVgroupMsg(pVgroup, NULL);
} else {
mgmtAddToShellQueue(ahandle);
}
}

4
src/query/CMakeLists.txt
View File

@ -12,4 +12,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(query ${SRC})
TARGET_LINK_LIBRARIES(query tsdb tutil m rt)
ENDIF ()
ENDIF ()
ADD_SUBDIRECTORY(tests)

4
src/query/inc/qast.h
View File

@ -20,6 +20,7 @@
extern "C" {
#endif
#include <tskiplist.h>
#include "os.h"
#include "taosmsg.h"
@ -93,8 +94,7 @@ void tSQLBinaryExprToString(tSQLBinaryExpr *pExpr, char *dst, int32_t *len);
void tSQLBinaryExprDestroy(tSQLBinaryExpr **pExprs, void (*fp)(void*));
void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExprs, struct tSkipList *pSkipList, tQueryResultset *result,
SBinaryFilterSupp *param);
void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param);
void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
char *(*cb)(void *, char *, int32_t));

2
src/query/inc/qsqltype.h
View File

@ -54,7 +54,7 @@ enum _sql_type {
TSDB_SQL_CONNECT,
TSDB_SQL_USE_DB, // 30
TSDB_SQL_META,
TSDB_SQL_METRIC,
TSDB_SQL_STABLEVGROUP,
TSDB_SQL_MULTI_META,
TSDB_SQL_HB,

8
src/query/inc/qtsbuf.h
View File

@ -48,10 +48,10 @@ typedef struct STSElem {
} STSElem;
typedef struct STSCursor {
int32_t vnodeIndex;
int32_t blockIndex;
int32_t tsIndex;
int32_t order;
int32_t vnodeIndex;
int32_t blockIndex;
int32_t tsIndex;
uint32_t order;
} STSCursor;
typedef struct STSBlock {

61
src/query/inc/queryExecutor.h
View File

@ -32,12 +32,6 @@ typedef struct SData {
char data[];
} SData;
enum {
ST_QUERY_KILLED = 0, // query killed
ST_QUERY_PAUSED = 1, // query paused, due to full of the response buffer
ST_QUERY_COMPLETED = 2, // query completed
};
struct SColumnFilterElem;
typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
@ -60,16 +54,20 @@ typedef struct SWindowStatus {
} SWindowStatus;
typedef struct SWindowResult {
uint16_t numOfRows;
uint16_t numOfRows; // number of rows of current time window
SPosInfo pos; // Position of current result in disk-based output buffer
SResultInfo* resultInfo; // For each result column, there is a resultInfo
STimeWindow window; // The time window that current result covers.
SWindowStatus status;
SWindowStatus status; // this result status: closed or opened
} SWindowResult;
typedef struct SResultRec {
int64_t pointsTotal;
int64_t pointsRead;
int64_t total; // total generated result size in rows
int64_t size; // current result set size in rows
int64_t capacity; // capacity of current result output buffer
// result size threshold in rows. If the result buffer is larger than this, pause query and return to client
int32_t threshold;
} SResultRec;
typedef struct SWindowResInfo {
@ -97,7 +95,6 @@ typedef struct SSingleColumnFilterInfo {
void* pData;
} SSingleColumnFilterInfo;
/* intermediate pos during multimeter query involves interval */
typedef struct STableQueryInfo {
int64_t lastKey;
STimeWindow win;
@ -105,16 +102,15 @@ typedef struct STableQueryInfo {
int16_t queryRangeSet; // denote if the query range is set, only available for interval query
int64_t tag;
STSCursor cur;
int32_t sid; // for retrieve the page id list
int32_t tid; // for retrieve the page id list
SWindowResInfo windowResInfo;
} STableQueryInfo;
typedef struct STableDataInfo {
int32_t numOfBlocks;
int32_t start; // start block index
int32_t start; // start block index
int32_t tableIndex;
void* pMeterObj;
int32_t groupIdx; // group id in table list
STableQueryInfo* pTableQInfo;
} STableDataInfo;
@ -143,7 +139,6 @@ typedef struct SQuery {
int32_t pos;
int64_t pointsOffset; // the number of points offset to save read data
SData** sdata;
int32_t capacity;
SSingleColumnFilterInfo* pFilterInfo;
} SQuery;
@ -170,20 +165,16 @@ typedef struct SQueryRuntimeEnv {
} SQueryRuntimeEnv;
typedef struct SQInfo {
uint64_t signature;
void* signature;
TSKEY startTime;
int64_t elapsedTime;
SResultRec rec;
int32_t pointsReturned;
TSKEY elapsedTime;
int32_t pointsInterpo;
int32_t code; // error code to returned to client
int32_t killed; // denotes if current query is killed
int32_t code; // error code to returned to client
sem_t dataReady;
SArray* pTableIdList; // table list
SArray* pTableIdList; // table id list
SQueryRuntimeEnv runtimeEnv;
int32_t subgroupIdx;
int32_t offset; /* offset in group result set of subgroup */
// tSidSet* pSidSet;
T_REF_DECLARE()
/*
@ -205,7 +196,7 @@ typedef struct SQInfo {
* @param pQInfo
* @return
*/
int32_t qCreateQueryInfo(SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo);
int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo);
/**
* query on single table
@ -213,19 +204,25 @@ int32_t qCreateQueryInfo(SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo);
*/
void qTableQuery(SQInfo* pQInfo);
/**
* query on super table
* @param pReadMsg
*/
void qSuperTableQuery(void* pReadMsg);
/**
* wait for the query completed, and retrieve final results to client
* @param pQInfo
*/
int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo, int32_t *numOfRows, int32_t* rowsize);
int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo);
/**
*
* @param pQInfo
* @param pRsp
* @return
*/
int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* contLen);
//int32_t qBuildQueryResult(SQInfo* pQInfo, void* pBuf);
/**
*
* @param pQInfo
* @return
*/
bool qHasMoreResultsToRetrieve(SQInfo* pQInfo);
#endif // TDENGINE_QUERYEXECUTOR_H

2
src/query/inc/tsqlfunction.h
View File

@ -219,7 +219,7 @@ typedef struct SQLAggFuncElem {
void (*distSecondaryMergeFunc)(SQLFunctionCtx *pCtx);
int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus);
int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId);
} SQLAggFuncElem;
typedef struct SPatternCompareInfo {

228
src/query/src/qast.c
View File

@ -14,6 +14,9 @@
*/
#include "qast.h"
#include <tarray.h>
#include <tskiplist.h>
#include "../../client/inc/tschemautil.h"
#include "os.h"
#include "qsqlparser.h"
#include "qsyntaxtreefunction.h"
@ -105,7 +108,7 @@ static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols,
return NULL;
}
size_t nodeSize = sizeof(tSQLSyntaxNode);
size_t nodeSize = sizeof(tSQLSyntaxNode);
tSQLSyntaxNode *pNode = NULL;
if (pToken->type == TK_ID || pToken->type == TK_TBNAME) {
@ -237,9 +240,7 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tSQLSyntaxNode
}
static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) {
SSQLToken t0;
t0 = tStrGetToken(str, i, false, 0, NULL);
SSQLToken t0 = tStrGetToken(str, i, false, 0, NULL);
if (t0.n == 0) {
return NULL;
}
@ -341,7 +342,8 @@ void tSQLBinaryExprFromString(tSQLBinaryExpr **pExpr, SSchema *pSchema, int32_t
return;
}
int32_t pos = 0;
int32_t pos = 0;
tSQLSyntaxNode *pStxNode = createSyntaxTree(pSchema, numOfCols, src, &pos);
if (pStxNode != NULL) {
assert(pStxNode->nodeType == TSQL_NODE_EXPR);
@ -705,126 +707,120 @@ static bool filterItem(tSQLBinaryExpr *pExpr, const void *pItem, SBinaryFilterSu
* @param pSchema tag schemas
* @param fp filter callback function
*/
static UNUSED_FUNC void tSQLBinaryTraverseOnResult(tSQLBinaryExpr *pExpr, tQueryResultset *pResult, SBinaryFilterSupp *param) {
int32_t n = 0;
for (int32_t i = 0; i < pResult->num; ++i) {
void *pItem = pResult->pRes[i];
static void tSQLBinaryTraverseOnResult(tSQLBinaryExpr *pExpr, SArray *pResult, SBinaryFilterSupp *param) {
size_t size = taosArrayGetSize(pResult);
SArray* array = taosArrayInit(size, POINTER_BYTES);
for (int32_t i = 0; i < size; ++i) {
void *pItem = taosArrayGetP(pResult, i);
if (filterItem(pExpr, pItem, param)) {
pResult->pRes[n++] = pResult->pRes[i];
taosArrayPush(array, &pItem);
}
}
pResult->num = n;
taosArrayCopy(pResult, array);
}
//static void tSQLBinaryTraverseOnSkipList(tSQLBinaryExpr *pExpr, tQueryResultset *pResult, tSkipList *pSkipList,
// SBinaryFilterSupp *param) {
// int32_t n = 0;
// SSkipListIterator iter = {0};
//
// int32_t ret = tSkipListIteratorReset(pSkipList, &iter);
// assert(ret == 0);
//
// pResult->pRes = calloc(pSkipList->nSize, POINTER_BYTES);
//
// while (tSkipListIteratorNext(&iter)) {
// tSkipListNode *pNode = tSkipListIteratorGet(&iter);
// if (filterItem(pExpr, pNode, param)) {
// pResult->pRes[n++] = pNode;
// }
// }
//
// pResult->num = n;
//}
static void tSQLBinaryTraverseOnSkipList(tSQLBinaryExpr *pExpr, SArray *pResult, SSkipList *pSkipList,
SBinaryFilterSupp *param) {
SSkipListIterator* iter = tSkipListCreateIter(pSkipList);
while (tSkipListIterNext(iter)) {
SSkipListNode *pNode = tSkipListIterGet(iter);
if (filterItem(pExpr, pNode, param)) {
taosArrayPush(pResult, SL_GET_NODE_DATA(pNode));
}
}
}
// post-root order traverse syntax tree
//void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, tSkipList *pSkipList, tQueryResultset *result,
// SBinaryFilterSupp *param) {
// if (pExpr == NULL) {
// return;
// }
//
// tSQLSyntaxNode *pLeft = pExpr->pLeft;
// tSQLSyntaxNode *pRight = pExpr->pRight;
//
// // recursive traverse left child branch
// if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) {
// uint8_t weight = pLeft->pExpr->filterOnPrimaryKey + pRight->pExpr->filterOnPrimaryKey;
//
// if (weight == 0 && result->num > 0 && pSkipList == NULL) {
// /**
// * Perform the filter operation based on the initial filter result, which is obtained from filtering from index.
// * Since no index presented, the filter operation is done by scan all elements in the result set.
// *
// * if the query is a high selectivity filter, only small portion of meters are retrieved.
// */
// tSQLBinaryTraverseOnResult(pExpr, result, param);
// } else if (weight == 0) {
// /**
// * apply the hierarchical expression to every node in skiplist for find the qualified nodes
// */
void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) {
if (pExpr == NULL) {
return;
}
tSQLSyntaxNode *pLeft = pExpr->pLeft;
tSQLSyntaxNode *pRight = pExpr->pRight;
// recursive traverse left child branch
if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) {
uint8_t weight = pLeft->pExpr->filterOnPrimaryKey + pRight->pExpr->filterOnPrimaryKey;
if (weight == 0 && taosArrayGetSize(result) > 0 && pSkipList == NULL) {
/**
* Perform the filter operation based on the initial filter result, which is obtained from filtering from index.
* Since no index presented, the filter operation is done by scan all elements in the result set.
*
* if the query is a high selectivity filter, only small portion of meters are retrieved.
*/
tSQLBinaryTraverseOnResult(pExpr, result, param);
} else if (weight == 0) {
/**
* apply the hierarchical expression to every node in skiplist for find the qualified nodes
*/
assert(taosArrayGetSize(result) == 0);
tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param);
} else if (weight == 2 || (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_OR)) {
tQueryResultset rLeft = {0};
tQueryResultset rRight = {0};
tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, &rLeft, param);
tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, &rRight, param);
if (pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) { // CROSS
intersect(&rLeft, &rRight, result);
} else if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or
merge(&rLeft, &rRight, result);
} else {
assert(false);
}
free(rLeft.pRes);
free(rRight.pRes);
} else {
/*
* (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here
*
* first, we filter results based on the skiplist index, which is the initial filter stage,
* then, we conduct the secondary filter operation based on the result from the initial filter stage.
*/
assert(pExpr->nSQLBinaryOptr == TSDB_RELATION_AND);
tSQLBinaryExpr *pFirst = NULL;
tSQLBinaryExpr *pSecond = NULL;
if (pLeft->pExpr->filterOnPrimaryKey == 1) {
pFirst = pLeft->pExpr;
pSecond = pRight->pExpr;
} else {
pFirst = pRight->pExpr;
pSecond = pLeft->pExpr;
}
assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL);
// we filter the result based on the skiplist index in the first place
tSQLBinaryExprTraverse(pFirst, pSkipList, result, param);
/*
* recursively perform the filter operation based on the initial results,
* So, we do not set the skiplist index as a parameter
*/
tSQLBinaryExprTraverse(pSecond, NULL, result, param);
}
} else { // column project
assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE);
param->setupInfoFn(pExpr, param->pExtInfo);
if (pSkipList == NULL) {
tSQLListTraverseOnResult(pExpr, param->fp, result);
} else {
// assert(result->num == 0);
// tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param);
// } else if (weight == 2 || (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_OR)) {
// tQueryResultset rLeft = {0};
// tQueryResultset rRight = {0};
//
// tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, &rLeft, param);
// tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, &rRight, param);
//
// if (pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) { // CROSS
// intersect(&rLeft, &rRight, result);
// } else if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or
// merge(&rLeft, &rRight, result);
// } else {
// assert(false);
// }
//
// free(rLeft.pRes);
// free(rRight.pRes);
// } else {
// /*
// * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here
// *
// * first, we filter results based on the skiplist index, which is the initial filter stage,
// * then, we conduct the secondary filter operation based on the result from the initial filter stage.
// */
// assert(pExpr->nSQLBinaryOptr == TSDB_RELATION_AND);
//
// tSQLBinaryExpr *pFirst = NULL;
// tSQLBinaryExpr *pSecond = NULL;
// if (pLeft->pExpr->filterOnPrimaryKey == 1) {
// pFirst = pLeft->pExpr;
// pSecond = pRight->pExpr;
// } else {
// pFirst = pRight->pExpr;
// pSecond = pLeft->pExpr;
// }
//
// assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL);
//
// // we filter the result based on the skiplist index in the first place
// tSQLBinaryExprTraverse(pFirst, pSkipList, result, param);
//
// /*
// * recursively perform the filter operation based on the initial results,
// * So, we do not set the skiplist index as a parameter
// */
// tSQLBinaryExprTraverse(pSecond, NULL, result, param);
// }
// } else { // column project
// assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE);
//
// param->setupInfoFn(pExpr, param->pExtInfo);
// if (pSkipList == NULL) {
// tSQLListTraverseOnResult(pExpr, param->fp, result);
// } else {
// assert(result->num == 0);
//// tSQLDoFilterInitialResult(pSkipList, param->fp, pExpr->info, result);
// }
// }
//}
// tSQLDoFilterInitialResult(pSkipList, param->fp, pExpr->info, result);
}
}
}
void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
char *(*getSourceDataBlock)(void *, char *, int32_t)) {

1851
src/query/src/queryExecutor.c
View File

File diff suppressed because it is too large Load Diff

15
src/query/tests/CMakeLists.txt Normal file
View File

@ -0,0 +1,15 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest)
FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib)
IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
MESSAGE(STATUS "gTest library found, build unit test")
INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(queryTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread)
ENDIF()

121
src/query/tests/histogramTest.cpp Normal file
View File

@ -0,0 +1,121 @@
#include <gtest/gtest.h>
#include <sys/time.h>
#include <cassert>
#include <iostream>
#include "taos.h"
#include "tsdb.h"
#include "tstoken.h"
#include "tutil.h"
#include "qhistogram.h"
/* test validate the names for table/database */
TEST(testCase, histogram_binary_search) {
SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN);
pHisto->numOfEntries = 10;
for (int32_t i = 0; i < 10; ++i) {
pHisto->elems[i].num = 1;
pHisto->elems[i].val = i;
}
int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1);
assert(idx == 1);
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9);
assert(idx == 9);
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20);
assert(idx == 10);
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1);
assert(idx == 0);
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9);
assert(idx == 4);
free(pHisto);
}
TEST(testCase, histogram_add) {
SHistogramInfo* pHisto = NULL;
/**
* use arrayList, elapsed time is:
* before:
* 10,000,000 45sec, bin:1000 (-O0) / 17sec. bin:1000, (-O3)
*
* after:
*
*/
struct timeval systemTime;
gettimeofday(&systemTime, NULL);
int64_t st =
(int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000;
for (int32_t i = 0; i < 10000; ++i) {
tHistogramAdd(&pHisto, i);
// tHistogramPrint(pHisto);
}
//
gettimeofday(&systemTime, NULL);
int64_t et =
(int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000;
printf("total elapsed time: %ld\n", et - st);
printf("elements: %d, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries);
tHistogramPrint(pHisto);
printf("%ld\n", tHistogramSum(pHisto, 1.5));
printf("%ld\n", tHistogramSum(pHisto, 2));
printf("%ld\n", tHistogramSum(pHisto, 3));
printf("%ld\n", tHistogramSum(pHisto, 4));
printf("%ld\n", tHistogramSum(pHisto, 5));
printf("%ld\n", tHistogramSum(pHisto, 6));
for (int32_t i = 399; i < 400; ++i) {
printf("val:%d, %ld\n", i, tHistogramSum(pHisto, i));
}
double ratio[] = {0 / 100, 20.0 / 100, 88.0 / 100, 100 / 100};
double* res = tHistogramUniform(pHisto, ratio, 4);
for (int32_t i = 0; i < 4; ++i) {
printf("%f\n", res[i]);
}
SHistogramInfo* pHisto1 = NULL;
for (int32_t i = (90000 - 1); i >= 80000; --i) {
tHistogramAdd(&pHisto1, i);
}
tHistogramPrint(pHisto1);
SHistogramInfo* pRes = tHistogramMerge(pHisto1, pHisto, MAX_HISTOGRAM_BIN);
assert(pRes->numOfElems == pHisto->numOfElems + pHisto1->numOfElems);
tHistogramPrint(pRes);
tHistogramDestroy(&pHisto);
tHistogramDestroy(&pHisto1);
tHistogramDestroy(&pRes);
free(res);
}
TEST(testCase, heapsort) {
// int32_t num = 20;
//
// SHeapEntry* pEntry = tHeapCreate(num);
//
// for(int32_t i=0; i<num; ++i) {
// pEntry[i].val = num - 1 - i;
// }
//
// tHeapSort(pEntry, num);
//
// for(int32_t i=0; i<num; ++i) {
// printf("%lf, ", pEntry[i].val);
// }
//
// printf("\n");
//
// free(pEntry);
}

74
src/query/tests/patternMatchTest.cpp Normal file
View File

@ -0,0 +1,74 @@
#include <gtest/gtest.h>
#include <sys/time.h>
#include <cassert>
#include <iostream>
#include "tsqlfunction.h"
TEST(testCase, patternMatchTest) {
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
const char* str = "abcdef";
int32_t ret = patternMatch("a%b%", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "tm01";
ret = patternMatch("tm__", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "tkm1";
ret = patternMatch("t%m1", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "tkm1";
ret = patternMatch("%m1", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "";
ret = patternMatch("%_", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
str = "1";
ret = patternMatch("%__", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
str = "";
ret = patternMatch("%", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = " ";
ret = patternMatch("_", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "!";
ret = patternMatch("%_", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefg";
ret = patternMatch("abc%fg", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("abc%fg", str, 7, &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("abc%f_", str, 6, &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
str = "abcdefgabcdeju";
ret = patternMatch("abc%f_", str, 1, &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH);
str = "abcdefgabcdeju";
ret = patternMatch("ab", str, 2, &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("a%", str, 2, &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("a__", str, 2, &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH);
}

35
src/query/tests/resultBufferTest.cpp Normal file
View File

@ -0,0 +1,35 @@
#include <gtest/gtest.h>
#include <cassert>
#include <iostream>
#include "taos.h"
#include "qresultBuf.h"
#include "tsdb.h"
namespace {
// simple test
void simpleTest() {
SDiskbasedResultBuf* pResultBuf = NULL;
int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64);
int32_t pageId = 0;
int32_t groupId = 0;
tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
ASSERT_TRUE(pBufPage != NULL);
ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64);
ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L);
SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
ASSERT_EQ(list.size, 1);
ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
destroyResultBuf(pResultBuf);
}
} // namespace
TEST(testCase, resultBufferTest) {
simpleTest();
}

451
src/query/tests/tsBufTest.cpp Normal file
View File

@ -0,0 +1,451 @@
#include <gtest/gtest.h>
#include <cassert>
#include <iostream>
#include "taos.h"
#include "tsdb.h"
#include "tstoken.h"
#include "ttime.h"
#include "tutil.h"
#include "qtsbuf.h"
namespace {
/**
*
* @param num total number
* @param step gap between two consecutive ts
* @return
*/
int64_t* createTsList(int32_t num, int64_t start, int32_t step) {
int64_t* pList = (int64_t*)malloc(num * sizeof(int64_t));
for (int64_t i = 0; i < num; ++i) {
pList[i] = start + i * step;
}
return pList;
}
// simple test
void simpleTest() {
STSBuf* pTSBuf = tsBufCreate(true);
// write 10 ts points
int32_t num = 10;
int64_t tag = 1;
int64_t* list = createTsList(10, 10000000, 30);
tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num);
EXPECT_EQ(pTSBuf->block.tag, tag);
EXPECT_EQ(pTSBuf->numOfVnodes, 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf);
}
// one large list of ts, the ts list need to be split into several small blocks
void largeTSTest() {
STSBuf* pTSBuf = tsBufCreate(true);
// write 10 ts points
int32_t num = 1000000;
int64_t tag = 1;
int64_t* list = createTsList(num, 10000000, 30);
tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t));
// the data has been flush to disk, no data in cache
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.tag, tag);
EXPECT_EQ(pTSBuf->numOfVnodes, 1);
EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf);
}
void multiTagsTest() {
STSBuf* pTSBuf = tsBufCreate(true);
int32_t num = 10000;
int64_t tag = 1;
int64_t start = 10000000;
int32_t numOfTags = 50;
int32_t step = 30;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
EXPECT_EQ(pTSBuf->numOfVnodes, 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf);
}
void multiVnodeTagsTest() {
STSBuf* pTSBuf = tsBufCreate(true);
int32_t num = 10000;
int64_t start = 10000000;
int32_t numOfTags = 50;
int32_t step = 30;
// 2000 vnodes
for (int32_t j = 0; j < 20; ++j) {
// vnodeId:0
start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
}
EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf);
}
void loadDataTest() {
STSBuf* pTSBuf = tsBufCreate(true);
int32_t num = 10000;
int64_t oldStart = 10000000;
int32_t numOfTags = 50;
int32_t step = 30;
int32_t numOfVnode = 200;
// 10000 vnodes
for (int32_t j = 0; j < numOfVnode; ++j) {
// vnodeId:0
int64_t start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
printf("%d - %lld\n", i, list[0]);
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
}
EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
// create from exists file
STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false);
EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder);
EXPECT_EQ(pNewBuf->numOfVnodes, numOfVnode);
EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize);
EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset);
EXPECT_EQ(pNewBuf->pData[0].info.numOfBlocks, pTSBuf->pData[0].info.numOfBlocks);
EXPECT_EQ(pNewBuf->pData[0].info.compLen, pTSBuf->pData[0].info.compLen);
EXPECT_STREQ(pNewBuf->path, pTSBuf->path);
tsBufResetPos(pNewBuf);
int64_t s = taosGetTimestampUs();
printf("start:%lld\n", s);
int32_t x = 0;
while (tsBufNextPos(pNewBuf)) {
STSElem elem = tsBufGetElem(pNewBuf);
if (++x == 100000000) {
break;
}
// printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
}
int64_t e = taosGetTimestampUs();
printf("end:%lld, elapsed:%lld, total obj:%d\n", e, e - s, x);
}
void randomIncTsTest() {}
void TSTraverse() {
// 10000 vnodes
int32_t num = 200000;
int64_t oldStart = 10000000;
int32_t numOfTags = 3;
int32_t step = 30;
int32_t numOfVnode = 2;
STSBuf* pTSBuf = tsBufCreate(true);
for (int32_t j = 0; j < numOfVnode; ++j) {
// vnodeId:0
int64_t start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]);
free(list);
start += step * num;
list = createTsList(num, start, step);
tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]);
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
}
tsBufResetPos(pTSBuf);
////////////////////////////////////////////////////////////////////////////////////////
// reverse traverse
int64_t s = taosGetTimestampUs();
printf("start:%lld\n", s);
pTSBuf->cur.order = TSQL_SO_DESC;
// complete reverse traverse
int32_t x = 0;
while (tsBufNextPos(pTSBuf)) {
STSElem elem = tsBufGetElem(pTSBuf);
// printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
}
// specify the data block with vnode and tags value
tsBufResetPos(pTSBuf);
pTSBuf->cur.order = TSQL_SO_DESC;
int32_t startVnode = 1;
int32_t startTag = 2;
tsBufGetElemStartPos(pTSBuf, startVnode, startTag);
int32_t totalOutput = 10;
while (1) {
STSElem elem = tsBufGetElem(pTSBuf);
printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
if (!tsBufNextPos(pTSBuf)) {
break;
}
if (--totalOutput <= 0) {
totalOutput = 10;
tsBufGetElemStartPos(pTSBuf, startVnode, --startTag);
if (startTag == 0) {
startVnode -= 1;
startTag = 3;
}
if (startVnode < 0) {
break;
}
}
}
/////////////////////////////////////////////////////////////////////////////////
// traverse
pTSBuf->cur.order = TSQL_SO_ASC;
tsBufResetPos(pTSBuf);
// complete forwards traverse
while (tsBufNextPos(pTSBuf)) {
STSElem elem = tsBufGetElem(pTSBuf);
// printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
}
// specify the data block with vnode and tags value
tsBufResetPos(pTSBuf);
pTSBuf->cur.order = TSQL_SO_ASC;
startVnode = 1;
startTag = 2;
tsBufGetElemStartPos(pTSBuf, startVnode, startTag);
totalOutput = 10;
while (1) {
STSElem elem = tsBufGetElem(pTSBuf);
printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
if (!tsBufNextPos(pTSBuf)) {
break;
}
if (--totalOutput <= 0) {
totalOutput = 10;
tsBufGetElemStartPos(pTSBuf, startVnode, --startTag);
if (startTag < 0) {
startVnode -= 1;
startTag = 3;
}
if (startVnode < 0) {
break;
}
}
}
}
void performanceTest() {}
void emptyTagTest() {}
void invalidFileTest() {
const char* cmd = "touch /tmp/test";
// create empty file
system(cmd);
STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true);
EXPECT_TRUE(pNewBuf == NULL);
pNewBuf = tsBufCreateFromFile("/tmp/911", true);
EXPECT_TRUE(pNewBuf == NULL);
}
void mergeDiffVnodeBufferTest() {
STSBuf* pTSBuf1 = tsBufCreate(true);
STSBuf* pTSBuf2 = tsBufCreate(true);
int32_t step = 30;
int32_t num = 1000;
int32_t numOfTags = 10;
// vnodeId:0
int64_t start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t));
tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
tsBufFlush(pTSBuf2);
tsBufMerge(pTSBuf1, pTSBuf2, 9);
EXPECT_EQ(pTSBuf1->numOfVnodes, 2);
EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
tsBufDisplay(pTSBuf1);
tsBufDestory(pTSBuf2);
tsBufDestory(pTSBuf1);
}
void mergeIdenticalVnodeBufferTest() {
STSBuf* pTSBuf1 = tsBufCreate(true);
STSBuf* pTSBuf2 = tsBufCreate(true);
int32_t step = 30;
int32_t num = 1000;
int32_t numOfTags = 10;
// vnodeId:0
int64_t start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
for (int32_t i = numOfTags; i < numOfTags * 2; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
tsBufFlush(pTSBuf2);
tsBufMerge(pTSBuf1, pTSBuf2, 12);
EXPECT_EQ(pTSBuf1->numOfVnodes, 1);
EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
tsBufResetPos(pTSBuf1);
while (tsBufNextPos(pTSBuf1)) {
STSElem elem = tsBufGetElem(pTSBuf1);
EXPECT_EQ(elem.vnode, 12);
printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
}
tsBufDestory(pTSBuf1);
tsBufDestory(pTSBuf2);
}
} // namespace
TEST(testCase, tsBufTest) {
simpleTest();
largeTSTest();
multiTagsTest();
multiVnodeTagsTest();
loadDataTest();
invalidFileTest();
// randomIncTsTest();
TSTraverse();
mergeDiffVnodeBufferTest();
mergeIdenticalVnodeBufferTest();
}

769
src/query/tests/unitTest.cpp Normal file
View File

@ -0,0 +1,769 @@
#include <gtest/gtest.h>
#include <cassert>
#include <iostream>
#include "taos.h"
#include "tsdb.h"
#include "../../client/inc/tscUtil.h"
#include "ttime.h"
#include "tutil.h"
#include "tvariant.h"
#include "ttokendef.h"
namespace {
int32_t testValidateName(char* name) {
SSQLToken token = {0};
token.z = name;
token.n = strlen(name);
token.type = 0;
tSQLGetToken(name, &token.type);
return tscValidateName(&token);
}
}
static void _init_tvariant_bool(tVariant* t) {
t->i64Key = TSDB_FALSE;
t->nType = TSDB_DATA_TYPE_BOOL;
}
static void _init_tvariant_tinyint(tVariant* t) {
t->i64Key = -27;
t->nType = TSDB_DATA_TYPE_TINYINT;
}
static void _init_tvariant_int(tVariant* t) {
t->i64Key = -23997659;
t->nType = TSDB_DATA_TYPE_INT;
}
static void _init_tvariant_bigint(tVariant* t) {
t->i64Key = -3333333333333;
t->nType = TSDB_DATA_TYPE_BIGINT;
}
static void _init_tvariant_float(tVariant* t) {
t->dKey = -8991212199.8987878776;
t->nType = TSDB_DATA_TYPE_FLOAT;
}
static void _init_tvariant_binary(tVariant* t) {
tVariantDestroy(t);
t->pz = (char*)calloc(1, 20); //"2e3");
t->nType = TSDB_DATA_TYPE_BINARY;
strcpy(t->pz, "2e5");
t->nLen = strlen(t->pz);
}
static void _init_tvariant_nchar(tVariant* t) {
tVariantDestroy(t);
t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE);
t->nType = TSDB_DATA_TYPE_NCHAR;
wcscpy(t->wpz, L"-2000000.8765");
t->nLen = wcslen(t->wpz);
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
/* test validate the names for table/database */
TEST(testCase, db_table_name) {
char t01[] = "abc";
EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS);
char t02[] = "'abc'";
EXPECT_EQ(testValidateName(t02), TSDB_CODE_SUCCESS);
char t1[] = "abc.def";
EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS);
printf("%s\n", t1);
char t2[] = "'abc.def'";
EXPECT_EQ(testValidateName(t2), TSDB_CODE_SUCCESS);
printf("%s\n", t2);
char t3[] = "'abc'.def";
EXPECT_EQ(testValidateName(t3), TSDB_CODE_SUCCESS);
printf("%s\n", t3);
char t4[] = "'abc'.'def'";
EXPECT_EQ(testValidateName(t4), TSDB_CODE_SUCCESS);
char t5[] = "table.'def'";
EXPECT_EQ(testValidateName(t5), TSDB_CODE_INVALID_SQL);
char t6[] = "'table'.'def'";
EXPECT_EQ(testValidateName(t6), TSDB_CODE_INVALID_SQL);
char t7[] = "'_ab1234'.'def'";
EXPECT_EQ(testValidateName(t7), TSDB_CODE_SUCCESS);
printf("%s\n", t7);
char t8[] = "'_ab&^%1234'.'def'";
EXPECT_EQ(testValidateName(t8), TSDB_CODE_INVALID_SQL);
char t9[] = "'_123'.'gtest中文'";
EXPECT_EQ(testValidateName(t9), TSDB_CODE_INVALID_SQL);
char t10[] = "abc.'gtest中文'";
EXPECT_EQ(testValidateName(t10), TSDB_CODE_INVALID_SQL);
char t10_1[] = "abc.'中文gtest'";
EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_INVALID_SQL);
char t11[] = "'192.168.0.1'.abc";
EXPECT_EQ(testValidateName(t11), TSDB_CODE_INVALID_SQL);
char t12[] = "192.168.0.1.abc";
EXPECT_EQ(testValidateName(t12), TSDB_CODE_INVALID_SQL);
char t13[] = "abc.";
EXPECT_EQ(testValidateName(t13), TSDB_CODE_INVALID_SQL);
char t14[] = ".abc";
EXPECT_EQ(testValidateName(t14), TSDB_CODE_INVALID_SQL);
char t15[] = ".'abc'";
EXPECT_EQ(testValidateName(t15), TSDB_CODE_INVALID_SQL);
char t16[] = ".abc'";
EXPECT_EQ(testValidateName(t16), TSDB_CODE_INVALID_SQL);
char t17[] = "123a.\"abc\"";
EXPECT_EQ(testValidateName(t17), TSDB_CODE_INVALID_SQL);
printf("%s\n", t17);
char t18[] = "a.\"abc\"";
EXPECT_EQ(testValidateName(t18), TSDB_CODE_SUCCESS);
printf("%s\n", t18);
char t19[] = "'_ab1234'.'def'.'ab123'";
EXPECT_EQ(testValidateName(t19), TSDB_CODE_INVALID_SQL);
char t20[] = "'_ab1234*&^'";
EXPECT_EQ(testValidateName(t20), TSDB_CODE_INVALID_SQL);
char t21[] = "'1234_abc'";
EXPECT_EQ(testValidateName(t21), TSDB_CODE_INVALID_SQL);
// =======Containing capital letters=================
char t30[] = "ABC";
EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS);
char t31[] = "'ABC'";
EXPECT_EQ(testValidateName(t31), TSDB_CODE_SUCCESS);
char t32[] = "ABC.def";
EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS);
char t33[] = "'ABC.def";
EXPECT_EQ(testValidateName(t33), TSDB_CODE_INVALID_SQL);
char t33_0[] = "abc.DEF'";
EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_INVALID_SQL);
char t34[] = "'ABC.def'";
//int32_t tmp0 = testValidateName(t34);
EXPECT_EQ(testValidateName(t34), TSDB_CODE_SUCCESS);
char t35[] = "'ABC'.def";
EXPECT_EQ(testValidateName(t35), TSDB_CODE_SUCCESS);
char t36[] = "'ABC'.'DEF'";
EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS);
char t37[] = "abc.'DEF'";
EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS);
char t37_1[] = "abc.'_123DEF'";
EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS);
char t38[] = "'abc'.'DEF'";
EXPECT_EQ(testValidateName(t38), TSDB_CODE_SUCCESS);
// do not use key words
char t39[] = "table.'DEF'";
EXPECT_EQ(testValidateName(t39), TSDB_CODE_INVALID_SQL);
char t40[] = "'table'.'DEF'";
EXPECT_EQ(testValidateName(t40), TSDB_CODE_INVALID_SQL);
char t41[] = "'_abXYZ1234'.'deFF'";
EXPECT_EQ(testValidateName(t41), TSDB_CODE_SUCCESS);
char t42[] = "'_abDEF&^%1234'.'DIef'";
EXPECT_EQ(testValidateName(t42), TSDB_CODE_INVALID_SQL);
char t43[] = "'_123'.'Gtest中文'";
EXPECT_EQ(testValidateName(t43), TSDB_CODE_INVALID_SQL);
char t44[] = "'aABC'.'Gtest中文'";
EXPECT_EQ(testValidateName(t44), TSDB_CODE_INVALID_SQL);
char t45[] = "'ABC'.";
EXPECT_EQ(testValidateName(t45), TSDB_CODE_INVALID_SQL);
char t46[] = ".'ABC'";
EXPECT_EQ(testValidateName(t46), TSDB_CODE_INVALID_SQL);
char t47[] = "a.\"aTWc\"";
EXPECT_EQ(testValidateName(t47), TSDB_CODE_SUCCESS);
// ================has space =================
char t60[] = " ABC ";
EXPECT_EQ(testValidateName(t60), TSDB_CODE_INVALID_SQL);
char t60_1[] = " ABC ";
EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_INVALID_SQL);
char t61[] = "' ABC '";
EXPECT_EQ(testValidateName(t61), TSDB_CODE_SUCCESS);
char t61_1[] = "' ABC '";
EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_SUCCESS);
char t62[] = " ABC . def ";
EXPECT_EQ(testValidateName(t62), TSDB_CODE_INVALID_SQL);
char t63[] = "' ABC . def ";
EXPECT_EQ(testValidateName(t63), TSDB_CODE_INVALID_SQL);
char t63_0[] = " abc . DEF ' ";
EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_INVALID_SQL);
char t64[] = " ' ABC . def ' ";
//int32_t tmp1 = testValidateName(t64);
EXPECT_EQ(testValidateName(t64), TSDB_CODE_INVALID_SQL);
char t65[] = " ' ABC '. def ";
EXPECT_EQ(testValidateName(t65), TSDB_CODE_INVALID_SQL);
char t66[] = "' ABC '.' DEF '";
EXPECT_EQ(testValidateName(t66), TSDB_CODE_SUCCESS);
char t67[] = "abc . ' DEF '";
EXPECT_EQ(testValidateName(t67), TSDB_CODE_INVALID_SQL);
char t68[] = "' abc '.' DEF '";
EXPECT_EQ(testValidateName(t68), TSDB_CODE_SUCCESS);
// do not use key words
char t69[] = "table.'DEF'";
EXPECT_EQ(testValidateName(t69), TSDB_CODE_INVALID_SQL);
char t70[] = "'table'.'DEF'";
EXPECT_EQ(testValidateName(t70), TSDB_CODE_INVALID_SQL);
char t71[] = "'_abXYZ1234 '.' deFF '";
EXPECT_EQ(testValidateName(t71), TSDB_CODE_SUCCESS);
char t72[] = "'_abDEF&^%1234'.' DIef'";
EXPECT_EQ(testValidateName(t72), TSDB_CODE_INVALID_SQL);
char t73[] = "'_123'.' Gtest中文'";
EXPECT_EQ(testValidateName(t73), TSDB_CODE_INVALID_SQL);
char t74[] = "' aABC'.'Gtest中文'";
EXPECT_EQ(testValidateName(t74), TSDB_CODE_INVALID_SQL);
char t75[] = "' ABC '.";
EXPECT_EQ(testValidateName(t75), TSDB_CODE_INVALID_SQL);
char t76[] = ".' ABC'";
EXPECT_EQ(testValidateName(t76), TSDB_CODE_INVALID_SQL);
char t77[] = " a . \"aTWc\" ";
EXPECT_EQ(testValidateName(t77), TSDB_CODE_INVALID_SQL);
char t78[] = " a.\"aTWc \"";
EXPECT_EQ(testValidateName(t78), TSDB_CODE_INVALID_SQL);
// ===============muti string by space ===================
// There's no such case.
//char t160[] = "A BC";
//EXPECT_EQ(testValidateName(t160), TSDB_CODE_INVALID_SQL);
//printf("end:%s\n", t160);
// There's no such case.
//char t161[] = "' A BC '";
//EXPECT_EQ(testValidateName(t161), TSDB_CODE_INVALID_SQL);
char t162[] = " AB C . de f ";
EXPECT_EQ(testValidateName(t162), TSDB_CODE_INVALID_SQL);
char t163[] = "' AB C . de f ";
EXPECT_EQ(testValidateName(t163), TSDB_CODE_INVALID_SQL);
char t163_0[] = " ab c . DE F ' ";
EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_INVALID_SQL);
char t164[] = " ' AB C . de f ' ";
//int32_t tmp2 = testValidateName(t164);
EXPECT_EQ(testValidateName(t164), TSDB_CODE_INVALID_SQL);
char t165[] = " ' A BC '. de f ";
EXPECT_EQ(testValidateName(t165), TSDB_CODE_INVALID_SQL);
char t166[] = "' AB C '.' DE F '";
EXPECT_EQ(testValidateName(t166), TSDB_CODE_INVALID_SQL);
char t167[] = "ab c . ' D EF '";
EXPECT_EQ(testValidateName(t167), TSDB_CODE_INVALID_SQL);
char t168[] = "' a bc '.' DE F '";
EXPECT_EQ(testValidateName(t168), TSDB_CODE_INVALID_SQL);
}
/* test parse time function */
TEST(testCase, parse_time) {
taos_options(TSDB_OPTION_TIMEZONE, "GMT-8");
char t1[] = "2018-1-1 1:1:1.952798";
char t13[] = "1970-1-1 0:0:0";
int64_t time = 0, time1 = 0;
taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952);
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND);
char t2[] = "2018-1-1T1:1:1.952Z";
taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952 + 28800000);
char t3[] = "2018-1-1 1:01:01.952";
taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952);
char t4[] = "2018-1-1 1:01:01.9";
char t5[] = "2018-1-1 1:01:1.900";
char t6[] = "2018-01-01 1:1:1.90";
char t7[] = "2018-01-01 01:01:01.9";
char t8[] = "2018-01-01 01:01:01.9007865";
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t9[] = "2017-4-3 1:1:2.980";
char t10[] = "2017-4-3T2:1:2.98+9:00";
taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t11[] = "2017-4-3T2:1:2.98+09:00";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t12[] = "2017-4-3T2:1:2.98+0900";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taos_options(TSDB_OPTION_TIMEZONE, "UTC");
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 0);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
char t14[] = "1970-1-1T0:0:0Z";
taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 0);
char t40[] = "1970-1-1 0:0:0.999999999";
taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND);
char t41[] = "1997-1-1 0:0:0.999999999";
taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 852048000999);
int64_t k = timezone;
char t42[] = "1997-1-1T0:0:0.999999999Z";
taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND);
////////////////////////////////////////////////////////////////////
// illegal timestamp format
char t15[] = "2017-12-33 0:0:0";
EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1);
char t16[] = "2017-12-31 99:0:0";
EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1);
char t17[] = "2017-12-31T9:0:0";
EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1);
char t18[] = "2017-12-31T9:0:0.Z";
EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1);
char t19[] = "2017-12-31 9:0:0.-1";
EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1);
char t20[] = "2017-12-31 9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0);
EXPECT_EQ(time, 1514682000100);
char t21[] = "2017-12-31T9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1);
char t22[] = "2017-12-31 9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0);
char t23[] = "2017-12-31T9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0);
}
TEST(testCase, tvariant_convert) {
// 1. bool data to all other data types
tVariant t = {0};
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
EXPECT_EQ(t.i64Key, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
EXPECT_EQ(t.i64Key, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "FALSE");
tVariantDestroy(&t);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"FALSE");
tVariantDestroy(&t);
// 2. tinyint to other data types
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 1);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
EXPECT_EQ(t.i64Key, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
EXPECT_EQ(t.i64Key, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
EXPECT_EQ(t.i64Key, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-27");
tVariantDestroy(&t);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-27");
tVariantDestroy(&t);
// 3. int to other data
// types//////////////////////////////////////////////////////////////////
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 1);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
EXPECT_EQ(t.i64Key, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-23997659");
tVariantDestroy(&t);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-23997659");
tVariantDestroy(&t);
// 4. bigint to other data
// type//////////////////////////////////////////////////////////////////////////////
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 1);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-3333333333333");
tVariantDestroy(&t);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-3333333333333");
tVariantDestroy(&t);
// 5. float to other data
// types////////////////////////////////////////////////////////////////////////
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 1);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, -8991212199);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-8991212199.898788");
tVariantDestroy(&t);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-8991212199.898788");
tVariantDestroy(&t);
// 6. binary to other data types
// //////////////////////////////////////////////////////////////////
t.pz = "true";
t.nLen = strlen(t.pz);
t.nType = TSDB_DATA_TYPE_BINARY;
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 1);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.dKey, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.dKey, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "2e5");
tVariantDestroy(&t);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"2e5");
tVariantDestroy(&t);
// 7. nchar to other data types
// //////////////////////////////////////////////////////////////////
t.wpz = L"FALSE";
t.nLen = wcslen(t.wpz);
t.nType = TSDB_DATA_TYPE_NCHAR;
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 0);
_init_tvariant_nchar(&t);
EXPECT_LE(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, -2000000);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-2000000.8765");
tVariantDestroy(&t);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-2000000.8765");
tVariantDestroy(&t);
}
TEST(testCase, tGetToken_Test) {
char* s = ".123 ";
uint32_t type = 0;
int32_t len = tSQLGetToken(s, &type);
EXPECT_EQ(type, TK_FLOAT);
EXPECT_EQ(len, strlen(s) - 1);
char s1[] = "1.123e10 ";
len = tSQLGetToken(s1, &type);
EXPECT_EQ(type, TK_FLOAT);
EXPECT_EQ(len, strlen(s1) - 1);
char s4[] = "0xff ";
len = tSQLGetToken(s4, &type);
EXPECT_EQ(type, TK_HEX);
EXPECT_EQ(len, strlen(s4) - 1);
// invalid data type
char s2[] = "e10 ";
len = tSQLGetToken(s2, &type);
EXPECT_FALSE(type == TK_FLOAT);
char s3[] = "1.1.1.1";
len = tSQLGetToken(s3, &type);
EXPECT_EQ(type, TK_IPTOKEN);
EXPECT_EQ(len, strlen(s3));
char s5[] = "0x ";
len = tSQLGetToken(s5, &type);
EXPECT_FALSE(type == TK_HEX);
}
static SSQLToken createStrToken(char* s) {
SSQLToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)};
t.type = TK_STRING;
t.z = s;
t.n = strlen(s);
return t;
}
TEST(testCase, isValidNumber_test) {
SSQLToken t1 = createStrToken("123abc");
EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL);
t1 = createStrToken("0xabc");
EXPECT_EQ(isValidNumber(&t1), TK_HEX);
t1 = createStrToken("0b11101");
EXPECT_EQ(isValidNumber(&t1), TK_BIN);
t1 = createStrToken(".134abc");
EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL);
t1 = createStrToken("1e1 ");
EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL);
t1 = createStrToken("1+2");
EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL);
t1 = createStrToken("-0x123");
EXPECT_EQ(isValidNumber(&t1), TK_HEX);
t1 = createStrToken("-1");
EXPECT_EQ(isValidNumber(&t1), TK_INTEGER);
t1 = createStrToken("-0b1110");
EXPECT_EQ(isValidNumber(&t1), TK_BIN);
t1 = createStrToken("-.234");
EXPECT_EQ(isValidNumber(&t1), TK_FLOAT);
}
TEST(testCase, getTempFilePath_test) {
char path[4096] = {0};
memset(path, 1, 4096);
getTmpfilePath("new_tmp", path);
printf("%s\n", path);
}

35
src/rpc/src/rpcMain.c
View File

@ -32,6 +32,7 @@
#include "rpcServer.h"
#include "rpcHead.h"
#include "trpc.h"
#include "hash.h"
#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest))
#define rpcHeadFromCont(cont) ((SRpcHead *) (cont - sizeof(SRpcHead)))
@ -262,7 +263,8 @@ void *rpcOpen(SRpcInit *pInit) {
}
if (pRpc->connType == TAOS_CONN_SERVER) {
pRpc->hash = taosInitStrHash(pRpc->sessions, sizeof(pRpc), taosHashString);
// pRpc->hash = taosInitStrHash(pRpc->sessions, sizeof(pRpc), taosHashString);
pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true);
if (pRpc->hash == NULL) {
tError("%s failed to init string hash", pRpc->label);
rpcClose(pRpc);
@ -296,7 +298,8 @@ void rpcClose(void *param) {
}
}
taosCleanUpStrHash(pRpc->hash);
// taosCleanUpStrHash(pRpc->hash);
taosHashCleanup(pRpc->hash);
taosTmrCleanUp(pRpc->tmrCtrl);
taosIdPoolCleanUp(pRpc->idPool);
rpcCloseConnCache(pRpc->pCache);
@ -364,7 +367,7 @@ void rpcSendRequest(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg) {
// for TDengine, all the query, show commands shall have TCP connection
char type = pMsg->msgType;
if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_RETRIEVE ||
type == TSDB_MSG_TYPE_CM_STABLE_META || type == TSDB_MSG_TYPE_CM_TABLES_META ||
type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META ||
type == TSDB_MSG_TYPE_CM_SHOW )
pContext->connType = RPC_CONN_TCPC;
@ -531,8 +534,10 @@ static void rpcCloseConn(void *thandle) {
if ( pRpc->connType == TAOS_CONN_SERVER) {
char hashstr[40] = {0};
sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType);
taosDeleteStrHash(pRpc->hash, hashstr);
size_t size = sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType);
// taosDeleteStrHash(pRpc->hash, hashstr);
// taosHashRemove(pRpc->hash, hashstr, size);
rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg
pConn->pRspMsg = NULL;
pConn->inType = 0;
@ -580,10 +585,11 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
char hashstr[40] = {0};
SRpcHead *pHead = (SRpcHead *)pRecv->msg;
sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType);
size_t size = sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType);
// check if it is already allocated
SRpcConn **ppConn = (SRpcConn **)(taosGetStrHashData(pRpc->hash, hashstr));
// SRpcConn **ppConn = (SRpcConn **)(taosGetStrHashData(pRpc->hash, hashstr));
SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size));
if (ppConn) pConn = *ppConn;
if (pConn) return pConn;
@ -615,7 +621,9 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
pConn->localPort = (pRpc->localPort + pRpc->index);
}
taosAddStrHash(pRpc->hash, hashstr, (char *)&pConn);
// taosAddStrHash(pRpc->hash, hashstr, (char *)&pConn);
taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES);
tTrace("%s %p, rpc connection is allocated, sid:%d id:%s port:%u",
pRpc->label, pConn, sid, pConn->user, pConn->localPort);
}
@ -863,18 +871,19 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->port);
}
if (terrno != TSDB_CODE_ALREADY_PROCESSED) {
if (terrno != 0) { // parsing error
int32_t code = terrno;
if (code != TSDB_CODE_ALREADY_PROCESSED) {
if (code != 0) { // parsing error
if ( rpcIsReq(pHead->msgType) ) {
rpcSendErrorMsgToPeer(pRecv, terrno);
tTrace("%s %p, %s is sent with error code:0x%x", pRpc->label, pConn, taosMsg[pHead->msgType+1], terrno);
rpcSendErrorMsgToPeer(pRecv, code);
tTrace("%s %p, %s is sent with error code:%x", pRpc->label, pConn, taosMsg[pHead->msgType+1], code);
}
} else { // parsing OK
rpcProcessIncomingMsg(pConn, pHead);
}
}
if (terrno) rpcFreeMsg(pRecv->msg);
if (code) rpcFreeMsg(pRecv->msg);
return pConn;
}

16
src/sdb/CMakeLists.txt
View File

@ -1,16 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(inc)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(sdb ${SRC})
TARGET_LINK_LIBRARIES(sdb trpc)
IF (TD_CLUSTER)
TARGET_LINK_LIBRARIES(sdb)
ENDIF()
ENDIF ()

141
src/sdb/inc/sdbint.h
View File

@ -1,141 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _sdbint_header_
#define _sdbint_header_
#include <errno.h>
#include <pthread.h>
#include <semaphore.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "hashint.h"
#include "hashstr.h"
#include "sdb.h"
#include "tchecksum.h"
#include "tlog.h"
#include "trpc.h"
#include "tutil.h"
#define sdbError(...) \
if (sdbDebugFlag & DEBUG_ERROR) { \
tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \
}
#define sdbWarn(...) \
if (sdbDebugFlag & DEBUG_WARN) { \
tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \
}
#define sdbTrace(...) \
if (sdbDebugFlag & DEBUG_TRACE) { \
tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \
}
#define sdbPrint(...) \
{ tprintf("MND-SDB ", 255, __VA_ARGS__); }
#define mpeerError(...) \
if (sdbDebugFlag & DEBUG_ERROR) { \
tprintf("ERROR MND-MPEER ", 255, __VA_ARGS__); \
}
#define mpeerWarn(...) \
if (sdbDebugFlag & DEBUG_WARN) { \
tprintf("WARN MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \
}
#define mpeerTrace(...) \
if (sdbDebugFlag & DEBUG_TRACE) { \
tprintf("MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \
}
#define mpeerPrint(...) \
{ tprintf("MND-MPEER ", 255, __VA_ARGS__); }
#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__)
#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__)
#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__)
#define SDB_MAX_PEERS 4
#define SDB_DELIMITER 0xFFF00F00
#define SDB_ENDCOMMIT 0xAFFFAAAF
typedef struct {
uint64_t swVersion;
int16_t sdbFileVersion;
char reserved[6];
TSCKSUM checkSum;
} SSdbHeader;
typedef struct {
char type;
// short rowSize;
char *row;
} SSdbUpdate;
typedef struct _SSdbTable {
SSdbHeader header;
int maxRows;
int dbId;
int32_t maxRowSize;
char name[TSDB_DB_NAME_LEN];
char fn[128];
int keyType;
uint32_t autoIndex;
int64_t numOfRows;
int64_t id;
int64_t size;
void * iHandle;
int fd;
void *(*appTool)(char, void *, char *, int, int *);
pthread_mutex_t mutex;
SSdbUpdate * update;
int numOfUpdates;
int updatePos;
} SSdbTable;
typedef struct {
int64_t id;
int64_t offset;
int rowSize;
void * row;
} SRowMeta;
typedef struct {
int32_t delimiter;
int32_t rowSize;
int64_t id;
char data[];
} SRowHead;
typedef struct {
uint8_t dbId;
char type;
uint64_t version;
short dataLen;
char data[];
} SForwardMsg;
extern SSdbTable *tableList[];
extern int sdbMaxPeers;
extern int sdbNumOfTables;
extern int64_t sdbVersion;
int sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int dataLen);
int mpeerRetrieveRows(int fd, SSdbTable *pTable, uint64_t version);
void sdbResetTable(SSdbTable *pTable);
extern const int16_t sdbFileVersion;
#endif

1017
src/sdb/src/sdbEngine.c
View File

File diff suppressed because it is too large Load Diff

58
src/sdb/src/sdbstr.c
View File

@ -1,58 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "sdbint.h"
int32_t (*mpeerInitMnodesFp)(char *directory) = NULL;
void (*mpeerCleanUpMnodesFp)() = NULL;
int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL;
char *sdbStatusStr[] = {
"offline",
"unsynced",
"syncing",
"serving",
"null"
};
char *sdbRoleStr[] = {
"unauthed",
"undecided",
"master",
"slave",
"null"
};
int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) {
if (mpeerForwardRequestFp) {
return mpeerForwardRequestFp(pTable, type, data, dataLen);
} else {
return 0;
}
}
int32_t sdbInitPeers(char *directory) {
if (mpeerInitMnodesFp) {
return (*mpeerInitMnodesFp)(directory);
} else {
return 0;
}
}
void sdbCleanUpPeers() {
if (mpeerCleanUpMnodesFp) {
(*mpeerCleanUpMnodesFp)();
}
}

3
src/util/CMakeLists.txt
View File

@ -29,6 +29,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
MESSAGE(STATUS "Failed to find iconv, use default encoding method")
ENDIF ()
ENDIF ()
ADD_SUBDIRECTORY(tests)
ELSEIF (TD_WINDOWS_64)
ADD_DEFINITIONS(-DUSE_LIBICONV)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
@ -113,4 +115,3 @@ ENDIF()
#ENDIF ()

0
src/sdb/inc/hashint.h → src/util/inc/hashint.h
View File

0
src/sdb/inc/hashstr.h → src/util/inc/hashstr.h
View File

16
src/util/inc/tarray.h
View File

@ -87,7 +87,21 @@ size_t taosArrayGetSize(const SArray* pArray);
void* taosArrayInsert(SArray* pArray, size_t index, void* pData);
/**
*
* remove data entry of the given index
* @param pArray
* @param index
*/
void taosArrayRemove(SArray* pArray, size_t index);
/**
* copy the whole array from source to destination
* @param pDst
* @param pSrc
*/
void taosArrayCopy(SArray* pDst, SArray* pSrc);
/**
* destroy array list
* @param pArray
*/
void taosArrayDestroy(SArray* pArray);

6
src/util/inc/tglobalcfg.h
View File

@ -103,13 +103,7 @@ extern int tsReplications;
extern int tsNumOfMPeers;
extern int tsMaxShellConns;
extern int tsMaxAccounts;
extern int tsMaxUsers;
extern int tsMaxDbs;
extern int tsMaxTables;
extern int tsMaxDnodes;
extern int tsMaxVGroups;
extern char tsMgmtZone[];
extern char tsLocalIp[];
extern char tsDefaultDB[];

6
src/util/inc/tidpool.h
View File

@ -34,11 +34,7 @@ void taosIdPoolCleanUp(void *handle);
int taosIdPoolNumOfUsed(void *handle);
void taosIdPoolReinit(void *handle);
void taosIdPoolMarkStatus(void *handle, int id, int status);
void taosIdPoolSetFreeList(void *handle);
void taosIdPoolMarkStatus(void *handle, int id);
#ifdef __cplusplus
}

19
src/util/inc/tlog.h
View File

@ -239,6 +239,25 @@ extern uint32_t cdebugFlag;
#define monitorLWarn(...) taosLogWarn(__VA_ARGS__) monitorWarn(__VA_ARGS__)
#define monitorLPrint(...) taosLogPrint(__VA_ARGS__) monitorPrint(__VA_ARGS__)
#define sdbError(...) \
if (sdbDebugFlag & DEBUG_ERROR) { \
tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \
}
#define sdbWarn(...) \
if (sdbDebugFlag & DEBUG_WARN) { \
tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \
}
#define sdbTrace(...) \
if (sdbDebugFlag & DEBUG_TRACE) { \
tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \
}
#define sdbPrint(...) \
{ tprintf("MND-SDB ", 255, __VA_ARGS__); }
#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__)
#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__)
#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__)
#ifdef __cplusplus
}
#endif

42
src/util/inc/tskiplist.h
View File

@ -51,7 +51,7 @@ typedef struct SSkipListNode {
#define SL_GET_BACKWARD_POINTER(n, _l) \
((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)]
#define SL_GET_NODE_DATA(n) ((char*)(n) + SL_NODE_HEADER_SIZE((n)->level))
#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level))
#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n)))
#define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n))
@ -106,25 +106,25 @@ typedef struct tSkipListState {
typedef struct SSkipListKeyInfo {
uint8_t dupKey : 2; // if allow duplicated key in the skip list
uint8_t type : 6; // key type
uint8_t type : 4; // key type
uint8_t freeNode:2; // free node when destroy the skiplist
uint8_t len; // maximum key length, used in case of string key
} SSkipListKeyInfo;
typedef struct SSkipList {
__compar_fn_t comparFn;
__sl_key_fn_t keyFn;
uint32_t size;
uint8_t maxLevel;
uint8_t level;
SSkipListKeyInfo keyInfo;
__compar_fn_t comparFn;
__sl_key_fn_t keyFn;
uint32_t size;
uint8_t maxLevel;
uint8_t level;
SSkipListKeyInfo keyInfo;
pthread_rwlock_t *lock;
SSkipListNode * pHead;
SSkipListNode * pHead; // point to the first element
SSkipListNode * pTail; // point to the last element
void * lastKey; // last key in the skiplist
#if SKIP_LIST_RECORD_PERFORMANCE
tSkipListState state; // skiplist state
#endif
} SSkipList;
/*
@ -147,7 +147,7 @@ typedef struct SSkipListIterator {
* @return
*/
SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe,
__sl_key_fn_t fn);
uint8_t freeNode, __sl_key_fn_t fn);
/**
*
@ -182,21 +182,28 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode);
* @param keyType
* @return
*/
SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType);
SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType);
/**
* get the size of skip list
* @param pSkipList
* @return
*/
size_t tSkipListGetSize(const SSkipList* pSkipList);
size_t tSkipListGetSize(const SSkipList *pSkipList);
/**
* display skip list of the given level, for debug purpose only
* @param pSkipList
* @param nlevel
*/
void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
/**
* create skiplist iterator
* @param pSkipList
* @return
*/
SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList);
SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList);
/**
* forward the skip list iterator
@ -217,7 +224,7 @@ SSkipListNode *tSkipListIterGet(SSkipListIterator *iter);
* @param iter
* @return
*/
void* tSkipListDestroyIter(SSkipListIterator* iter);
void *tSkipListDestroyIter(SSkipListIterator *iter);
/*
* remove only one node of the pKey value.
@ -234,7 +241,6 @@ bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey);
*/
void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
#ifdef __cplusplus
}
#endif

8
src/util/inc/tutil.h
View File

@ -176,6 +176,14 @@ uint32_t ip2uint(const char *const ip_addr);
void taosSetAllocMode(int mode, const char* path, bool autoDump);
void taosDumpMemoryLeak();
#define TD_EQ 0x1
#define TD_GT 0x2
#define TD_LT 0x4
#define TD_GE (TD_EQ | TD_GT)
#define TD_LE (TD_EQ | TD_LT)
void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size,
int (*compar)(const void *, const void *), int flags);
#ifdef TAOS_MEM_CHECK
void * taos_malloc(size_t size, const char *file, uint32_t line);

0
src/sdb/src/hashint.c → src/util/src/hashint.c
View File

0
src/sdb/src/hashstr.c → src/util/src/hashstr.c
View File

32
src/util/src/tarray.c
View File

@ -128,6 +128,38 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) {
return dst;
}
void taosArrayRemove(SArray* pArray, size_t index) {
assert(index < pArray->size);
if (index == pArray->size - 1) {
taosArrayPop(pArray);
return;
}
size_t remain = pArray->size - index - 1;
memmove(pArray->pData + index * pArray->elemSize, pArray->pData + (index + 1) * pArray->elemSize, remain * pArray->elemSize);
pArray->size -= 1;
}
void taosArrayCopy(SArray* pDst, SArray* pSrc) {
assert(pSrc != NULL && pDst != NULL);
if (pDst->capacity < pSrc->size) {
void* pData = realloc(pDst->pData, pSrc->size * pSrc->elemSize);
if (pData == NULL) { // todo handle oom
} else {
pDst->pData = pData;
pDst->capacity = pSrc->size;
}
}
memcpy(pDst->pData, pSrc->pData, pSrc->elemSize * pSrc->size);
pDst->elemSize = pSrc->elemSize;
pDst->capacity = pSrc->size;
pDst->size = pSrc->size;
}
void taosArrayDestroy(SArray* pArray) {
if (pArray == NULL) {
return;

28
src/util/src/tglobalcfg.c
View File

@ -112,13 +112,7 @@ int tsReplications = TSDB_REPLICA_MIN_NUM;
int tsNumOfMPeers = 3;
int tsMaxShellConns = 2000;
int tsMaxAccounts = 100;
int tsMaxUsers = 1000;
int tsMaxDbs = 1000;
int tsMaxTables = 650000;
int tsMaxDnodes = 1000;
int tsMaxVGroups = 1000;
char tsMgmtZone[16] = "rzone";
int tsMaxTables = 100000;
char tsLocalIp[TSDB_IPv4ADDR_LEN] = {0};
char tsDefaultDB[TSDB_DB_NAME_LEN] = {0};
@ -612,28 +606,10 @@ static void doInitGlobalConfig() {
1, 8640000, 0, TSDB_CFG_UTYPE_SECOND);
// mgmt configs
tsInitConfigOption(cfg++, "mgmtZone", tsMgmtZone, TSDB_CFG_VTYPE_STRING,
TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER,
0, 0, 16, TSDB_CFG_UTYPE_NONE);
tsInitConfigOption(cfg++, "maxAccounts", &tsMaxAccounts, TSDB_CFG_VTYPE_INT,
TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER,
1, 1000, 0, TSDB_CFG_UTYPE_NONE);
tsInitConfigOption(cfg++, "maxUsers", &tsMaxUsers, TSDB_CFG_VTYPE_INT,
TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW,
1, 1000, 0, TSDB_CFG_UTYPE_NONE);
tsInitConfigOption(cfg++, "maxDbs", &tsMaxDbs, TSDB_CFG_VTYPE_INT,
TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW,
1, 10000, 0, TSDB_CFG_UTYPE_NONE);
tsInitConfigOption(cfg++, "maxTables", &tsMaxTables, TSDB_CFG_VTYPE_INT,
TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW,
1, 100000000, 0, TSDB_CFG_UTYPE_NONE);
tsInitConfigOption(cfg++, "maxDnodes", &tsMaxDnodes, TSDB_CFG_VTYPE_INT,
TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER,
1, 1000, 0, TSDB_CFG_UTYPE_NONE);
tsInitConfigOption(cfg++, "maxVGroups", &tsMaxVGroups, TSDB_CFG_VTYPE_INT,
TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW,
1, 1000000, 0, TSDB_CFG_UTYPE_NONE);
tsInitConfigOption(cfg++, "minSlidingTime", &tsMinSlidingTime, TSDB_CFG_VTYPE_INT,
TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW,
10, 1000000, 0, TSDB_CFG_UTYPE_MS);

136
src/util/src/tidpool.c
View File

@ -15,88 +15,81 @@
#include "os.h"
#include "tlog.h"
#include <stdbool.h>
typedef struct {
int maxId;
int numOfFree;
int freeSlot;
int * freeList;
bool * freeList;
pthread_mutex_t mutex;
} id_pool_t;
void *taosInitIdPool(int maxId) {
id_pool_t *pIdPool;
int * idList, i;
if (maxId < 3) maxId = 3;
pIdPool = (id_pool_t *)malloc(sizeof(id_pool_t));
id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t));
if (pIdPool == NULL) return NULL;
idList = (int *)malloc(sizeof(int) * (size_t)maxId);
if (idList == NULL) {
pIdPool->freeList = calloc(maxId, sizeof(bool));
if (pIdPool->freeList == NULL) {
free(pIdPool);
return NULL;
}
memset(pIdPool, 0, sizeof(id_pool_t));
pIdPool->maxId = maxId;
pIdPool->numOfFree = maxId - 1;
pIdPool->numOfFree = maxId;
pIdPool->freeSlot = 0;
pIdPool->freeList = idList;
pthread_mutex_init(&pIdPool->mutex, NULL);
for (i = 1; i < maxId; ++i) idList[i - 1] = i;
pTrace("pool:%p is setup, maxId:%d", pIdPool, pIdPool->maxId);
return (void *)pIdPool;
return pIdPool;
}
int taosAllocateId(void *handle) {
id_pool_t *pIdPool;
int id = -1;
if (handle == NULL) return id;
pIdPool = (id_pool_t *)handle;
if (pIdPool->maxId < 3) pError("pool:%p is messed up, maxId:%d", pIdPool, pIdPool->maxId);
if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex");
if (pIdPool->numOfFree > 0) {
id = pIdPool->freeList[pIdPool->freeSlot];
pIdPool->freeSlot = (pIdPool->freeSlot + 1) % pIdPool->maxId;
pIdPool->numOfFree--;
id_pool_t *pIdPool = handle;
if (handle == NULL) {
return -1;
}
if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex");
int slot = -1;
pthread_mutex_lock(&pIdPool->mutex);
return id;
if (pIdPool->numOfFree > 0) {
for (int i = 0; i < pIdPool->maxId; ++i) {
slot = (i + pIdPool->freeSlot) % pIdPool->maxId;
if (!pIdPool->freeList[slot]) {
pIdPool->freeList[slot] = true;
pIdPool->freeSlot = slot + 1;
pIdPool->numOfFree--;
break;
}
}
}
pthread_mutex_unlock(&pIdPool->mutex);
return slot + 1;
}
void taosFreeId(void *handle, int id) {
id_pool_t *pIdPool;
int slot;
id_pool_t *pIdPool = handle;
if (handle == NULL) return;
pIdPool = (id_pool_t *)handle;
if (pIdPool->freeList == NULL || pIdPool->maxId == 0) return;
if (id <= 0 || id >= pIdPool->maxId) return;
if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex");
pthread_mutex_lock(&pIdPool->mutex);
slot = (pIdPool->freeSlot + pIdPool->numOfFree) % pIdPool->maxId;
pIdPool->freeList[slot] = id;
pIdPool->numOfFree++;
int slot = (id - 1) % pIdPool->maxId;
if (pIdPool->freeList[slot]) {
pIdPool->freeList[slot] = false;
pIdPool->numOfFree++;
}
if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex");
pthread_mutex_unlock(&pIdPool->mutex);
}
void taosIdPoolCleanUp(void *handle) {
id_pool_t *pIdPool;
id_pool_t *pIdPool = handle;
if (handle == NULL) return;
pIdPool = (id_pool_t *)handle;
if (pIdPool == NULL) return;
pTrace("pool:%p is cleaned", pIdPool);
@ -110,42 +103,21 @@ void taosIdPoolCleanUp(void *handle) {
}
int taosIdPoolNumOfUsed(void *handle) {
id_pool_t *pIdPool = (id_pool_t *)handle;
return pIdPool->maxId - pIdPool->numOfFree - 1;
id_pool_t *pIdPool = handle;
return pIdPool->maxId - pIdPool->numOfFree;
}
void taosIdPoolReinit(void *handle) {
id_pool_t *pIdPool;
void taosIdPoolMarkStatus(void *handle, int id) {
id_pool_t *pIdPool = handle;
pthread_mutex_lock(&pIdPool->mutex);
pIdPool = (id_pool_t *)handle;
pIdPool->numOfFree = 0;
pIdPool->freeSlot = 0;
for (int i = 0; i < pIdPool->maxId; ++i) pIdPool->freeList[i] = 0;
}
void taosIdPoolMarkStatus(void *handle, int id, int status) {
id_pool_t *pIdPool = (id_pool_t *)handle;
pIdPool->freeList[id] = status;
}
void taosIdPoolSetFreeList(void *handle) {
id_pool_t *pIdPool;
int pos = 0;
pIdPool = (id_pool_t *)handle;
pIdPool->numOfFree = 0;
pIdPool->freeSlot = 0;
for (int i = 1; i < pIdPool->maxId; ++i) {
if (pIdPool->freeList[i] == 0) {
pIdPool->freeList[pos] = i;
pIdPool->numOfFree++;
pos++;
}
int slot = (id - 1) % pIdPool->maxId;
if (!pIdPool->freeList[slot]) {
pIdPool->freeList[slot] = true;
pIdPool->numOfFree--;
}
pthread_mutex_unlock(&pIdPool->mutex);
}
int taosUpdateIdPool(id_pool_t *handle, int maxId) {
@ -154,18 +126,14 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) {
return -1;
}
int *idList, i;
idList = (int *)malloc(sizeof(int) * (size_t)maxId);
int *idList = calloc(maxId, sizeof(bool));
if (idList == NULL) {
return -1;
}
for (i = 1; i < maxId; ++i) {
idList[i - 1] = i;
}
if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex");
pthread_mutex_lock(&pIdPool->mutex);
memcpy(idList, pIdPool->freeList, sizeof(int) * (size_t)pIdPool->maxId);
memcpy(idList, pIdPool->freeList, sizeof(bool) * pIdPool->maxId);
pIdPool->numOfFree += (maxId - pIdPool->maxId);
pIdPool->maxId = maxId;
@ -173,7 +141,7 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) {
pIdPool->freeList = idList;
free(oldIdList);
if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex");
pthread_mutex_unlock(&pIdPool->mutex);
return 0;
}

13
src/util/src/tlist.c
View File

@ -100,6 +100,8 @@ SListNode *tdListPopHead(SList *list) {
list->head = node->next;
}
list->numOfEles--;
node->next = NULL;
node->prev = NULL;
return node;
}
@ -113,6 +115,7 @@ SListNode *tdListPopTail(SList *list) {
list->tail = node->prev;
}
list->numOfEles--;
node->next = node->prev = NULL;
return node;
}
@ -131,6 +134,7 @@ SListNode *tdListPopNode(SList *list, SListNode *node) {
node->next->prev = node->prev;
}
list->numOfEles--;
node->next = node->prev = NULL;
return node;
}
@ -138,11 +142,10 @@ SListNode *tdListPopNode(SList *list, SListNode *node) {
// Move all node elements from src to dst, the dst is assumed as an empty list
void tdListMove(SList *src, SList *dst) {
// assert(dst->eleSize == src->eleSize);
dst->numOfEles = src->numOfEles;
dst->head = src->head;
dst->tail = src->tail;
src->numOfEles = 0;
src->head = src->tail = NULL;
SListNode *node = NULL;
while ((node = tdListPopHead(src)) != NULL) {
tdListAppendNode(dst, node);
}
}
void tdListNodeGetData(SList *list, SListNode *node, void *target) { memcpy(target, node->data, list->eleSize); }

333
src/util/src/tskiplist.c
View File

@ -15,7 +15,6 @@
#include "os.h"
#include "tlog.h"
// #include "tsdb.h"
#include "tskiplist.h"
#include "tutil.h"
@ -52,28 +51,27 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) {
level = 1;
pSkipList->level = 1;
} else {
if (level > pSkipList->level && pSkipList->level < pSkipList->maxLevel) {
level = (++pSkipList->level);
if (level > pSkipList->level) {
if (pSkipList->level < pSkipList->maxLevel) {
level = (++pSkipList->level);
} else {
level = pSkipList->level;
}
}
}
assert(level <= pSkipList->maxLevel);
return level;
}
static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, int32_t level, SSkipListNode *pNode);
#define DO_MEMSET_PTR_AREA(n) do {\
int32_t _l = (n)->level;\
memset(pNode, 0, SL_NODE_HEADER_SIZE(_l));\
(n)->level = _l;\
} while(0)
void tSkipListDoRecordPut(SSkipList *pSkipList) {
#if SKIP_LIST_RECORD_PERFORMANCE
const int32_t MAX_RECORD_NUM = 1000;
if (pSkipList->state.nInsertObjs == MAX_RECORD_NUM) {
pSkipList->state.nInsertObjs = 1;
pSkipList->state.nTotalStepsForInsert = 0;
pSkipList->state.nTotalElapsedTimeForInsert = 0;
} else {
pSkipList->state.nInsertObjs++;
}
#endif
}
static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode);
static SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode);
int32_t compareInt32Val(const void *pLeft, const void *pRight) {
int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight);
@ -143,28 +141,12 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
}
int32_t compareStrVal(const void *pLeft, const void *pRight) {
// SSkipListKey *pL = (SSkipListKey *)pLeft;
// SSkipListKey *pR = (SSkipListKey *)pRight;
//
// if (pL->nLen == 0 && pR->nLen == 0) {
// return 0;
// }
//
// // handle only one-side bound compare situation, there is only lower bound or only upper bound
// if (pL->nLen == -1) {
// return 1; // no lower bound, lower bound is minimum, always return -1;
// } else if (pR->nLen == -1) {
// return -1; // no upper bound, upper bound is maximum situation, always return 1;
// }
//
// int32_t ret = strcmp(((SSkipListKey *)pLeft)->pz, ((SSkipListKey *)pRight)->pz);
//
// if (ret == 0) {
// return 0;
// } else {
// return ret > 0 ? 1 : -1;
// }
return 0;
int32_t ret = strcmp(pLeft, pRight);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
int32_t compareWStrVal(const void *pLeft, const void *pRight) {
@ -281,8 +263,30 @@ static __compar_fn_t getKeyComparator(int32_t keyType) {
return comparFn;
}
static bool initForwardBackwardPtr(SSkipList* pSkipList) {
uint32_t maxLevel = pSkipList->maxLevel;
// head info
pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel) * 2);
if (pSkipList->pHead == NULL) {
return false;
}
pSkipList->pHead->level = pSkipList->maxLevel;
// tail info
pSkipList->pTail = (SSkipListNode*) ((char*) pSkipList->pHead + SL_NODE_HEADER_SIZE(maxLevel));
pSkipList->pTail->level = pSkipList->maxLevel;
for(int32_t i = 0; i < maxLevel; ++i) {
SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail;
SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead;
}
return true;
}
SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t lock,
__sl_key_fn_t fn) {
uint8_t freeNode, __sl_key_fn_t fn) {
SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList));
if (pSkipList == NULL) {
return NULL;
@ -292,22 +296,24 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, ui
maxLevel = MAX_SKIP_LIST_LEVEL;
}
pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey};
pSkipList->keyFn = fn;
pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey, .freeNode = freeNode};
pSkipList->keyFn = fn;
pSkipList->comparFn = getKeyComparator(keyType);
pSkipList->maxLevel = maxLevel;
pSkipList->level = 1;
pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel));
pSkipList->pHead->level = pSkipList->maxLevel;
pSkipList->level = 1;
if (!initForwardBackwardPtr(pSkipList)) {
tfree(pSkipList);
return NULL;
}
if (lock) {
pSkipList->lock = calloc(1, sizeof(pthread_rwlock_t));
if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) {
tfree(pSkipList->pHead);
tfree(pSkipList);
return NULL;
}
}
@ -349,16 +355,17 @@ void *tSkipListDestroy(SSkipList *pSkipList) {
pthread_rwlock_wrlock(pSkipList->lock);
}
SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); // pSkipList->pHead.pForward[0];
SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0);
while (pNode) {
while (pNode != pSkipList->pTail) {
SSkipListNode *pTemp = pNode;
pNode = SL_GET_FORWARD_POINTER(pNode, 0);
tfree(pTemp);
if (pSkipList->keyInfo.freeNode) {
tfree(pTemp);
}
}
tfree(pSkipList->pHead);
if (pSkipList->lock) {
pthread_rwlock_unlock(pSkipList->lock);
pthread_rwlock_destroy(pSkipList->lock);
@ -381,94 +388,108 @@ void tSkipListRandNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSi
}
SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) {
if (pSkipList == NULL) {
if (pSkipList == NULL || pNode == NULL) {
return NULL;
}
if (pSkipList->lock) {
pthread_rwlock_wrlock(pSkipList->lock);
}
// record one node is put into skiplist
tSkipListDoRecordPut(pSkipList);
// the new key is greater than the last key of skiplist append it at last position
char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode);
if (pSkipList->size == 0 || pSkipList->comparFn(pSkipList->lastKey, newDatakey) < 0) {
return tSkipListDoAppend(pSkipList, pNode);
}
// find the appropriated position to insert data
SSkipListNode *px = pSkipList->pHead;
SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0};
bool identical = false;
for (int32_t i = pSkipList->level - 1; i >= 0; --i) {
SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i);
while (p != NULL) {
while (p != pSkipList->pTail) {
char *key = SL_GET_NODE_KEY(pSkipList, p);
char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode);
// if the forward element is less than the specified key, forward one step
if (pSkipList->comparFn(key, newDatakey) < 0) {
int32_t ret = pSkipList->comparFn(key, newDatakey);
if (ret < 0) {
px = p;
p = SL_GET_FORWARD_POINTER(px, i);
} else {
if (identical == false) {
identical = (ret == 0);
}
break;
}
}
#if SKIP_LIST_RECORD_PERFORMANCE
pSkipList->state.nTotalStepsForInsert++;
#endif
forward[i] = px;
}
// if the skip list does not allowed identical key inserted, the new data will be discarded.
if (pSkipList->keyInfo.dupKey == 0 && forward[0] != pSkipList->pHead) {
char *key = SL_GET_NODE_KEY(pSkipList, forward[0]);
char *pNewDataKey = SL_GET_NODE_KEY(pSkipList, pNode);
if (pSkipList->comparFn(key, pNewDataKey) == 0) {
if (pSkipList->lock) {
pthread_rwlock_unlock(pSkipList->lock);
}
return forward[0];
if (pSkipList->keyInfo.dupKey == 0 && identical) {
if (pSkipList->lock) {
pthread_rwlock_unlock(pSkipList->lock);
}
return forward[0];
}
#if SKIP_LIST_RECORD_PERFORMANCE
recordNodeEachLevel(pSkipList, level);
#endif
int32_t level = SL_GET_NODE_LEVEL(pNode);
tSkipListDoInsert(pSkipList, forward, level, pNode);
atomic_add_fetch_32(&pSkipList->size, 1);
#if SKIP_LIST_RECORD_PERFORMANCE
pSkipList->state.nTotalMemSize += getOneNodeSize(pKey, level);
#endif
if (pSkipList->lock) {
pthread_rwlock_unlock(pSkipList->lock);
}
tSkipListDoInsert(pSkipList, forward, pNode);
return pNode;
}
void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, int32_t level, SSkipListNode *pNode) {
for (int32_t i = 0; i < level; ++i) {
void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) {
DO_MEMSET_PTR_AREA(pNode);
for (int32_t i = 0; i < pNode->level; ++i) {
SSkipListNode *x = forward[i];
if (x != NULL) {
// if (x != pSkipList->pTail) {
SL_GET_BACKWARD_POINTER(pNode, i) = x;
SSkipListNode *pForward = SL_GET_FORWARD_POINTER(x, i);
if (pForward) {
SL_GET_BACKWARD_POINTER(pForward, i) = pNode;
}
SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i);
// if (next) {
SL_GET_BACKWARD_POINTER(next, i) = pNode;
// }
SL_GET_FORWARD_POINTER(pNode, i) = SL_GET_FORWARD_POINTER(x, i);
SL_GET_FORWARD_POINTER(pNode, i) = next;
SL_GET_FORWARD_POINTER(x, i) = pNode;
} else {
SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode;
SL_GET_BACKWARD_POINTER(pSkipList->pHead, i) = (pSkipList->pHead);
}
// } else {
// SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode;
// }
}
atomic_add_fetch_32(&pSkipList->size, 1);
if (pSkipList->lock) {
pthread_rwlock_unlock(pSkipList->lock);
}
}
SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode) {
// do clear pointer area
DO_MEMSET_PTR_AREA(pNode);
for(int32_t i = 0; i < pNode->level; ++i) {
SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i);
SL_GET_FORWARD_POINTER(prev, i) = pNode;
SL_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail;
SL_GET_BACKWARD_POINTER(pNode, i) = prev;
SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode;
}
pSkipList->lastKey = SL_GET_NODE_KEY(pSkipList, pNode);
atomic_add_fetch_32(&pSkipList->size, 1);
if (pSkipList->lock) {
pthread_rwlock_unlock(pSkipList->lock);
}
return pNode;
}
SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType) {
@ -691,89 +712,6 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) {
// return NULL;
//}
//
// int32_t tSkipListIterateList(SSkipList *pSkipList, SSkipListNode ***pRes, bool (*fp)(SSkipListNode *, void *),
// void *param) {
// (*pRes) = (SSkipListNode **)calloc(1, POINTER_BYTES * pSkipList->nSize);
// if (NULL == *pRes) {
// pError("error skiplist %p, malloc failed", pSkipList);
// return -1;
// }
//
// pthread_rwlock_rdlock(&pSkipList->lock);
// SSkipListNode *pStartNode = pSkipList->pHead.pForward[0];
// int32_t num = 0;
//
// for (int32_t i = 0; i < pSkipList->nSize; ++i) {
// if (pStartNode == NULL) {
// pError("error skiplist %p, required length:%d, actual length:%d", pSkipList, pSkipList->nSize, i - 1);
//#ifdef _DEBUG_VIEW
// tSkipListPrint(pSkipList, 1);
//#endif
// break;
// }
//
// if (fp == NULL || (fp != NULL && fp(pStartNode, param) == true)) {
// (*pRes)[num++] = pStartNode;
// }
//
// pStartNode = pStartNode->pForward[0];
// }
//
// pthread_rwlock_unlock(&pSkipList->lock);
//
// if (num == 0) {
// free(*pRes);
// *pRes = NULL;
// } else if (num < pSkipList->nSize) { // free unused memory
// char *tmp = realloc((*pRes), num * POINTER_BYTES);
// assert(tmp != NULL);
//
// *pRes = (SSkipListNode **)tmp;
// }
//
// return num;
//}
//
// int32_t tSkipListIteratorReset(SSkipList *pSkipList, SSkipListIterator *iter) {
// if (pSkipList == NULL) {
// return -1;
// }
//
// iter->pSkipList = pSkipList;
// if (pSkipList->lock) {
// pthread_rwlock_rdlock(&pSkipList->lock);
// }
// iter->cur = NULL; // pSkipList->pHead.pForward[0];
// iter->num = pSkipList->size;
//
// if (pSkipList->lock) {
// pthread_rwlock_unlock(&pSkipList->lock);
// }
//
// return 0;
//}
//
// bool tSkipListIteratorNext(SSkipListIterator *iter) {
// if (iter->num == 0 || iter->pSkipList == NULL) {
// return false;
// }
//
// SSkipList *pSkipList = iter->pSkipList;
//
// pthread_rwlock_rdlock(&pSkipList->lock);
// if (iter->cur == NULL) {
// iter->cur = pSkipList->pHead.pForward[0];
// } else {
// iter->cur = iter->cur->pForward[0];
// }
//
// pthread_rwlock_unlock(&pSkipList->lock);
//
// return iter->cur != NULL;
//}
//
// SSkipListNode *tSkipListIteratorGet(SSkipListIterator *iter) { return iter->cur; }
//
// int32_t tSkipListRangeQuery(SSkipList *pSkipList, tSKipListQueryCond *pCond, SSkipListNode ***pRes) {
// pSkipList->state.queryCount++;
// SSkipListNode *pStart = tSkipListParQuery(pSkipList, &pCond->lowerBnd, pCond->lowerBndRelOptr);
@ -841,12 +779,20 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
}
SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1);
int32_t id = 1;
while (p) {
int32_t id = 1;
char* prev = NULL;
while (p != pSkipList->pTail) {
char *key = SL_GET_NODE_KEY(pSkipList, p);
if (prev != NULL) {
assert(pSkipList->comparFn(prev, key) < 0);
}
switch (pSkipList->keyInfo.type) {
case TSDB_DATA_TYPE_INT:
fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key);
break;
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_BIGINT:
@ -862,7 +808,8 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
fprintf(stdout, "\n");
}
prev = SL_GET_NODE_KEY(pSkipList, p);
p = SL_GET_FORWARD_POINTER(p, nlevel - 1);
// p = p->pForward[nlevel - 1];
}
}

69
src/util/src/tutil.c
View File

@ -617,3 +617,72 @@ char *taosCharsetReplace(char *charsetstr) {
return strdup(charsetstr);
}
#define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx))
void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, int (*compar)(const void *, const void *), int flags) {
// TODO: need to check the correctness of this function
int l = 0;
int r = nmemb;
int idx = 0;
int comparison;
if (flags == TD_EQ) {
return bsearch(key, base, nmemb, size, compar);
} else if (flags == TD_GE) {
if ((*compar)(key, elePtrAt(base, size, 0)) <= 0) return elePtrAt(base, size, 0);
if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) > 0) return NULL;
while (l < r) {
idx = (l + r) / 2;
comparison = (*compar)(key, elePtrAt(base, size, idx));
if (comparison < 0) {
r = idx;
} else if (comparison > 0) {
l = idx + 1;
} else {
return elePtrAt(base, size, idx);
}
}
if ((*compar)(key, elePtrAt(base, size, idx) < 0)) {
return elePtrAt(base, size, idx);
} else {
if (idx + 1 > nmemb - 1) {
return NULL;
} else {
return elePtrAt(base, size, idx + 1);
}
}
} else if (flags == TD_LE) {
if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) >= 0) return elePtrAt(base, size, nmemb - 1);
if ((*compar)(key, elePtrAt(base, size, 0)) < 0) return NULL;
while (l < r) {
idx = (l + r) / 2;
comparison = (*compar)(key, elePtrAt(base, size, idx));
if (comparison < 0) {
r = idx;
} else if (comparison > 0) {
l = idx + 1;
} else {
return elePtrAt(base, size, idx);
}
}
if ((*compar)(key, elePtrAt(base, size, idx)) > 0) {
return elePtrAt(base, size, idx);
} else {
if (idx == 0) {
return NULL;
} else {
return elePtrAt(base, size, idx - 1);
}
}
} else {
assert(0);
return NULL;
}
return NULL;
}

15
src/util/tests/CMakeLists.txt Normal file
View File

@ -0,0 +1,15 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest)
FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib)
IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
MESSAGE(STATUS "gTest library found, build unit test")
INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(utilTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(utilTest tutil gtest pthread)
ENDIF()

142
src/util/tests/cacheTest.cpp Normal file
View File

@ -0,0 +1,142 @@
#include <iostream>
#include <gtest/gtest.h>
#include <sys/time.h>
#include "taos.h"
//#include "tsdb.h"
//#include "testCommon.h"
#include "tstoken.h"
#include "tutil.h"
#include "tcache.h"
#include "ttimer.h"
#include "ttime.h"
namespace {
int32_t tsMaxMgmtConnections = 10000;
int32_t tsMaxMeterConnections = 200;
}
// test cache
TEST(testCase, client_cache_test) {
const int32_t REFRESH_TIME_IN_SEC = 2;
void* tscTmr = taosTmrInit (tsMaxMgmtConnections*2, 200, 6000, "TSC");
SCacheObj* tscCacheHandle = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
char* key1 = "test1";
char* data1 = "test11";
char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1), 1);
sleep(REFRESH_TIME_IN_SEC+1);
printf("obj is still valid: %s\n", cachedObj);
char* data2 = "test22";
taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
/* the object is cleared by cache clean operation */
cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2), 20);
printf("after updated: %s\n", cachedObj);
printf("start to remove data from cache\n");
taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
printf("end of removing data from cache\n");
getchar();
char* key3 = "test2";
char* data3 = "kkkkkkk";
char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3), 1);
printf("%s\n", cachedObj2);
taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
sleep(3);
char* d = (char*) taosCacheAcquireByName(tscCacheHandle, key3);
// assert(d == NULL);
char* key5 = "test5";
char* data5 = "data5kkkkk";
cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5), 20);
char* data6= "new Data after updated";
taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6), 20);
printf("%s\n", cachedObj2);
taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true);
char* data7 = "add call update procedure";
cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7), 20);
printf("%s\n=======================================\n\n", cachedObj2);
char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5);
taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true);
taosCacheRelease(tscCacheHandle, (void**) &cc, false);
char* data8 = "ttft";
char* key6 = "key6";
char* ft = (char*) taosCachePut(tscCacheHandle, key6, data8, strlen(data8), 20);
taosCacheRelease(tscCacheHandle, (void**) &ft, false);
/**
* 140ns
*/
uint64_t startTime = taosGetTimestampUs();
printf("Cache Performance Test\nstart time:%lld\n", startTime);
for(int32_t i=0; i<1000; ++i) {
char* dd = (char*) taosCacheAcquireByName(tscCacheHandle, key6);
if (dd != NULL) {
// printf("get the data\n");
} else {
printf("data has been released\n");
}
taosCacheRelease(tscCacheHandle, (void**) &dd, false);
}
uint64_t endTime = taosGetTimestampUs();
int64_t el = endTime - startTime;
printf("End of Test, %lld\nTotal Elapsed Time:%lld us.avg:%f us\n", endTime, el, el/1000.0);
taosCacheCleanup(tscCacheHandle);
}
TEST(testCase, cache_resize_test) {
const int32_t REFRESH_TIME_IN_SEC = 2;
void* tscTmr = taosTmrInit (1000*2, 200, 6000, "TSC");
auto* pCache = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
char key[256] = {0};
char data[1024] = "abcdefghijk";
int32_t len = strlen(data);
uint64_t startTime = taosGetTimestampUs();
int32_t num = 10000;
for(int32_t i = 0; i < num; ++i) {
int32_t len = sprintf(key, "abc_%7d", i);
taosCachePut(pCache, key, data, len, 3600);
}
uint64_t endTime = taosGetTimestampUs();
printf("add %d object cost:%lld us, avg:%f us\n", num, endTime - startTime, (endTime-startTime)/(double)num);
startTime = taosGetTimestampUs();
for(int32_t i = 0; i < num; ++i) {
int32_t len = sprintf(key, "abc_%7d", i);
void* k = taosCacheAcquireByName(pCache, key);
assert(k != 0);
}
endTime = taosGetTimestampUs();
printf("retrieve %d object cost:%lld us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num);
taosCacheCleanup(pCache);
taosMsleep(20000);
getchar();
}

156
src/util/tests/hashTest.cpp Normal file
View File

@ -0,0 +1,156 @@
#include <gtest/gtest.h>
#include <limits.h>
#include <taosdef.h>
#include <iostream>
#include "hash.h"
#include "taos.h"
#include "ttime.h"
namespace {
// the simple test code for basic operations
void simpleTest() {
auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
ASSERT_EQ(taosHashGetSize(hashTable), 0);
// put 400 elements in the hash table
for(int32_t i = -200; i < 200; ++i) {
taosHashPut(hashTable, (const char*) &i, sizeof(int32_t), (char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), 400);
for(int32_t i = 0; i < 200; ++i) {
char* p = (char*) taosHashGet(hashTable, (const char*) &i, sizeof(int32_t));
ASSERT_TRUE(p != nullptr);
ASSERT_EQ(*reinterpret_cast<int32_t*>(p), i);
}
for(int32_t i = 1000; i < 2000; ++i) {
taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), 400);
for(int32_t i = 0; i < 100; ++i) {
taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), 300);
for(int32_t i = 100; i < 150; ++i) {
taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), 250);
taosHashCleanup(hashTable);
}
void stringKeyTest() {
auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false);
ASSERT_EQ(taosHashGetSize(hashTable), 0);
char key[128] = {0};
// put 200 elements in the hash table
for(int32_t i = 0; i < 1000; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), 1000);
for(int32_t i = 0; i < 1000; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
char* p = (char*) taosHashGet(hashTable, key, len);
ASSERT_TRUE(p != nullptr);
ASSERT_EQ(*reinterpret_cast<int32_t*>(p), i);
}
for(int32_t i = 500; i < 1000; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
taosHashRemove(hashTable, key, len);
}
ASSERT_EQ(taosHashGetSize(hashTable), 500);
for(int32_t i = 0; i < 499; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
taosHashRemove(hashTable, key, len);
}
ASSERT_EQ(taosHashGetSize(hashTable), 1);
taosHashCleanup(hashTable);
}
void functionTest() {
}
/**
* evaluate the performance issue, by add 10million elements in to hash table in
* a single threads situation
*/
void noLockPerformanceTest() {
auto* hashTable = (SHashObj*) taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false);
ASSERT_EQ(taosHashGetSize(hashTable), 0);
char key[128] = {0};
int32_t num = 5000000;
int64_t st = taosGetTimestampUs();
// put 10M elements in the hash table
for(int32_t i = 0; i < num; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), num);
int64_t et = taosGetTimestampUs();
printf("Elpased time:%" PRId64 " us to add %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num);
st = taosGetTimestampUs();
for(int32_t i = 0; i < num; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
char* p = (char*) taosHashGet(hashTable, key, len);
ASSERT_TRUE(p != nullptr);
ASSERT_EQ(*reinterpret_cast<int32_t*>(p), i);
}
et = taosGetTimestampUs();
printf("Elpased time:%" PRId64 " us to fetch all %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num);
printf("The maximum length of overflow linklist in hash table is:%d\n", taosHashGetMaxOverflowLinkLength(hashTable));
taosHashCleanup(hashTable);
}
void multithreadsTest() {
//todo
}
// check the function robustness
void invalidOperationTest() {
}
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
TEST(testCase, hashTest) {
// simpleTest();
// stringKeyTest();
// noLockPerformanceTest();
// multithreadsTest();
}

355
src/util/tests/skiplistTest.cpp Normal file
View File

@ -0,0 +1,355 @@
#include <gtest/gtest.h>
#include <limits.h>
#include <taosdef.h>
#include <iostream>
#include "taosmsg.h"
#include "tskiplist.h"
#include "ttime.h"
#include "tutil.h"
namespace {
char* getkey(const void* data) { return (char*)(data); }
void doubleSkipListTest() {
SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0, false, true, getkey);
double doubleVal[1000] = {0};
int32_t size = 20000;
printf("generated %d keys is: \n", size);
for (int32_t i = 0; i < size; ++i) {
if (i < 1000) {
doubleVal[i] = i * 0.997;
}
int32_t level = 0;
int32_t size = 0;
tSkipListRandNodeInfo(pSkipList, &level, &size);
auto d = (SSkipListNode*)calloc(1, size + sizeof(double) * 2);
d->level = level;
double* key = (double*)SL_GET_NODE_KEY(pSkipList, d);
key[0] = i * 0.997;
key[1] = i * 0.997;
tSkipListPut(pSkipList, d);
}
printf("the first level of skip list is:\n");
tSkipListPrint(pSkipList, 1);
#if 0
SSkipListNode **pNodes = NULL;
SSkipListKey sk;
for (int32_t i = 0; i < 100; ++i) {
sk.nType = TSDB_DATA_TYPE_DOUBLE;
int32_t idx = abs((i * rand()) % 1000);
sk.dKey = doubleVal[idx];
int32_t size = tSkipListGets(pSkipList, &sk, &pNodes);
printf("the query result size is: %d\n", size);
for (int32_t j = 0; j < size; ++j) {
printf("the result is: %lf\n", pNodes[j]->key.dKey);
}
if (size > 0) {
tfree(pNodes);
}
}
#endif
printf("double test end...\n");
tSkipListDestroy(pSkipList);
}
void randKeyTest() {
SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_INT, sizeof(int32_t), 0, false, true, getkey);
int32_t size = 200000;
srand(time(NULL));
printf("generated %d keys is: \n", size);
for (int32_t i = 0; i < size; ++i) {
int32_t level = 0;
int32_t s = 0;
tSkipListRandNodeInfo(pSkipList, &level, &s);
auto d = (SSkipListNode*)calloc(1, s + sizeof(int32_t) * 2);
d->level = level;
int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d);
key[0] = rand() % 1000000000;
key[1] = key[0];
tSkipListPut(pSkipList, d);
}
printf("the first level of skip list is:\n");
tSkipListPrint(pSkipList, 1);
printf("the sec level of skip list is:\n");
tSkipListPrint(pSkipList, 2);
printf("the 5 level of skip list is:\n");
tSkipListPrint(pSkipList, 5);
tSkipListDestroy(pSkipList);
}
void stringKeySkiplistTest() {
const int32_t max_key_size = 12;
SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_BINARY, max_key_size, 0, false, true, getkey);
int32_t level = 0;
int32_t headsize = 0;
tSkipListRandNodeInfo(pSkipList, &level, &headsize);
auto pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double));
pNode->level = level;
char* d = SL_GET_NODE_DATA(pNode);
strncpy(d, "nyse", 5);
*(double*)(d + max_key_size) = 12;
tSkipListPut(pSkipList, pNode);
tSkipListRandNodeInfo(pSkipList, &level, &headsize);
pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double));
pNode->level = level;
d = SL_GET_NODE_DATA(pNode);
strncpy(d, "beijing", 8);
*(double*)(d + max_key_size) = 911;
tSkipListPut(pSkipList, pNode);
printf("level one------------------\n");
tSkipListPrint(pSkipList, 1);
#if 0
SSkipListNode **pRes = NULL;
int32_t ret = tSkipListGets(pSkipList, &key1, &pRes);
assert(ret == 1);
assert(strcmp(pRes[0]->key.pz, "beijing") == 0);
assert(pRes[0]->key.nType == TSDB_DATA_TYPE_BINARY);
tSkipListDestroyKey(&key1);
tSkipListDestroyKey(&key);
tSkipListDestroy(pSkipList);
free(pRes);
#endif
tSkipListDestroy(pSkipList);
int64_t s = taosGetTimestampUs();
pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_BINARY, 20, 0, false, true, getkey);
char k[256] = {0};
int32_t total = 10000;
for (int32_t i = 0; i < total; ++i) {
int32_t n = sprintf(k, "abc_%d_%d", i, i);
tSkipListRandNodeInfo(pSkipList, &level, &headsize);
auto pNode = (SSkipListNode*)calloc(1, headsize + 20 + sizeof(double));
pNode->level = level;
char* d = SL_GET_NODE_DATA(pNode);
strncpy(d, k, strlen(k));
tSkipListPut(pSkipList, pNode);
}
int64_t e = taosGetTimestampUs();
printf("elapsed time:%lld us to insert %d data, avg:%f us\n", (e - s), total, (double)(e - s) / total);
printf("level two------------------\n");
tSkipListPrint(pSkipList, 1);
#if 0
SSkipListNode **pres = NULL;
s = taosGetTimestampMs();
for (int32_t j = 0; j < total; ++j) {
int32_t n = sprintf(k, "abc_%d_%d", j, j);
key = tSkipListCreateKey(TSDB_DATA_TYPE_BINARY, k, n);
int32_t num = tSkipListGets(pSkipList, &key, &pres);
assert(num > 0);
// tSkipListRemove(pSkipList, &key);
tSkipListRemoveNode(pSkipList, pres[0]);
if (num > 0) {
tfree(pres);
}
}
e = taosGetTimestampMs();
printf("elapsed time:%lldms\n", e - s);
#endif
tSkipListDestroy(pSkipList);
}
void skiplistPerformanceTest() {
SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0, false, false, getkey);
int32_t size = 1000000;
int64_t prev = taosGetTimestampMs();
int64_t s = prev;
int32_t level = 0;
int32_t headsize = 0;
int32_t unit = MAX_SKIP_LIST_LEVEL * POINTER_BYTES * 2 + sizeof(double) * 2 + sizeof(int16_t);
char* total = (char*)calloc(1, unit * size);
char* p = total;
for (int32_t i = 0; i < size; ++i) {
tSkipListRandNodeInfo(pSkipList, &level, &headsize);
SSkipListNode* d = (SSkipListNode*)p;
p += headsize + sizeof(double) * 2;
d->level = level;
double* v = (double*)SL_GET_NODE_DATA(d);
v[0] = i * 0.997;
v[1] = i * 0.997;
tSkipListPut(pSkipList, d);
if (i % 100000 == 0) {
int64_t cur = taosGetTimestampMs();
int64_t elapsed = cur - prev;
printf("add %d, elapsed time: %lld ms, avg elapsed:%f ms, total:%d\n", 100000, elapsed, elapsed / 100000.0, i);
prev = cur;
}
}
int64_t e = taosGetTimestampMs();
printf("total:%lld ms, avg:%f\n", e - s, (e - s) / (double)size);
printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level);
assert(tSkipListGetSize(pSkipList) == size);
// printf("the level of skiplist is:\n");
//
// printf("level two------------------\n");
// tSkipListPrint(pSkipList, 2);
//
// printf("level three------------------\n");
// tSkipListPrint(pSkipList, 3);
//
// printf("level four------------------\n");
// tSkipListPrint(pSkipList, 4);
//
// printf("level nine------------------\n");
// tSkipListPrint(pSkipList, 10);
int64_t st = taosGetTimestampMs();
#if 0
for (int32_t i = 0; i < 100000; i += 1) {
key.dKey = i * 0.997;
tSkipListRemove(pSkipList, &key);
}
#endif
int64_t et = taosGetTimestampMs();
printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st);
assert(tSkipListGetSize(pSkipList) == size);
tSkipListDestroy(pSkipList);
tfree(total);
}
// todo not support duplicated key yet
void duplicatedKeyTest() {
#if 0
SSkipListKey key;
key.nType = TSDB_DATA_TYPE_INT;
SSkipListNode **pNodes = NULL;
SSkipList *pSkipList = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_INT, sizeof(int));
for (int32_t i = 0; i < 10000; ++i) {
for (int32_t j = 0; j < 5; ++j) {
key.i64Key = i;
tSkipListPut(pSkipList, "", &key, 1);
}
}
tSkipListPrint(pSkipList, 1);
for (int32_t i = 0; i < 100; ++i) {
key.i64Key = rand() % 1000;
int32_t size = tSkipListGets(pSkipList, &key, &pNodes);
assert(size == 5);
tfree(pNodes);
}
tSkipListDestroy(pSkipList);
#endif
}
} // namespace
TEST(testCase, skiplist_test) {
assert(sizeof(SSkipListKey) == 8);
srand(time(NULL));
stringKeySkiplistTest();
doubleSkipListTest();
skiplistPerformanceTest();
duplicatedKeyTest();
randKeyTest();
// tSKipListQueryCond q;
// q.upperBndRelOptr = true;
// q.lowerBndRelOptr = true;
// q.upperBnd.nType = TSDB_DATA_TYPE_DOUBLE;
// q.lowerBnd.nType = TSDB_DATA_TYPE_DOUBLE;
// q.lowerBnd.dKey = 120;
// q.upperBnd.dKey = 171.989;
/*
int32_t size = tSkipListQuery(pSkipList, &q, &pNodes);
for (int32_t i = 0; i < size; ++i) {
printf("-----%lf\n", pNodes[i]->key.dKey);
}
printf("the range query result size is: %d\n", size);
tfree(pNodes);
SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20);
for (int32_t i = 0; i < 8; i += 2) {
pKeys[i].dKey = i * 0.997;
pKeys[i].nType = TSDB_DATA_TYPE_DOUBLE;
printf("%lf ", pKeys[i].dKey);
}
int32_t r = tSkipListPointQuery(pSkipList, pKeys, 8, EXCLUDE_POINT_QUERY, &pNodes);
printf("\nthe exclude query result is: %d\n", r);
for (int32_t i = 0; i < r; ++i) {
// printf("%lf ", pNodes[i]->key.dKey);
}
tfree(pNodes);
free(pKeys);*/
}

178
src/util/tests/stringTest.cpp Normal file
View File

@ -0,0 +1,178 @@
#include <gtest/gtest.h>
#include <limits.h>
#include <taosdef.h>
#include <iostream>
#include "taos.h"
#include "tutil.h"
TEST(testCase, string_dequote_test) {
char t1[] = "'abc'";
int32_t len = strdequote(t1);
EXPECT_EQ(3, len);
EXPECT_STRCASEEQ(t1, "abc");
char t2[] = "\"abc\"";
len = strdequote(t2);
EXPECT_EQ(3, len);
EXPECT_STRCASEEQ(t1, "abc");
char t21[] = " abc ";
strtrim(t21);
EXPECT_STREQ("abc", t21);
EXPECT_EQ(3, strlen(t21));
}
TEST(testCase, string_replace_test) {
char t3[] = "abc01abc02abc";
char* ret = strreplace(t3, "abc", "7");
EXPECT_EQ(strlen(ret), 7);
EXPECT_STREQ("7017027", ret);
free(ret);
char t4[] = "a01a02b03c04d05";
ret = strreplace(t4, "0", "9999999999");
EXPECT_EQ(strlen(ret), 5 * 10 + 10);
EXPECT_STREQ("a99999999991a99999999992b99999999993c99999999994d99999999995", ret);
free(ret);
char t5[] = "abc";
ret = strreplace(t5, "abc", "12345678901234567890");
EXPECT_EQ(strlen(ret), 20);
EXPECT_STREQ("12345678901234567890", ret);
free(ret);
char t6[] = "abc";
ret = strreplace(t6, "def", "abc");
EXPECT_EQ(strlen(ret), 3);
EXPECT_STREQ("abc", ret);
free(ret);
char t7[] = "abcde000000000000001234";
ret = strreplace(t7, "ab", "0000000");
EXPECT_EQ(strlen(ret), 28);
EXPECT_STREQ("0000000cde000000000000001234", ret);
free(ret);
char t8[] = "abc\ndef";
char t[] = {10, 0};
char f1[] = "\\n";
int32_t fx = strlen(f1);
ret = strreplace(t8, "\n", "\\n");
EXPECT_EQ(strlen(ret), 8);
EXPECT_STREQ("abc\\ndef", ret);
free(ret);
char t9[] = "abc\\ndef";
ret = strreplace(t9, "\\n", "\n");
EXPECT_EQ(strlen(ret), 7);
EXPECT_STREQ("abc\ndef", ret);
free(ret);
char t10[] = "abcdef";
ret = strreplace(t10, "", "0");
EXPECT_EQ(strlen(ret), 6);
EXPECT_STREQ("abcdef", ret);
free(ret);
}
TEST(testCase, string_tolower_test) {
char t[1024] = {1};
memset(t, 1, tListLen(t));
const char* a1 = "ABC";
strtolower(t, a1);
EXPECT_STREQ(t, "abc");
memset(t, 1, tListLen(t));
const char* a2 = "ABC\'ABC\'D";
strtolower(t, a2);
EXPECT_STREQ(t, "abc\'ABC\'d");
memset(t, 1, tListLen(t));
const char* a3 = "";
strtolower(t, a3);
EXPECT_STREQ(t, "");
memset(t, 1, tListLen(t));
const char* a4 = "\"AbcDEF\"";
strtolower(t, a4);
EXPECT_STREQ(t, a4);
memset(t, 1, tListLen(t));
const char* a5 = "1234\"AbcDEF\"456";
strtolower(t, a5);
EXPECT_STREQ(t, a5);
memset(t, 1, tListLen(t));
const char* a6 = "1234";
strtolower(t, a6);
EXPECT_STREQ(t, a6);
}
TEST(testCase, string_strnchr_test) {
char t[1024] = {0};
memset(t, 1, tListLen(t));
char a1[] = "AB.C";
EXPECT_TRUE(strnchr(a1, '.', strlen(a1), true) != NULL);
char a2[] = "abc.";
EXPECT_TRUE(strnchr(a2, '.', strlen(a2), true) != NULL);
char a8[] = "abc.";
EXPECT_TRUE(strnchr(a8, '.', 1, true) == NULL);
char a3[] = ".abc";
EXPECT_TRUE(strnchr(a3, '.', strlen(a3), true) != NULL);
char a4[] = "'.abc'";
EXPECT_TRUE(strnchr(a4, '.', strlen(a4), true) == NULL);
char a5[] = "'.abc.'abc";
EXPECT_TRUE(strnchr(a5, '.', strlen(a5), true) == NULL);
char a6[] = "0123456789.";
EXPECT_TRUE(strnchr(a6, '.', strlen(a6), true) != NULL);
char a7[] = "0123456789.";
EXPECT_TRUE(strnchr(a7, '.', 3, true) == NULL);
char a9[] = "0123456789.";
EXPECT_TRUE(strnchr(a9, '.', 0, true) == NULL);
char a10[] = "0123456789'.'";
EXPECT_TRUE(strnchr(a10, '.', strlen(a10), true) == NULL);
}
// TEST(testCase, cache_resize_test) {
// char a11[] = "abc'.'";
// EXPECT_TRUE(strnchr(a11, '.', strlen(a11), false) != NULL);
// char a12[] = "abc'-'";
// EXPECT_TRUE(strnchr(a12, '-', strlen(a12), false) != NULL);
// char a15[] = "abc'-'";
// EXPECT_TRUE(strnchr(a15, '-', strlen(a15), true) == NULL);
// char a13[] = "'-'";
// EXPECT_TRUE(strnchr(a13, '-', strlen(a13), false) != NULL);
// char a14[] = "'-'";
// EXPECT_TRUE(strnchr(a14, '-', strlen(a14), true) == NULL);
// char a16[] = "'-'.";
// EXPECT_TRUE(strnchr(a16, '.', strlen(a16), true) != NULL);
// }

4
src/vnode/detail/src/vnodeQueryImpl.c
View File

@ -5312,7 +5312,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
while (1) {
// check if query is killed or not set the status of query to pass the status check
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK);
return cnt;
}
@ -6375,7 +6375,7 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) {
pRuntimeEnv->scanFlag = REPEAT_SCAN;
/* check if query is killed or not */
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK);
return;
}

24
src/vnode/detail/src/vnodeQueryProcess.c
View File

@ -105,7 +105,7 @@ static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
int32_t start = pSupporter->pSidSet->starterPos[groupIdx];
int32_t end = pSupporter->pSidSet->starterPos[groupIdx + 1] - 1;
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
return;
}
@ -276,7 +276,7 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
int64_t st = taosGetTimestampUs();
while (1) {
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
break;
}
@ -363,7 +363,7 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
int32_t j = QUERY_IS_ASC_QUERY(pQuery) ? 0 : numOfBlocks - 1;
for (; j < numOfBlocks && j >= 0; j += step) {
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
break;
}
@ -603,7 +603,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
pSupporter->meterIdx = start;
for (int32_t k = start; k <= end; ++k, pSupporter->meterIdx++) {
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK);
return;
}
@ -630,7 +630,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
pSupporter->subgroupIdx);
for (int32_t k = start; k <= end; ++k) {
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK);
return;
}
@ -681,7 +681,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
while (pSupporter->meterIdx < pSupporter->numOfMeters) {
int32_t k = pSupporter->meterIdx;
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK);
return;
}
@ -958,7 +958,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) {
doMultiMeterSupplementaryScan(pQInfo);
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
dTrace("QInfo:%p query killed, abort", pQInfo);
return;
}
@ -998,7 +998,7 @@ static void vnodeSingleTableFixedOutputProcessor(SQInfo *pQInfo) {
vnodeScanAllData(pRuntimeEnv);
doFinalizeResult(pRuntimeEnv);
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
return;
}
@ -1033,7 +1033,7 @@ static void vnodeSingleTableMultiOutputProcessor(SQInfo *pQInfo) {
vnodeScanAllData(pRuntimeEnv);
doFinalizeResult(pRuntimeEnv);
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
return;
}
@ -1087,7 +1087,7 @@ static void vnodeSingleMeterIntervalMainLooper(STableQuerySupportObj *pSupporter
initCtxOutputBuf(pRuntimeEnv);
vnodeScanAllData(pRuntimeEnv);
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
return;
}
@ -1301,7 +1301,7 @@ void vnodeSingleTableQuery(SSchedMsg *pMsg) {
pQInfo->useconds += (taosGetTimestampUs() - st);
/* check if query is killed or not */
if (isQueryKilled(pQuery)) {
if (isQueryKilled(pQInfo)) {
dTrace("QInfo:%p query is killed", pQInfo);
pQInfo->over = 1;
} else {
@ -1345,7 +1345,7 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) {
/* record the total elapsed time */
pQInfo->useconds += (taosGetTimestampUs() - st);
pQInfo->over = isQueryKilled(pQuery) ? 1 : 0;
pQInfo->over = isQueryKilled(pQInfo) ? 1 : 0;
taosInterpoSetStartInfo(&pQInfo->pTableQuerySupporter->runtimeEnv.interpoInfo, pQuery->pointsRead,
pQInfo->query.interpoType);

Some files were not shown because too many files have changed in this diff Show More