xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into feature/crash_gen

This commit is contained in:
Steven Li 2020-05-24 15:15:20 -07:00
parent 5b4e7a6d54 282d29815f
commit 636249571b
295 changed files with 32792 additions and 3206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.gitignore vendored
View File

@ -41,3 +41,27 @@ pysim/
# Doxygen Generated files
html/
/.vs
/CMakeFiles/3.10.2
/CMakeCache.txt
/Makefile
/*.cmake
/deps
/src/cq/test/CMakeFiles/cqtest.dir/*.cmake
*.cmake
/src/cq/test/CMakeFiles/cqtest.dir/*.make
*.make
link.txt
*.internal
*.includecache
*.marks
Makefile
CMakeError.log
*.log
/CMakeFiles/CMakeRuleHashes.txt
/CMakeFiles/Makefile2
/CMakeFiles/TargetDirectories.txt
/CMakeFiles/cmake.check_cache
/out/isenseconfig/WSL-Clang-Debug
/out/isenseconfig/WSL-GCC-Debug
/test/cfg

5
.travis.yml
View File

@ -26,6 +26,7 @@ matrix:
- python3-pip
- python3-setuptools
- valgrind
- psmisc
before_script:
- cd ${TRAVIS_BUILD_DIR}
@ -89,6 +90,7 @@ matrix:
esac
- os: linux
dist: bionic
language: c
compiler: gcc
env: COVERITY_SCAN=true
@ -124,6 +126,7 @@ matrix:
branch_pattern: coverity_scan
- os: linux
dist: bionic
language: c
compiler: gcc
env: ENV_COVER=true
@ -142,6 +145,7 @@ matrix:
- python3-pip
- python3-setuptools
- lcov
- psmisc
before_script:
- cd ${TRAVIS_BUILD_DIR}
@ -228,6 +232,7 @@ matrix:
- make > /dev/null
- os: linux
dist: bionic
language: c
compiler: clang
env: DESC="linux/clang build"

25
CMakeSettings.json Normal file
View File

@ -0,0 +1,25 @@
{
"configurations": [
{
"name": "WSL-GCC-Debug",
"generator": "Unix Makefiles",
"configurationType": "Debug",
"buildRoot": "${projectDir}\\build\\",
"installRoot": "${projectDir}\\out\\install\\${name}",
"cmakeExecutable": "/usr/bin/cmake",
"cmakeCommandArgs": "",
"buildCommandArgs": "",
"ctestCommandArgs": "",
"inheritEnvironments": [ "linux_x64" ],
"wslPath": "${defaultWSLPath}",
"addressSanitizerRuntimeFlags": "detect_leaks=0",
"variables": [
{
"name": "CMAKE_INSTALL_PREFIX",
"value": "/mnt/d/TDengine/TDengine/build",
"type": "PATH"
}
]
}
]
}

8
src/client/inc/tscSecondaryMerge.h
View File

@ -21,7 +21,7 @@ extern "C" {
#endif
#include "qextbuffer.h"
#include "qinterpolation.h"
#include "qfill.h"
#include "taosmsg.h"
#include "tlosertree.h"
#include "tsclient.h"
@ -60,7 +60,7 @@ typedef struct SLocalReducer {
char * prevRowOfInput;
tFilePage * pResultBuf;
int32_t nResultBufSize;
char * pBufForInterpo; // intermediate buffer for interpolation
// char * pBufForInterpo; // intermediate buffer for interpolation
tFilePage * pTempBuffer;
struct SQLFunctionCtx *pCtx;
int32_t rowSize; // size of each intermediate result.
@ -68,9 +68,9 @@ typedef struct SLocalReducer {
bool hasPrevRow; // cannot be released
bool hasUnprocessedRow;
tOrderDescriptor * pDesc;
SColumnModel * resColModel;
SColumnModel * resColModel;
tExtMemBuffer ** pExtMemBuffer; // disk-based buffer
SInterpolationInfo interpolationInfo; // interpolation support structure
SFillInfo* pFillInfo; // interpolation support structure
char * pFinalRes; // result data after interpo
tFilePage * discardData;
SResultInfo * pResInfo;

13
src/client/inc/tsclient.h
View File

@ -30,10 +30,10 @@ extern "C" {
#include "tsqlfunction.h"
#include "tutil.h"
#include "qExecutor.h"
#include "qsqlparser.h"
#include "qsqltype.h"
#include "qtsbuf.h"
#include "queryExecutor.h"
// forward declaration
struct SSqlInfo;
@ -210,7 +210,7 @@ typedef struct SQueryInfo {
SLimitVal slimit;
STagCond tagCond;
SOrderVal order;
int16_t interpoType; // interpolate type
int16_t fillType; // interpolate type
int16_t numOfTables;
STableMetaInfo **pTableMetaInfo;
struct STSBuf * tsBuf;
@ -226,11 +226,8 @@ typedef struct {
int command;
uint8_t msgType;
union {
bool existsCheck; // check if the table exists or not
bool autoCreated; // if the table is missing, on-the-fly create it. during getmeterMeta
int8_t dataSourceType; // load data from file or not
};
bool autoCreated; // if the table is missing, on-the-fly create it. during getmeterMeta
int8_t dataSourceType; // load data from file or not
union {
int32_t count;
@ -263,7 +260,7 @@ typedef struct SResRec {
typedef struct {
int64_t numOfRows; // num of results in current retrieved
int64_t numOfTotal; // num of total results
int64_t numOfTotalInCurrentClause; // num of total result in current subclause
int64_t numOfClauseTotal; // num of total result in current subclause
char * pRsp;
int32_t rspType;
int32_t rspLen;

2
src/client/src/tscAsync.c
View File

@ -147,7 +147,7 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) {
// local merge has handle this situation during super table non-projection query.
if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE) {
pRes->numOfTotalInCurrentClause += pRes->numOfRows;
pRes->numOfClauseTotal += pRes->numOfRows;
}
(*pSql->fetchFp)(param, tres, numOfRows);

27
src/client/src/tscFunctionImpl.c
View File

@ -16,8 +16,8 @@
#include "os.h"
#include "qast.h"
#include "qextbuffer.h"
#include "qfill.h"
#include "qhistogram.h"
#include "qinterpolation.h"
#include "qpercentile.h"
#include "qsyntaxtreefunction.h"
#include "qtsbuf.h"
@ -3418,6 +3418,7 @@ static void spread_function(SQLFunctionCtx *pCtx) {
int32_t numOfElems = pCtx->size;
// todo : opt with pre-calculated result
// column missing cause the hasNull to be true
if (usePreVal(pCtx)) {
numOfElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
@ -3446,13 +3447,13 @@ static void spread_function(SQLFunctionCtx *pCtx) {
}
}
} else {
if (pInfo->min > pCtx->param[1].dKey) {
pInfo->min = pCtx->param[1].dKey;
}
if (pInfo->max < pCtx->param[2].dKey) {
pInfo->max = pCtx->param[2].dKey;
}
// if (pInfo->min > pCtx->param[1].dKey) {
// pInfo->min = pCtx->param[1].dKey;
// }
//
// if (pInfo->max < pCtx->param[2].dKey) {
// pInfo->max = pCtx->param[2].dKey;
// }
}
void *pData = GET_INPUT_CHAR(pCtx);
@ -3866,16 +3867,16 @@ static void interp_function(SQLFunctionCtx *pCtx) {
SInterpInfoDetail *pInfoDetail = interpInfo.pInterpDetail;
/* set no output result */
if (pInfoDetail->type == TSDB_INTERPO_NONE) {
if (pInfoDetail->type == TSDB_FILL_NONE) {
pCtx->param[3].i64Key = 0;
} else if (pInfoDetail->primaryCol == 1) {
*(TSKEY *)pCtx->aOutputBuf = pInfoDetail->ts;
} else {
if (pInfoDetail->type == TSDB_INTERPO_NULL) {
if (pInfoDetail->type == TSDB_FILL_NULL) {
setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
} else if (pInfoDetail->type == TSDB_INTERPO_SET_VALUE) {
} else if (pInfoDetail->type == TSDB_FILL_SET_VALUE) {
tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->inputType);
} else if (pInfoDetail->type == TSDB_INTERPO_PREV) {
} else if (pInfoDetail->type == TSDB_FILL_PREV) {
char *data = pCtx->param[1].pz;
char *pVal = data + TSDB_KEYSIZE;
@ -3886,7 +3887,7 @@ static void interp_function(SQLFunctionCtx *pCtx) {
assignVal(pCtx->aOutputBuf, pVal, pCtx->outputBytes, pCtx->outputType);
}
} else if (pInfoDetail->type == TSDB_INTERPO_LINEAR) {
} else if (pInfoDetail->type == TSDB_FILL_LINEAR) {
char *data1 = pCtx->param[1].pz;
char *data2 = pCtx->param[2].pz;

2
src/client/src/tscLocal.c
View File

@ -56,7 +56,7 @@ static int32_t getToStringLength(const char *pData, int32_t length, int32_t type
} break;
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT:
len = sprintf(buf, "%" PRId64 "", *(int64_t *)pData);
len = sprintf(buf, "%" PRId64, *(int64_t *)pData);
break;
case TSDB_DATA_TYPE_BOOL:
len = MAX_BOOL_TYPE_LENGTH;

36
src/client/src/tscParseInsert.c
View File

@ -42,35 +42,35 @@ enum {
static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows);
static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) {
int32_t numType = isValidNumber(pToken);
if (TK_ILLEGAL == numType) {
return numType;
}
// int32_t numType = isValidNumber(pToken);
// if (TK_ILLEGAL == numType) {
// return numType;
// }
int32_t radix = 10;
if (numType == TK_HEX) {
if (pToken->type == TK_HEX) {
radix = 16;
} else if (numType == TK_OCT) {
} else if (pToken->type == TK_OCT) {
radix = 8;
} else if (numType == TK_BIN) {
} else if (pToken->type == TK_BIN) {
radix = 2;
}
errno = 0;
*value = strtoll(pToken->z, endPtr, radix);
return numType;
return pToken->type;
}
static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) {
int32_t numType = isValidNumber(pToken);
if (TK_ILLEGAL == numType) {
return numType;
}
// int32_t numType = isValidNumber(pToken);
// if (TK_ILLEGAL == numType) {
// return numType;
// }
errno = 0;
*value = strtod(pToken->z, endPtr);
return numType;
return pToken->type;
}
int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) {
@ -927,6 +927,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
}
}
// 3. calculate the actual data size of STagData
pCmd->payloadLen = sizeof(pTag->name) + sizeof(pTag->dataLen);
for (int32_t t = 0; t < numOfTags; ++t) {
pTag->dataLen += pTagSchema[t].bytes;
pCmd->payloadLen += pTagSchema[t].bytes;
}
pTag->dataLen = htonl(pTag->dataLen);
if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) {
return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr);
}
@ -948,7 +956,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
} else {
sql = sToken.z;
}
code = tscGetTableMeta(pSql, pTableMetaInfo);
code = tscGetMeterMetaEx(pSql, pTableMetaInfo, false);
if (pCmd->curSql == NULL) {
assert(code == TSDB_CODE_ACTION_IN_PROGRESS);

2
src/client/src/tscPrepare.c
View File

@ -447,7 +447,7 @@ static int insertStmtExecute(STscStmt* stmt) {
SSqlRes *pRes = &pSql->res;
pRes->numOfRows = 0;
pRes->numOfTotal = 0;
pRes->numOfTotalInCurrentClause = 0;
pRes->numOfClauseTotal = 0;
pRes->qhandle = 0;

2
src/client/src/tscProfile.c
View File

@ -23,8 +23,6 @@
void tscSaveSlowQueryFp(void *handle, void *tmrId);
void *tscSlowQueryConn = NULL;
bool tscSlowQueryConnInitialized = false;
TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
void *param, void **taos);
void tscInitConnCb(void *param, TAOS_RES *result, int code) {
char *sql = param;

46
src/client/src/tscSQLParser.c
View File

@ -4020,19 +4020,19 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) {
}
if (strncasecmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) {
pQueryInfo->interpoType = TSDB_INTERPO_NONE;
pQueryInfo->fillType = TSDB_FILL_NONE;
} else if (strncasecmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4) {
pQueryInfo->interpoType = TSDB_INTERPO_NULL;
pQueryInfo->fillType = TSDB_FILL_NULL;
for (int32_t i = START_INTERPO_COL_IDX; i < size; ++i) {
TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
setNull((char*)&pQueryInfo->defaultVal[i], pFields->type, pFields->bytes);
}
} else if (strncasecmp(pItem->pVar.pz, "prev", 4) == 0 && pItem->pVar.nLen == 4) {
pQueryInfo->interpoType = TSDB_INTERPO_PREV;
pQueryInfo->fillType = TSDB_FILL_PREV;
} else if (strncasecmp(pItem->pVar.pz, "linear", 6) == 0 && pItem->pVar.nLen == 6) {
pQueryInfo->interpoType = TSDB_INTERPO_LINEAR;
pQueryInfo->fillType = TSDB_FILL_LINEAR;
} else if (strncasecmp(pItem->pVar.pz, "value", 5) == 0 && pItem->pVar.nLen == 5) {
pQueryInfo->interpoType = TSDB_INTERPO_SET_VALUE;
pQueryInfo->fillType = TSDB_FILL_SET_VALUE;
if (pFillToken->nExpr == 1) {
return invalidSqlErrMsg(pQueryInfo->msg, msg1);
@ -4416,6 +4416,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(pQueryInfo->msg, msg13);
}
pAlterSQL->tagData.dataLen = pTagsSchema->bytes;
// validate the length of binary
if ((pTagsSchema->type == TSDB_DATA_TYPE_BINARY || pTagsSchema->type == TSDB_DATA_TYPE_NCHAR) &&
@ -4539,11 +4540,13 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
return TSDB_CODE_INVALID_SQL;
}
const SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[14] = {
{"resetLog", 8}, {"resetQueryCache", 15}, {"dDebugFlag", 10}, {"rpcDebugFlag", 12},
{"tmrDebugFlag", 12}, {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"mDebugFlag", 10},
{"sdbDebugFlag", 12}, {"httpDebugFlag", 13}, {"monitorDebugFlag", 16}, {"qDebugflag", 10},
{"debugFlag", 9}, {"monitor", 7}};
const int DNODE_DYNAMIC_CFG_OPTIONS_SIZE = 17;
const SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[] = {
{"resetLog", 8}, {"resetQueryCache", 15}, {"debugFlag", 9}, {"mDebugFlag", 10},
{"dDebugFlag", 10}, {"sdbDebugFlag", 12}, {"vDebugFlag", 10}, {"cDebugFlag", 10},
{"httpDebugFlag", 13}, {"monitorDebugFlag", 16}, {"rpcDebugFlag", 12}, {"uDebugFlag", 10},
{"tmrDebugFlag", 12}, {"qDebugflag", 10}, {"sDebugflag", 10}, {"tsdbDebugFlag", 13},
{"monitor", 7}};
SSQLToken* pOptionToken = &pOptions->a[1];
@ -4555,8 +4558,8 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
return TSDB_CODE_SUCCESS;
}
}
} else if ((strncasecmp(DNODE_DYNAMIC_CFG_OPTIONS[13].name, pOptionToken->z, pOptionToken->n) == 0) &&
(DNODE_DYNAMIC_CFG_OPTIONS[13].len == pOptionToken->n)) {
} else if ((strncasecmp(DNODE_DYNAMIC_CFG_OPTIONS[DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1].name, pOptionToken->z, pOptionToken->n) == 0) &&
(DNODE_DYNAMIC_CFG_OPTIONS[DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1].len == pOptionToken->n)) {
SSQLToken* pValToken = &pOptions->a[2];
int32_t val = strtol(pValToken->z, NULL, 10);
if (val != 0 && val != 1) {
@ -4572,7 +4575,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
return TSDB_CODE_INVALID_SQL;
}
for (int32_t i = 2; i < tListLen(DNODE_DYNAMIC_CFG_OPTIONS) - 1; ++i) {
for (int32_t i = 2; i < DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1; ++i) {
const SDNodeDynConfOption* pOption = &DNODE_DYNAMIC_CFG_OPTIONS[i];
if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) {
@ -4823,7 +4826,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBIn
static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
pMsg->maxTables = htonl(pCreateDb->maxTablesPerVnode);
pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize);
pMsg->numOfBlocks = htonl(pCreateDb->numOfBlocks);
pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks);
pMsg->daysPerFile = htonl(pCreateDb->daysPerFile);
pMsg->commitTime = htonl(pCreateDb->commitTime);
pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock);
@ -5550,11 +5553,11 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
// too long tag values will return invalid sql, not be truncated automatically
SSchema* pTagSchema = tscGetTableTagSchema(pStableMeterMetaInfo->pTableMeta);
char* tagVal = pCreateTable->usingInfo.tagdata.data;
STagData* pTag = &pCreateTable->usingInfo.tagdata;
char* tagVal = pTag->data;
int32_t ret = TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < pList->nExpr; ++i) {
if (pTagSchema[i].type == TSDB_DATA_TYPE_BINARY || pTagSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
// validate the length of binary
if (pList->a[i].pVar.nLen + VARSTR_HEADER_SIZE > pTagSchema[i].bytes) {
@ -5562,7 +5565,15 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
}
ret = tVariantDump(&(pList->a[i].pVar), varDataVal(tagVal), pTagSchema[i].type);
varDataSetLen(tagVal, pList->a[i].pVar.nLen);
if (pList->a[i].pVar.nType == TSDB_DATA_TYPE_NULL) {
if (pTagSchema[i].type == TSDB_DATA_TYPE_BINARY) {
varDataSetLen(tagVal, sizeof(uint8_t));
} else {
varDataSetLen(tagVal, sizeof(uint32_t));
}
} else { // todo refactor
varDataSetLen(tagVal, pList->a[i].pVar.nLen);
}
} else {
ret = tVariantDump(&(pList->a[i].pVar), tagVal, pTagSchema[i].type);
}
@ -5585,6 +5596,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
return ret;
}
pTag->dataLen = tagVal - pTag->data;
return TSDB_CODE_SUCCESS;
}

336
src/client/src/tscSecondaryMerge.c
View File

@ -25,7 +25,7 @@
typedef struct SCompareParam {
SLocalDataSource **pLocalData;
tOrderDescriptor * pDesc;
int32_t numOfElems;
int32_t num;
int32_t groupOrderType;
} SCompareParam;
@ -47,11 +47,11 @@ int32_t treeComparator(const void *pLeft, const void *pRight, void *param) {
}
if (pParam->groupOrderType == TSDB_ORDER_DESC) { // desc
return compare_d(pDesc, pParam->numOfElems, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data,
pParam->numOfElems, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data);
return compare_d(pDesc, pParam->num, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data,
pParam->num, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data);
} else {
return compare_a(pDesc, pParam->numOfElems, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data,
pParam->numOfElems, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data);
return compare_a(pDesc, pParam->num, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data,
pParam->num, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data);
}
}
@ -132,6 +132,26 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
}
}
static SFillColInfo* createFillColInfo(SQueryInfo* pQueryInfo) {
int32_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo);
int32_t offset = 0;
SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo));
for(int32_t i = 0; i < numOfCols; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
pFillCol[i].col.bytes = pExpr->resBytes;
pFillCol[i].col.type = pExpr->resType;
pFillCol[i].flag = pExpr->colInfo.flag;
pFillCol[i].col.offset = offset;
pFillCol[i].functionId = pExpr->functionId;
pFillCol[i].defaultVal.i = pQueryInfo->defaultVal[i];
offset += pExpr->resBytes;
}
return pFillCol;
}
void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
SColumnModel *finalmodel, SSqlObj* pSql) {
SSqlCmd* pCmd = &pSql->cmd;
@ -217,24 +237,24 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
ds->pMemBuffer = pMemBuffer[i];
ds->flushoutIdx = j;
ds->filePage.numOfElems = 0;
ds->filePage.num = 0;
ds->pageId = 0;
ds->rowIdx = 0;
tscTrace("%p load data from disk into memory, orderOfVnode:%d, total:%d", pSql, i + 1, idx + 1);
tExtMemBufferLoadData(pMemBuffer[i], &(ds->filePage), j, 0);
#ifdef _DEBUG_VIEW
printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", ds->filePage.numOfElems);
printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", ds->filePage.num);
SSrcColumnInfo colInfo[256] = {0};
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
tscGetSrcColumnInfo(colInfo, pQueryInfo);
tColModelDisplayEx(pDesc->pColumnModel, ds->filePage.data, ds->filePage.numOfElems,
tColModelDisplayEx(pDesc->pColumnModel, ds->filePage.data, ds->filePage.num,
pMemBuffer[0]->numOfElemsPerPage, colInfo);
#endif
if (ds->filePage.numOfElems == 0) { // no data in this flush, the index does not increase
if (ds->filePage.num == 0) { // no data in this flush, the index does not increase
tscTrace("%p flush data is empty, ignore %d flush record", pSql, idx);
tfree(ds);
continue;
@ -254,7 +274,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
SCompareParam *param = malloc(sizeof(SCompareParam));
param->pLocalData = pReducer->pLocalDataSrc;
param->pDesc = pReducer->pDesc;
param->numOfElems = pReducer->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage;
param->num = pReducer->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
param->groupOrderType = pQueryInfo->groupbyExpr.orderType;
@ -295,25 +315,25 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
assert(finalRowLength <= pReducer->rowSize);
pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity);
pReducer->pBufForInterpo = calloc(1, pReducer->nResultBufSize);
// pReducer->pBufForInterpo = calloc(1, pReducer->nResultBufSize);
if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL ||
pReducer->pBufForInterpo == NULL || pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
/*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
tfree(pReducer->pTempBuffer);
tfree(pReducer->discardData);
tfree(pReducer->pResultBuf);
tfree(pReducer->pFinalRes);
tfree(pReducer->pBufForInterpo);
// tfree(pReducer->pBufForInterpo);
tfree(pReducer->prevRowOfInput);
pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
return;
}
size = tscSqlExprNumOfExprs(pQueryInfo);
pReducer->pTempBuffer->numOfElems = 0;
pReducer->pResInfo = calloc(size, sizeof(SResultInfo));
size_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo);
pReducer->pTempBuffer->num = 0;
pReducer->pResInfo = calloc(numOfCols, sizeof(SResultInfo));
tscCreateResPointerInfo(pRes, pQueryInfo);
tscInitSqlContext(pCmd, pReducer, pDesc);
@ -333,55 +353,58 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
pRes->numOfGroups = 0;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);;
int16_t prec = tinfo.precision;
int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey;
TSKEY stime = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey);
int64_t revisedSTime =
taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec);
SInterpolationInfo *pInterpoInfo = &pReducer->interpolationInfo;
taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
pReducer->rowSize);
taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, tinfo.precision);
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
SFillColInfo* pFillCol = createFillColInfo(pQueryInfo);
pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
4096, numOfCols, pQueryInfo->slidingTime, pQueryInfo->fillType, pFillCol);
}
int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) {
pInterpoInfo->pTags[0] = (char *)pInterpoInfo->pTags + POINTER_BYTES * pQueryInfo->groupbyExpr.numOfGroupCols;
if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 && pReducer->pFillInfo != NULL) {
pReducer->pFillInfo->pTags[0] = (char *)pReducer->pFillInfo->pTags + POINTER_BYTES * pQueryInfo->groupbyExpr.numOfGroupCols;
for (int32_t i = 1; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
SSchema *pSchema = getColumnModelSchema(pReducer->resColModel, startIndex + i - 1);
pInterpoInfo->pTags[i] = pSchema->bytes + pInterpoInfo->pTags[i - 1];
pReducer->pFillInfo->pTags[i] = pSchema->bytes + pReducer->pFillInfo->pTags[i - 1];
}
} else {
assert(pInterpoInfo->pTags == NULL);
if (pReducer->pFillInfo != NULL) {
assert(pReducer->pFillInfo->pTags == NULL);
}
}
}
static int32_t tscFlushTmpBufferImpl(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage,
int32_t orderType) {
if (pPage->numOfElems == 0) {
if (pPage->num == 0) {
return 0;
}
assert(pPage->numOfElems <= pDesc->pColumnModel->capacity);
assert(pPage->num <= pDesc->pColumnModel->capacity);
// sort before flush to disk, the data must be consecutively put on tFilePage.
if (pDesc->orderIdx.numOfCols > 0) {
tColDataQSort(pDesc, pPage->numOfElems, 0, pPage->numOfElems - 1, pPage->data, orderType);
tColDataQSort(pDesc, pPage->num, 0, pPage->num - 1, pPage->data, orderType);
}
#ifdef _DEBUG_VIEW
printf("%" PRIu64 " rows data flushed to disk after been sorted:\n", pPage->numOfElems);
tColModelDisplay(pDesc->pColumnModel, pPage->data, pPage->numOfElems, pPage->numOfElems);
printf("%" PRIu64 " rows data flushed to disk after been sorted:\n", pPage->num);
tColModelDisplay(pDesc->pColumnModel, pPage->data, pPage->num, pPage->num);
#endif
// write to cache after being sorted
if (tExtMemBufferPut(pMemoryBuf, pPage->data, pPage->numOfElems) < 0) {
if (tExtMemBufferPut(pMemoryBuf, pPage->data, pPage->num) < 0) {
tscError("failed to save data in temporary buffer");
return -1;
}
pPage->numOfElems = 0;
pPage->num = 0;
return 0;
}
@ -402,17 +425,17 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa
int32_t numOfRows, int32_t orderType) {
SColumnModel *pModel = pDesc->pColumnModel;
if (pPage->numOfElems + numOfRows <= pModel->capacity) {
if (pPage->num + numOfRows <= pModel->capacity) {
tColModelAppend(pModel, pPage, data, 0, numOfRows, numOfRows);
return 0;
}
// current buffer is overflow, flush data to extensive buffer
int32_t numOfRemainEntries = pModel->capacity - pPage->numOfElems;
int32_t numOfRemainEntries = pModel->capacity - pPage->num;
tColModelAppend(pModel, pPage, data, 0, numOfRemainEntries, numOfRows);
// current buffer is full, need to flushed to disk
assert(pPage->numOfElems == pModel->capacity);
assert(pPage->num == pModel->capacity);
int32_t ret = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType);
if (ret != 0) {
return -1;
@ -430,12 +453,12 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa
tColModelAppend(pModel, pPage, data, numOfRows - remain, numOfWriteElems, numOfRows);
if (pPage->numOfElems == pModel->capacity) {
if (pPage->num == pModel->capacity) {
if (tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType) != TSDB_CODE_SUCCESS) {
return -1;
}
} else {
pPage->numOfElems = numOfWriteElems;
pPage->num = numOfWriteElems;
}
remain -= numOfWriteElems;
@ -470,7 +493,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
tscTrace("%p waiting for delete procedure, status: %d", pSql, status);
}
taosDestoryInterpoInfo(&pLocalReducer->interpolationInfo);
taosDestoryFillInfo(pLocalReducer->pFillInfo);
if (pLocalReducer->pCtx != NULL) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
@ -503,8 +526,6 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
tfree(pLocalReducer->pLoserTree);
}
tfree(pLocalReducer->pBufForInterpo);
tfree(pLocalReducer->pFinalRes);
tfree(pLocalReducer->discardData);
@ -617,7 +638,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
(*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * 1);
(*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pSql->numOfSubs);
if (*pMemBuffer == NULL) {
tscError("%p failed to allocate memory", pSql);
pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
@ -740,7 +761,7 @@ int32_t loadNewDataFromDiskFor(SLocalReducer *pLocalReducer, SLocalDataSource *p
#if defined(_DEBUG_VIEW)
printf("new page load to buffer\n");
tColModelDisplay(pOneInterDataSrc->pMemBuffer->pColumnModel, pOneInterDataSrc->filePage.data,
pOneInterDataSrc->filePage.numOfElems, pOneInterDataSrc->pMemBuffer->pColumnModel->capacity);
pOneInterDataSrc->filePage.num, pOneInterDataSrc->pMemBuffer->pColumnModel->capacity);
#endif
*needAdjustLoserTree = true;
} else {
@ -761,7 +782,7 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource *
* since it's last record in buffer has been chosen to be processed, as the winner of loser-tree
*/
bool needToAdjust = true;
if (pOneInterDataSrc->filePage.numOfElems <= pOneInterDataSrc->rowIdx) {
if (pOneInterDataSrc->filePage.num <= pOneInterDataSrc->rowIdx) {
loadNewDataFromDiskFor(pLocalReducer, pOneInterDataSrc, &needToAdjust);
}
@ -787,23 +808,22 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource *
}
}
void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQueryInfo,
SInterpolationInfo *pInterpoInfo) {
void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQueryInfo, SFillInfo *pFillInfo) {
// discard following dataset in the same group and reset the interpolation information
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
int16_t prec = tinfo.precision;
int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey;
int64_t revisedSTime =
taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec);
taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
pLocalReducer->rowSize);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
if (pFillInfo != NULL) {
int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey;
int64_t revisedSTime =
taosGetIntervalStartTimestamp(stime, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit, tinfo.precision);
taosResetFillInfo(pFillInfo, revisedSTime);
}
pLocalReducer->discard = true;
pLocalReducer->discardData->numOfElems = 0;
pLocalReducer->discardData->num = 0;
SColumnModel *pModel = pLocalReducer->pDesc->pColumnModel;
tColModelAppend(pModel, pLocalReducer->discardData, pLocalReducer->prevRowOfInput, 0, 1, 1);
@ -856,6 +876,7 @@ static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRe
static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneOutput) {
SSqlCmd * pCmd = &pSql->cmd;
SSqlRes * pRes = &pSql->res;
tFilePage * pFinalDataPage = pLocalReducer->pResultBuf;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
@ -868,91 +889,65 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
assert(pRes->pLocalReducer == NULL);
}
if (pQueryInfo->intervalTime == 0 || pQueryInfo->interpoType == TSDB_INTERPO_NONE) {
// no interval query, no interpolation
if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) {
// no interval query, no fill operation
pRes->data = pLocalReducer->pFinalRes;
pRes->numOfRows = pFinalDataPage->numOfElems;
pRes->numOfTotalInCurrentClause += pRes->numOfRows;
pRes->numOfRows = pFinalDataPage->num;
pRes->numOfClauseTotal += pRes->numOfRows;
if (pQueryInfo->limit.offset > 0) {
if (pQueryInfo->limit.offset < pRes->numOfRows) {
int32_t prevSize = pFinalDataPage->numOfElems;
int32_t prevSize = pFinalDataPage->num;
tColModelErase(pLocalReducer->resColModel, pFinalDataPage, prevSize, 0, pQueryInfo->limit.offset - 1);
/* remove the hole in column model */
tColModelCompact(pLocalReducer->resColModel, pFinalDataPage, prevSize);
pRes->numOfRows -= pQueryInfo->limit.offset;
pRes->numOfTotalInCurrentClause -= pQueryInfo->limit.offset;
pRes->numOfClauseTotal -= pQueryInfo->limit.offset;
pQueryInfo->limit.offset = 0;
} else {
pQueryInfo->limit.offset -= pRes->numOfRows;
pRes->numOfRows = 0;
pRes->numOfTotalInCurrentClause = 0;
pRes->numOfClauseTotal = 0;
}
}
if (pQueryInfo->limit.limit >= 0 && pRes->numOfTotalInCurrentClause > pQueryInfo->limit.limit) {
if (pQueryInfo->limit.limit >= 0 && pRes->numOfClauseTotal > pQueryInfo->limit.limit) {
/* impose the limitation of output rows on the final result */
int32_t prevSize = pFinalDataPage->numOfElems;
int32_t overFlow = pRes->numOfTotalInCurrentClause - pQueryInfo->limit.limit;
int32_t prevSize = pFinalDataPage->num;
int32_t overflow = pRes->numOfClauseTotal - pQueryInfo->limit.limit;
assert(overflow < pRes->numOfRows);
assert(overFlow < pRes->numOfRows);
pRes->numOfTotalInCurrentClause = pQueryInfo->limit.limit;
pRes->numOfRows -= overFlow;
pFinalDataPage->numOfElems -= overFlow;
pRes->numOfClauseTotal = pQueryInfo->limit.limit;
pRes->numOfRows -= overflow;
pFinalDataPage->num -= overflow;
tColModelCompact(pLocalReducer->resColModel, pFinalDataPage, prevSize);
/* set remain data to be discarded, and reset the interpolation information */
savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, &pLocalReducer->interpolationInfo);
savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pLocalReducer->pFillInfo);
}
int32_t rowSize = tscGetResRowLength(pQueryInfo->exprList);
memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * rowSize);
pFinalDataPage->numOfElems = 0;
pFinalDataPage->num = 0;
return;
}
int64_t *pPrimaryKeys = (int64_t *)pLocalReducer->pBufForInterpo;
SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo;
int64_t actualETime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey;
SFillInfo *pFillInfo = pLocalReducer->pFillInfo;
int64_t actualETime = MAX(pQueryInfo->window.skey, pQueryInfo->window.ekey);
tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput);
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalReducer->resColModel->capacity);
}
char ** srcData = (char **)malloc((POINTER_BYTES + sizeof(int32_t)) * pQueryInfo->fieldsInfo.numOfOutput);
int32_t *functions = (int32_t *)((char *)srcData + pQueryInfo->fieldsInfo.numOfOutput * sizeof(void *));
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
srcData[i] =
pLocalReducer->pBufForInterpo + tscFieldInfoGetOffset(pQueryInfo, i) * pInterpoInfo->numOfRawDataInRows;
functions[i] = tscSqlExprGet(pQueryInfo, i)->functionId;
}
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
int8_t precision = tinfo.precision;
while (1) {
int32_t remains = taosNumOfRemainPoints(pInterpoInfo);
TSKEY etime = taosGetRevisedEndKey(actualETime, pQueryInfo->order.order, pQueryInfo->intervalTime,
pQueryInfo->slidingTimeUnit, precision);
int32_t nrows = taosGetNumOfResultWithInterpo(pInterpoInfo, pPrimaryKeys, remains, pQueryInfo->intervalTime, etime,
pLocalReducer->resColModel->capacity);
int32_t newRows = taosDoInterpoResult(pInterpoInfo, pQueryInfo->interpoType, pResPages, remains, nrows,
pQueryInfo->intervalTime, pPrimaryKeys, pLocalReducer->resColModel, srcData,
pQueryInfo->defaultVal, functions, pLocalReducer->resColModel->capacity);
assert(newRows <= nrows);
int64_t newRows = -1;
taosGenerateDataBlock(pFillInfo, pResPages, &newRows, pLocalReducer->resColModel->capacity);
if (pQueryInfo->limit.offset < newRows) {
newRows -= pQueryInfo->limit.offset;
@ -967,7 +962,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
pRes->data = pLocalReducer->pFinalRes;
pRes->numOfRows = newRows;
pRes->numOfTotalInCurrentClause += newRows;
pRes->numOfClauseTotal += newRows;
pQueryInfo->limit.offset = 0;
break;
@ -975,16 +970,15 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
pQueryInfo->limit.offset -= newRows;
pRes->numOfRows = 0;
int32_t rpoints = taosNumOfRemainPoints(pInterpoInfo);
int32_t rpoints = taosNumOfRemainRows(pFillInfo);
if (rpoints <= 0) {
if (!doneOutput) {
/* reduce procedure is not completed, but current results for interpolation are exhausted */
if (!doneOutput) { // reduce procedure has not completed yet, but current results for fill are exhausted
break;
}
/* all output for current group are completed */
int32_t totalRemainRows =
taosGetNumOfResWithoutLimit(pInterpoInfo, pPrimaryKeys, rpoints, pQueryInfo->intervalTime, actualETime);
taosGetNumOfResultWithFill(pFillInfo, rpoints, pFillInfo->slidingTime, actualETime);
if (totalRemainRows <= 0) {
break;
}
@ -993,17 +987,17 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
}
if (pRes->numOfRows > 0) {
if (pQueryInfo->limit.limit >= 0 && pRes->numOfTotalInCurrentClause > pQueryInfo->limit.limit) {
int32_t overFlow = pRes->numOfTotalInCurrentClause - pQueryInfo->limit.limit;
pRes->numOfRows -= overFlow;
if (pQueryInfo->limit.limit >= 0 && pRes->numOfClauseTotal > pQueryInfo->limit.limit) {
int32_t overflow = pRes->numOfClauseTotal - pQueryInfo->limit.limit;
pRes->numOfRows -= overflow;
assert(pRes->numOfRows >= 0);
pRes->numOfTotalInCurrentClause = pQueryInfo->limit.limit;
pFinalDataPage->numOfElems -= overFlow;
pRes->numOfClauseTotal = pQueryInfo->limit.limit;
pFinalDataPage->num -= overflow;
/* set remain data to be discarded, and reset the interpolation information */
savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pInterpoInfo);
savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pFillInfo);
}
if (pQueryInfo->order.order == TSDB_ORDER_ASC) {
@ -1017,18 +1011,17 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
}
}
pFinalDataPage->numOfElems = 0;
pFinalDataPage->num = 0;
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
tfree(pResPages[i]);
}
tfree(pResPages);
free(srcData);
}
static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) {
SColumnModel *pColumnModel = pLocalReducer->pDesc->pColumnModel;
assert(pColumnModel->capacity == 1 && tmpBuffer->numOfElems == 1);
assert(pColumnModel->capacity == 1 && tmpBuffer->num == 1);
// copy to previous temp buffer
for (int32_t i = 0; i < pColumnModel->numOfCols; ++i) {
@ -1038,7 +1031,7 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer)
memcpy(pLocalReducer->prevRowOfInput + offset, tmpBuffer->data + offset, pSchema->bytes);
}
tmpBuffer->numOfElems = 0;
tmpBuffer->num = 0;
pLocalReducer->hasPrevRow = true;
}
@ -1168,7 +1161,7 @@ int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {
pLocalReducer->hasPrevRow = false;
int32_t numOfRes = (int32_t)getNumOfResultLocal(pQueryInfo, pLocalReducer->pCtx);
pLocalReducer->pResultBuf->numOfElems += numOfRes;
pLocalReducer->pResultBuf->num += numOfRes;
fillMultiRowsOfTagsVal(pQueryInfo, numOfRes, pLocalReducer);
return numOfRes;
@ -1221,7 +1214,7 @@ static bool saveGroupResultInfo(SSqlObj *pSql) {
// pRes->pGroupRec = realloc(pRes->pGroupRec, pRes->numOfGroups*sizeof(SResRec));
// pRes->pGroupRec[pRes->numOfGroups-1].numOfRows = pRes->numOfRows;
// pRes->pGroupRec[pRes->numOfGroups-1].numOfTotalInCurrentClause = pRes->numOfTotalInCurrentClause;
// pRes->pGroupRec[pRes->numOfGroups-1].numOfClauseTotal = pRes->numOfClauseTotal;
return false;
}
@ -1244,37 +1237,38 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no
pRes->code = TSDB_CODE_SUCCESS;
/*
* ignore the output of the current group since this group is skipped by user
* Ignore the output of the current group since this group is skipped by user
* We set the numOfRows to be 0 and discard the possible remain results.
*/
if (pQueryInfo->slimit.offset > 0) {
pRes->numOfRows = 0;
pQueryInfo->slimit.offset -= 1;
pLocalReducer->discard = !noMoreCurrentGroupRes;
return false;
}
tColModelCompact(pModel, pResBuf, pModel->capacity);
memcpy(pLocalReducer->pBufForInterpo, pResBuf->data, pLocalReducer->nResultBufSize);
#ifdef _DEBUG_VIEW
printf("final result before interpo:\n");
tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->numOfElems, pResBuf->numOfElems);
assert(0);
// tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num);
#endif
SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo;
int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
int16_t offset = getColumnModelOffset(pModel, startIndex + i);
SSchema *pSchema = getColumnModelSchema(pModel, startIndex + i);
memcpy(pInterpoInfo->pTags[i], pLocalReducer->pBufForInterpo + offset * pResBuf->numOfElems, pSchema->bytes);
SFillInfo* pFillInfo = pLocalReducer->pFillInfo;
if (pFillInfo != NULL) {
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
TSKEY ekey = taosGetRevisedEndKey(pQueryInfo->window.ekey, pFillInfo->order, pFillInfo->slidingTime,
pQueryInfo->slidingTimeUnit, tinfo.precision);
taosFillSetStartInfo(pFillInfo, pResBuf->num, ekey);
taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf);
}
taosInterpoSetStartInfo(&pLocalReducer->interpolationInfo, pResBuf->numOfElems, pQueryInfo->interpoType);
doInterpolateResult(pSql, pLocalReducer, noMoreCurrentGroupRes);
return true;
}
@ -1290,7 +1284,7 @@ void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { //
static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer *pLocalReducer) {
// In handling data in other groups, we need to reset the interpolation information for a new group data
pRes->numOfRows = 0;
pRes->numOfTotalInCurrentClause = 0;
pRes->numOfClauseTotal = 0;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
@ -1302,13 +1296,13 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer
int8_t precision = tinfo.precision;
// for group result interpolation, do not return if not data is generated
if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) {
int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey;
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
TSKEY skey = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey);
int64_t newTime =
taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision);
taosInitInterpoInfo(&pLocalReducer->interpolationInfo, pQueryInfo->order.order, newTime,
pQueryInfo->groupbyExpr.numOfGroupCols, pLocalReducer->rowSize);
taosGetIntervalStartTimestamp(skey, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision);
// taosResetFillInfo(pLocalReducer->pFillInfo, pQueryInfo->order.order, newTime,
// pQueryInfo->groupbyExpr.numOfGroupCols, 4096, 0, NULL, pLocalReducer->rowSize);
taosResetFillInfo(pLocalReducer->pFillInfo, newTime);
}
}
@ -1320,26 +1314,26 @@ static bool doInterpolationForCurrentGroup(SSqlObj *pSql) {
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
SLocalReducer * pLocalReducer = pRes->pLocalReducer;
SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
SLocalReducer *pLocalReducer = pRes->pLocalReducer;
SFillInfo *pFillInfo = pLocalReducer->pFillInfo;
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
int8_t p = tinfo.precision;
if (taosHasRemainsDataForInterpolation(pInterpoInfo)) {
assert(pQueryInfo->interpoType != TSDB_INTERPO_NONE);
if (pFillInfo != NULL && taosNumOfRemainRows(pFillInfo) > 0) {
assert(pQueryInfo->fillType != TSDB_FILL_NONE);
tFilePage *pFinalDataBuf = pLocalReducer->pResultBuf;
int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pInterpoInfo->numOfRawDataInRows - 1));
int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1));
int32_t remain = taosNumOfRemainPoints(pInterpoInfo);
TSKEY ekey =
taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, p);
int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY *)pLocalReducer->pBufForInterpo, remain,
pQueryInfo->intervalTime, ekey, pLocalReducer->resColModel->capacity);
int32_t remain = taosNumOfRemainRows(pFillInfo);
TSKEY ekey = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit, p);
// the first column must be the timestamp column
int32_t rows = taosGetNumOfResultWithFill(pFillInfo, remain, ekey, pLocalReducer->resColModel->capacity);
if (rows > 0) { // do interpo
doInterpolateResult(pSql, pLocalReducer, false);
}
@ -1355,7 +1349,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
SSqlRes *pRes = &pSql->res;
SLocalReducer * pLocalReducer = pRes->pLocalReducer;
SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo;
SFillInfo *pFillInfo = pLocalReducer->pFillInfo;
bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow;
@ -1363,18 +1357,16 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
int8_t precision = tinfo.precision;
if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL ||
prevGroupCompleted) {
// if interpoType == TSDB_INTERPO_NONE, return directly
if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) {
// if fillType == TSDB_FILL_NONE, return directly
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey;
etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime,
pQueryInfo->slidingTimeUnit, precision);
int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, NULL, 0, pQueryInfo->intervalTime, etime,
pLocalReducer->resColModel->capacity);
pQueryInfo->slidingTimeUnit, tinfo.precision);
int32_t rows = taosGetNumOfResultWithFill(pFillInfo, 0, etime, pLocalReducer->resColModel->capacity);
if (rows > 0) { // do interpo
doInterpolateResult(pSql, pLocalReducer, true);
}
@ -1474,7 +1466,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
printf("chosen data in pTree[0] = %d\n", pTree->pNode[0].index);
#endif
assert((pTree->pNode[0].index < pLocalReducer->numOfBuffer) && (pTree->pNode[0].index >= 0) &&
tmpBuffer->numOfElems == 0);
tmpBuffer->num == 0);
// chosen from loser tree
SLocalDataSource *pOneDataSrc = pLocalReducer->pLocalDataSrc[pTree->pNode[0].index];
@ -1487,7 +1479,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
SSrcColumnInfo colInfo[256] = {0};
tscGetSrcColumnInfo(colInfo, pQueryInfo);
tColModelDisplayEx(pModel, tmpBuffer->data, tmpBuffer->numOfElems, pModel->capacity, colInfo);
tColModelDisplayEx(pModel, tmpBuffer->data, tmpBuffer->num, pModel->capacity, colInfo);
#endif
if (pLocalReducer->discard) {
@ -1495,7 +1487,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
/* current record belongs to the same group of previous record, need to discard it */
if (isSameGroup(pCmd, pLocalReducer, pLocalReducer->discardData->data, tmpBuffer)) {
tmpBuffer->numOfElems = 0;
tmpBuffer->num = 0;
pOneDataSrc->rowIdx += 1;
adjustLoserTreeFromNewData(pLocalReducer, pOneDataSrc, pTree);
@ -1509,7 +1501,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
continue;
} else {
pLocalReducer->discard = false;
pLocalReducer->discardData->numOfElems = 0;
pLocalReducer->discardData->num = 0;
if (saveGroupResultInfo(pSql)) {
pLocalReducer->status = TSC_LOCALREDUCE_READY;
@ -1538,17 +1530,17 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
tFilePage *pResBuf = pLocalReducer->pResultBuf;
/*
* if the previous group does NOT generate any result (pResBuf->numOfElems == 0),
* if the previous group does NOT generate any result (pResBuf->num == 0),
* continue to process results instead of return results.
*/
if ((!sameGroup && pResBuf->numOfElems > 0) || (pResBuf->numOfElems == pLocalReducer->resColModel->capacity)) {
if ((!sameGroup && pResBuf->num > 0) || (pResBuf->num == pLocalReducer->resColModel->capacity)) {
// does not belong to the same group
bool notSkipped = doGenerateFinalResults(pSql, pLocalReducer, !sameGroup);
// this row needs to discard, since it belongs to the group of previous
if (pLocalReducer->discard && sameGroup) {
pLocalReducer->hasUnprocessedRow = false;
tmpBuffer->numOfElems = 0;
tmpBuffer->num = 0;
} else {
// current row does not belongs to the previous group, so it is not be handled yet.
pLocalReducer->hasUnprocessedRow = true;
@ -1611,7 +1603,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
finalizeRes(pQueryInfo, pLocalReducer);
}
if (pLocalReducer->pResultBuf->numOfElems) {
if (pLocalReducer->pResultBuf->num) {
doGenerateFinalResults(pSql, pLocalReducer, true);
}
@ -1641,6 +1633,6 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen)
size_t allocSize = numOfRes * rowLen + sizeof(tFilePage) + 1;
pRes->pLocalReducer->pResultBuf = (tFilePage *)calloc(1, allocSize);
pRes->pLocalReducer->pResultBuf->numOfElems = numOfRes;
pRes->pLocalReducer->pResultBuf->num = numOfRes;
pRes->data = pRes->pLocalReducer->pResultBuf->data;
}

26
src/client/src/tscServer.c
View File

@ -571,7 +571,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid, dfltKey));
pQueryMsg->numOfTables = htonl(1); // set the number of tables
pMsg += sizeof(STableIdInfo);
} else {
int32_t index = pTableMetaInfo->vgroupIndex;
@ -601,8 +600,8 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
}
}
tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
pTableMeta->uid);
tscTrace("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
pTableMeta->sid, pTableMeta->uid);
return pMsg;
}
@ -652,7 +651,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->order = htons(pQueryInfo->order.order);
pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId);
pQueryMsg->interpoType = htons(pQueryInfo->interpoType);
pQueryMsg->fillType = htons(pQueryInfo->fillType);
pQueryMsg->limit = htobe64(pQueryInfo->limit.limit);
pQueryMsg->offset = htobe64(pQueryInfo->limit.offset);
pQueryMsg->numOfCols = htons(taosArrayGetSize(pQueryInfo->colList));
@ -780,7 +779,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
}
if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) {
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
*((int64_t *)pMsg) = htobe64(pQueryInfo->defaultVal[i]);
pMsg += sizeof(pQueryInfo->defaultVal[0]);
@ -1213,8 +1212,13 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int8_t type = pInfo->pCreateTableInfo->type;
if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value
memcpy(pMsg, &pInfo->pCreateTableInfo->usingInfo.tagdata, sizeof(STagData));
pMsg += sizeof(STagData);
STagData* pTag = &pInfo->pCreateTableInfo->usingInfo.tagdata;
*(int32_t*)pMsg = htonl(pTag->dataLen);
pMsg += sizeof(int32_t);
memcpy(pMsg, pTag->name, sizeof(pTag->name));
pMsg += sizeof(pTag->name);
memcpy(pMsg, pTag->data, pTag->dataLen);
pMsg += pTag->dataLen;
} else { // create (super) table
pSchema = (SSchema *)pCreateTableMsg->schema;
@ -1281,9 +1285,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name);
pAlterTableMsg->type = htons(pAlterInfo->type);
pAlterTableMsg->numOfCols = tscNumOfFields(pQueryInfo);
memcpy(pAlterTableMsg->tagVal, pAlterInfo->tagData.data, TSDB_MAX_TAGS_LEN);
pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo));
SSchema *pSchema = pAlterTableMsg->schema;
for (int i = 0; i < pAlterTableMsg->numOfCols; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
@ -1295,6 +1297,9 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
pMsg = (char *)pSchema;
pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen);
memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen);
pMsg += pAlterInfo->tagData.dataLen;
msgLen = pMsg - (char*)pAlterTableMsg;
pCmd->payloadLen = msgLen;
@ -1863,6 +1868,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
}
free(pTableMeta);
tscTrace("%p recv table meta: %"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->uid, pTableMeta->sid, pTableMetaInfo->name);
return TSDB_CODE_SUCCESS;
}

12
src/client/src/tscSql.c
View File

@ -228,7 +228,7 @@ int taos_query_imp(STscObj *pObj, SSqlObj *pSql) {
pRes->numOfRows = 1;
pRes->numOfTotal = 0;
pRes->numOfTotalInCurrentClause = 0;
pRes->numOfClauseTotal = 0;
pCmd->curSql = NULL;
if (NULL != pCmd->pTableList) {
@ -407,7 +407,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
// secondary merge has handle this situation
if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE) {
pRes->numOfTotalInCurrentClause += pRes->numOfRows;
pRes->numOfClauseTotal += pRes->numOfRows;
}
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
@ -490,8 +490,8 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
pSql->cmd.command = pQueryInfo->command;
pCmd->clauseIndex++;
pRes->numOfTotal += pRes->numOfTotalInCurrentClause;
pRes->numOfTotalInCurrentClause = 0;
pRes->numOfTotal += pRes->numOfClauseTotal;
pRes->numOfClauseTotal = 0;
pRes->rspType = 0;
pSql->numOfSubs = 0;
@ -790,7 +790,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
pRes->numOfRows = 1;
pRes->numOfTotal = 0;
pRes->numOfTotalInCurrentClause = 0;
pRes->numOfClauseTotal = 0;
tscTrace("%p Valid SQL: %s pObj:%p", pSql, sql, pObj);
@ -921,7 +921,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
SSqlRes *pRes = &pSql->res;
pRes->numOfTotal = 0; // the number of getting table meta from server
pRes->numOfTotalInCurrentClause = 0;
pRes->numOfClauseTotal = 0;
pRes->code = 0;

18
src/client/src/tscStream.c
View File

@ -165,7 +165,7 @@ static void tscSetTimestampForRes(SSqlStream *pStream, SSqlObj *pSql) {
if (timestamp != actualTimestamp) {
// reset the timestamp of each agg point by using start time of each interval
*((int64_t *)pRes->data) = actualTimestamp;
tscWarn("%p stream:%p, timestamp of points is:%" PRId64 ", reset to %" PRId64 "", pSql, pStream, timestamp, actualTimestamp);
tscWarn("%p stream:%p, timestamp of points is:%" PRId64 ", reset to %" PRId64, pSql, pStream, timestamp, actualTimestamp);
}
}
@ -208,7 +208,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
if (pStream->numOfRes == 0) {
if (pQueryInfo->interpoType == TSDB_INTERPO_SET_VALUE || pQueryInfo->interpoType == TSDB_INTERPO_NULL) {
if (pQueryInfo->fillType == TSDB_FILL_SET_VALUE || pQueryInfo->fillType == TSDB_FILL_NULL) {
SSqlRes *pRes = &pSql->res;
/* failed to retrieve any result in this retrieve */
@ -287,10 +287,10 @@ static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer)
return;
}
tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64 "", pStream->pSql, pStream,
tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql, pStream,
now + timer, timer, delay, pStream->stime, etime);
} else {
tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64 "", pStream->pSql, pStream,
tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql, pStream,
pStream->stime, timer, delay, pStream->stime - pStream->interval, pStream->stime - 1);
}
@ -380,7 +380,7 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
if (pQueryInfo->intervalTime < minIntervalTime) {
tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64 "", pSql, pStream,
tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64, pSql, pStream,
pQueryInfo->intervalTime, minIntervalTime);
pQueryInfo->intervalTime = minIntervalTime;
}
@ -397,14 +397,14 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
if (pQueryInfo->slidingTime == -1) {
pQueryInfo->slidingTime = pQueryInfo->intervalTime;
} else if (pQueryInfo->slidingTime < minSlidingTime) {
tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64 "", pSql, pStream,
tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64, pSql, pStream,
pQueryInfo->slidingTime, minSlidingTime);
pQueryInfo->slidingTime = minSlidingTime;
}
if (pQueryInfo->slidingTime > pQueryInfo->intervalTime) {
tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64 "", pSql, pStream,
tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64, pSql, pStream,
pQueryInfo->slidingTime, pQueryInfo->intervalTime);
pQueryInfo->slidingTime = pQueryInfo->intervalTime;
@ -433,11 +433,11 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in
} else { // timewindow based aggregation stream
if (stime == 0) { // no data in meter till now
stime = ((int64_t)taosGetTimestamp(pStream->precision) / pStream->interval) * pStream->interval;
tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64 "", pSql, pStream, stime);
tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64, pSql, pStream, stime);
} else {
int64_t newStime = (stime / pStream->interval) * pStream->interval;
if (newStime != stime) {
tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64 "", pSql, pStream, stime, newStime);
tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql, pStream, stime, newStime);
stime = newStime;
}
}

20
src/client/src/tscSubquery.c
View File

@ -771,7 +771,7 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
}
SSqlRes* pRes1 = &pParentSql->pSubs[i]->res;
pRes1->numOfTotalInCurrentClause += pRes1->numOfRows;
pRes1->numOfClauseTotal += pRes1->numOfRows;
}
// data has retrieved to client, build the join results
@ -1390,7 +1390,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]);
// clear local saved number of results
trsupport->localBuffer->numOfElems = 0;
trsupport->localBuffer->num = 0;
pthread_mutex_unlock(&trsupport->queryMutex);
tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSqlObj, pSql,
@ -1457,7 +1457,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
// data in from current vnode is stored in cache and disk
uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num;
tscTrace("%p sub:%p all data retrieved from ip:%u,vgId:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql,
pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId,
numOfRowsFromSubquery, idx);
@ -1465,11 +1465,11 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
#ifdef _DEBUG_VIEW
printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems);
printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->num);
SSrcColumnInfo colInfo[256] = {0};
tscGetSrcColumnInfo(colInfo, pQueryInfo);
tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems,
trsupport->localBuffer->numOfElems, colInfo);
tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->num,
trsupport->localBuffer->num, colInfo);
#endif
if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) {
@ -1834,7 +1834,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
pRes->tsrow[i] = pRes1->tsrow[pIndex->columnIndex];
}
pRes->numOfTotalInCurrentClause++;
pRes->numOfClauseTotal++;
break;
} else { // continue retrieve data from vnode
if (!tscHashRemainDataInSubqueryResultSet(pSql)) {
@ -1879,9 +1879,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) {
SSqlRes *pRes = &pSql->res;
if (pRes->tsrow[columnIndex] != NULL && isNull(pRes->tsrow[columnIndex], pField->type)) {
pRes->tsrow[columnIndex] = NULL;
} else if (pField->type == TSDB_DATA_TYPE_NCHAR) {
if (pRes->tsrow[columnIndex] != NULL && pField->type == TSDB_DATA_TYPE_NCHAR) {
// convert unicode to native code in a temporary buffer extra one byte for terminated symbol
if (pRes->buffer[columnIndex] == NULL) {
pRes->buffer[columnIndex] = malloc(pField->bytes + TSDB_NCHAR_SIZE);
@ -1893,7 +1891,7 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF
if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes - VARSTR_HEADER_SIZE, pRes->buffer[columnIndex])) {
pRes->tsrow[columnIndex] = pRes->buffer[columnIndex];
} else {
tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow);
tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow[columnIndex]);
pRes->tsrow[columnIndex] = NULL;
}
}

28
src/client/src/tscUtil.c
View File

@ -280,7 +280,7 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) {
return;
}
pQueryInfo->interpoType = TSDB_INTERPO_NONE;
pQueryInfo->fillType = TSDB_FILL_NONE;
tfree(pQueryInfo->defaultVal);
}
@ -1779,7 +1779,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond);
if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) {
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
pNewQueryInfo->defaultVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
memcpy(pNewQueryInfo->defaultVal, pQueryInfo->defaultVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
}
@ -1989,7 +1989,7 @@ int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* s
bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) {
assert(pQueryInfo != NULL && pQueryInfo->clauseLimit != 0);
return (pQueryInfo->clauseLimit > 0 && pRes->numOfTotalInCurrentClause >= pQueryInfo->clauseLimit);
return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit);
}
char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; }
@ -2037,7 +2037,7 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
while (++pTableMetaInfo->vgroupIndex < totalVgroups) {
tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql,
pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfTotalInCurrentClause);
pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfClauseTotal);
/*
* update the limit and offset value for the query on the next vnode,
@ -2045,11 +2045,11 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
*
* NOTE:
* if the pRes->offset is larger than 0, the start returned position has not reached yet.
* Therefore, the pRes->numOfRows, as well as pRes->numOfTotalInCurrentClause, must be 0.
* Therefore, the pRes->numOfRows, as well as pRes->numOfClauseTotal, must be 0.
* The pRes->offset value will be updated by virtual node, during query execution.
*/
if (pQueryInfo->clauseLimit >= 0) {
pQueryInfo->limit.limit = pQueryInfo->clauseLimit - pRes->numOfTotalInCurrentClause;
pQueryInfo->limit.limit = pQueryInfo->clauseLimit - pRes->numOfClauseTotal;
}
pQueryInfo->limit.offset = pRes->offset;
@ -2092,7 +2092,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) {
pSql->cmd.command = pQueryInfo->command;
//backup the total number of result first
int64_t num = pRes->numOfTotal + pRes->numOfTotalInCurrentClause;
int64_t num = pRes->numOfTotal + pRes->numOfClauseTotal;
tscFreeSqlResult(pSql);
pRes->numOfTotal = num;
@ -2126,16 +2126,26 @@ void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t column
int32_t realLen = varDataLen(pData);
assert(realLen <= bytes - VARSTR_HEADER_SIZE);
if (isNull(pData, type)) {
pRes->tsrow[columnIndex] = NULL;
} else {
pRes->tsrow[columnIndex] = pData + VARSTR_HEADER_SIZE;
}
if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor
*(char*) (pData + realLen + VARSTR_HEADER_SIZE) = 0;
}
pRes->tsrow[columnIndex] = pData + VARSTR_HEADER_SIZE;
pRes->length[columnIndex] = realLen;
} else {
assert(bytes == tDataTypeDesc[type].nSize);
pRes->tsrow[columnIndex] = pData;
if (isNull(pData, type)) {
pRes->tsrow[columnIndex] = NULL;
} else {
pRes->tsrow[columnIndex] = pData;
}
pRes->length[columnIndex] = bytes;
}
}

21
src/common/inc/tdataformat.h
View File

@ -106,9 +106,28 @@ typedef void *SDataRow;
SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
void tdFreeDataRow(SDataRow row);
void tdInitDataRow(SDataRow row, STSchema *pSchema);
int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset);
SDataRow tdDataRowDup(SDataRow row);
static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) {
ASSERT(value != NULL);
int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
*(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
memcpy(ptr, value, varDataTLen(value));
dataRowLen(row) += varDataTLen(value);
break;
default:
memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
break;
}
return 0;
}
// NOTE: offset here including the header size
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
switch (type) {

2
src/common/inc/tglobal.h
View File

@ -103,6 +103,7 @@ extern int32_t tsOfflineThreshold;
extern int32_t tsMgmtEqualVnodeNum;
extern int32_t tsEnableHttpModule;
extern int32_t tsEnableMqttModule;
extern int32_t tsEnableMonitorModule;
extern int32_t tsRestRowLimit;
@ -147,6 +148,7 @@ extern int32_t jniDebugFlag;
extern int32_t tmrDebugFlag;
extern int32_t sdbDebugFlag;
extern int32_t httpDebugFlag;
extern int32_t mqttDebugFlag;
extern int32_t monitorDebugFlag;
extern int32_t uDebugFlag;
extern int32_t rpcDebugFlag;

38
src/common/src/tdataformat.c
View File

@ -158,32 +158,6 @@ void tdFreeDataRow(SDataRow row) {
if (row) free(row);
}
/**
* Append a column value to the data row
* @param type: column type
* @param bytes: column bytes
* @param offset: offset in the data row tuple, not including the data row header
*/
int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) {
ASSERT(value != NULL);
int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
char * ptr = POINTER_SHIFT(row, dataRowLen(row));
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
*(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
memcpy(ptr, value, varDataTLen(value));
dataRowLen(row) += varDataTLen(value);
break;
default:
memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
break;
}
return 0;
}
SDataRow tdDataRowDup(SDataRow row) {
SDataRow trow = malloc(dataRowLen(row));
if (trow == NULL) return NULL;
@ -453,27 +427,25 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCol
TSKEY key1 = (*iter1 >= src1->numOfPoints) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
TSKEY key2 = (*iter2 >= src2->numOfPoints) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
if (key1 < key2) {
if (key1 <= key2) {
for (int i = 0; i < src1->numOfCols; i++) {
ASSERT(target->cols[i].type == src1->cols[i].type);
dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfPoints,
dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfPoints,
target->maxPoints);
}
target->numOfPoints++;
(*iter1)++;
} else if (key1 > key2) {
if (key1 == key2) (*iter2)++;
} else {
for (int i = 0; i < src2->numOfCols; i++) {
ASSERT(target->cols[i].type == src2->cols[i].type);
dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfPoints,
dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfPoints,
target->maxPoints);
}
target->numOfPoints++;
(*iter2)++;
} else {
// TODO: deal with duplicate keys
ASSERT(false);
}
}
}

40
src/common/src/tglobal.c
View File

@ -107,7 +107,6 @@ int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM;
int16_t tsAffectedRowsMod = 0;
int32_t tsNumOfMPeers = 3;
int32_t tsMaxShellConns = 2000;
int32_t tsMaxTables = 100000;
char tsDefaultDB[TSDB_DB_NAME_LEN] = {0};
char tsDefaultUser[64] = "root";
@ -121,6 +120,7 @@ int32_t tsOfflineThreshold = 86400*100; // seconds 10days
int32_t tsMgmtEqualVnodeNum = 4;
int32_t tsEnableHttpModule = 1;
int32_t tsEnableMqttModule = 0; // not finished yet, not started it by default
int32_t tsEnableMonitorModule = 0;
int32_t tsRestRowLimit = 10240;
@ -135,12 +135,14 @@ int32_t cDebugFlag = 135;
int32_t jniDebugFlag = 131;
int32_t odbcDebugFlag = 131;
int32_t httpDebugFlag = 131;
int32_t mqttDebugFlag = 131;
int32_t monitorDebugFlag = 131;
int32_t qDebugFlag = 131;
int32_t rpcDebugFlag = 135;
int32_t uDebugFlag = 131;
int32_t debugFlag = 131;
int32_t sDebugFlag = 135;
int32_t tsdbDebugFlag = 131;
// the maximum number of results for projection query on super table that are returned from
// one virtual node, to order according to timestamp
@ -212,11 +214,13 @@ void taosSetAllDebugFlag() {
jniDebugFlag = debugFlag;
odbcDebugFlag = debugFlag;
httpDebugFlag = debugFlag;
mqttDebugFlag = debugFlag;
monitorDebugFlag = debugFlag;
rpcDebugFlag = debugFlag;
uDebugFlag = debugFlag;
sDebugFlag = debugFlag;
//qDebugFlag = debugFlag;
tsdbDebugFlag = debugFlag;
qDebugFlag = debugFlag;
}
uPrint("all debug flag are set to %d", debugFlag);
}
@ -889,6 +893,17 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "mqtt";
cfg.ptr = &tsEnableMqttModule;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "monitor";
cfg.ptr = &tsEnableMonitorModule;
cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -1111,6 +1126,17 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "mqttDebugFlag";
cfg.ptr = &mqttDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "monitorDebugFlag";
cfg.ptr = &monitorDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -1131,6 +1157,16 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "tsdbDebugFlag";
cfg.ptr = &tsdbDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "tscEnableRecordSql";
cfg.ptr = &tsTscEnableRecordSql;
cfg.valType = TAOS_CFG_VTYPE_INT32;

1
src/common/src/tname.c
View File

@ -29,6 +29,7 @@ void extractTableName(const char* tableId, char* name) {
size_t s2 = strcspn(&tableId[s1 + 1], &TS_PATH_DELIMITER[0]);
strncpy(name, &tableId[s1 + s2 + 2], TSDB_TABLE_NAME_LEN);
name[TSDB_TABLE_NAME_LEN] = 0;
}
char* extractDBName(const char* tableId, char* name) {

4
src/common/src/ttypes.c
View File

@ -304,7 +304,7 @@ tDataTypeDescriptor tDataTypeDesc[11] = {
{TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", tsCompressFloat, tsDecompressFloat, getStatics_f},
{TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", tsCompressDouble, tsDecompressDouble, getStatics_d},
{TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", tsCompressString, tsDecompressString, NULL},
{TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, NULL},
{TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
{TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", tsCompressString, tsDecompressString, NULL},
};
@ -402,7 +402,7 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) {
*(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_DOUBLE_NULL;
}
break;
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_NCHAR: // todo : without length?
for (int32_t i = 0; i < numOfElems; ++i) {
*(uint32_t *)(val + i * bytes) = TSDB_DATA_NCHAR_NULL;
}

84
src/connector/python/linux/python2/taos/cursor.py
View File

@ -1,5 +1,7 @@
from .cinterface import CTaosInterface
from .error import *
from .constants import FieldType
class TDengineCursor(object):
"""Database cursor which is used to manage the context of a fetch operation.
@ -19,7 +21,7 @@ class TDengineCursor(object):
if the cursor has not had an operation invoked via the .execute*() method yet.
.rowcount:This read-only attribute specifies the number of rows that the last
.execute*() produced (for DQL statements like SELECT) or affected
.execute*() produced (for DQL statements like SELECT) or affected
"""
def __init__(self, connection=None):
@ -32,6 +34,7 @@ class TDengineCursor(object):
self._block_rows = -1
self._block_iter = 0
self._affected_rows = 0
self._logfile = ""
if connection is not None:
self._connection = connection
@ -44,13 +47,14 @@ class TDengineCursor(object):
raise OperationalError("Invalid use of fetch iterator")
if self._block_rows <= self._block_iter:
block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields)
block, self._block_rows = CTaosInterface.fetchBlock(
self._result, self._fields)
if self._block_rows == 0:
raise StopIteration
self._block = list(map(tuple, zip(*block)))
self._block_iter = 0
data = self._block[self._block_iter]
data = self._block[self._block_iter]
self._block_iter += 1
return data
@ -80,12 +84,15 @@ class TDengineCursor(object):
"""
pass
def log(self, logfile):
self._logfile = logfile
def close(self):
"""Close the cursor.
"""
if self._connection is None:
return False
self._connection.clear_result_set()
self._reset_result()
self._connection = None
@ -101,24 +108,33 @@ class TDengineCursor(object):
if not self._connection:
# TODO : change the exception raised here
raise ProgrammingError("Cursor is not connected")
self._connection.clear_result_set()
self._reset_result()
stmt = operation
if params is not None:
pass
res = CTaosInterface.query(self._connection._conn, stmt)
if (self._logfile):
with open(self._logfile, "a") as logfile:
logfile.write("%s;\n" % operation)
if res == 0:
if CTaosInterface.fieldsCount(self._connection._conn) == 0:
self._affected_rows += CTaosInterface.affectedRows(self._connection._conn)
self._affected_rows += CTaosInterface.affectedRows(
self._connection._conn)
return CTaosInterface.affectedRows(self._connection._conn)
else:
self._result, self._fields = CTaosInterface.useResult(self._connection._conn)
self._result, self._fields = CTaosInterface.useResult(
self._connection._conn)
return self._handle_result()
else:
raise ProgrammingError(CTaosInterface.errStr(self._connection._conn))
raise ProgrammingError(
CTaosInterface.errStr(
self._connection._conn))
def executemany(self, operation, seq_of_parameters):
"""Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.
@ -130,6 +146,37 @@ class TDengineCursor(object):
"""
pass
def istype(self, col, dataType):
if (dataType.upper() == "BOOL"):
if (self._description[col][1] == FieldType.C_BOOL):
return True
if (dataType.upper() == "TINYINT"):
if (self._description[col][1] == FieldType.C_TINYINT):
return True
if (dataType.upper() == "INT"):
if (self._description[col][1] == FieldType.C_INT):
return True
if (dataType.upper() == "BIGINT"):
if (self._description[col][1] == FieldType.C_INT):
return True
if (dataType.upper() == "FLOAT"):
if (self._description[col][1] == FieldType.C_FLOAT):
return True
if (dataType.upper() == "DOUBLE"):
if (self._description[col][1] == FieldType.C_DOUBLE):
return True
if (dataType.upper() == "BINARY"):
if (self._description[col][1] == FieldType.C_BINARY):
return True
if (dataType.upper() == "TIMESTAMP"):
if (self._description[col][1] == FieldType.C_TIMESTAMP):
return True
if (dataType.upper() == "NCHAR"):
if (self._description[col][1] == FieldType.C_NCHAR):
return True
return False
def fetchmany(self):
pass
@ -138,22 +185,22 @@ class TDengineCursor(object):
"""
if self._result is None or self._fields is None:
raise OperationalError("Invalid use of fetchall")
buffer = [[] for i in range(len(self._fields))]
self._rowcount = 0
while True:
block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
if num_of_fields == 0: break
block, num_of_fields = CTaosInterface.fetchBlock(
self._result, self._fields)
if num_of_fields == 0:
break
self._rowcount += num_of_fields
for i in range(len(self._fields)):
buffer[i].extend(block[i])
self._connection.clear_result_set()
return list(map(tuple, zip(*buffer)))
def nextset(self):
"""
"""
@ -176,12 +223,13 @@ class TDengineCursor(object):
self._block_rows = -1
self._block_iter = 0
self._affected_rows = 0
def _handle_result(self):
"""Handle the return result from query.
"""
self._description = []
for ele in self._fields:
self._description.append((ele['name'], ele['type'], None, None, None, None, False))
self._description.append(
(ele['name'], ele['type'], None, None, None, None, False))
return self._result

82
src/connector/python/linux/python3/taos/cursor.py
View File

@ -1,8 +1,10 @@
from .cinterface import CTaosInterface
from .error import *
from .constants import FieldType
# querySeqNum = 0
class TDengineCursor(object):
"""Database cursor which is used to manage the context of a fetch operation.
@ -21,7 +23,7 @@ class TDengineCursor(object):
if the cursor has not had an operation invoked via the .execute*() method yet.
.rowcount:This read-only attribute specifies the number of rows that the last
.execute*() produced (for DQL statements like SELECT) or affected
.execute*() produced (for DQL statements like SELECT) or affected
"""
def __init__(self, connection=None):
@ -34,6 +36,7 @@ class TDengineCursor(object):
self._block_rows = -1
self._block_iter = 0
self._affected_rows = 0
self._logfile = ""
if connection is not None:
self._connection = connection
@ -46,13 +49,14 @@ class TDengineCursor(object):
raise OperationalError("Invalid use of fetch iterator")
if self._block_rows <= self._block_iter:
block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields)
block, self._block_rows = CTaosInterface.fetchBlock(
self._result, self._fields)
if self._block_rows == 0:
raise StopIteration
self._block = list(map(tuple, zip(*block)))
self._block_iter = 0
data = self._block[self._block_iter]
data = self._block[self._block_iter]
self._block_iter += 1
return data
@ -82,12 +86,15 @@ class TDengineCursor(object):
"""
pass
def log(self, logfile):
self._logfile = logfile
def close(self):
"""Close the cursor.
"""
if self._connection is None:
return False
self._connection.clear_result_set()
self._reset_result()
self._connection = None
@ -103,14 +110,13 @@ class TDengineCursor(object):
if not self._connection:
# TODO : change the exception raised here
raise ProgrammingError("Cursor is not connected")
self._connection.clear_result_set()
self._reset_result()
stmt = operation
if params is not None:
pass
# global querySeqNum
# querySeqNum += 1
@ -119,15 +125,23 @@ class TDengineCursor(object):
res = CTaosInterface.query(self._connection._conn, stmt)
# print(" << Query ({}) Exec Done".format(localSeqNum))
if (self._logfile):
with open(self._logfile, "a") as logfile:
logfile.write("%s;\n" % operation)
if res == 0:
if CTaosInterface.fieldsCount(self._connection._conn) == 0:
self._affected_rows += CTaosInterface.affectedRows(self._connection._conn)
self._affected_rows += CTaosInterface.affectedRows(
self._connection._conn)
return CTaosInterface.affectedRows(self._connection._conn)
else:
self._result, self._fields = CTaosInterface.useResult(self._connection._conn)
self._result, self._fields = CTaosInterface.useResult(
self._connection._conn)
return self._handle_result()
else:
raise ProgrammingError(CTaosInterface.errStr(self._connection._conn))
raise ProgrammingError(
CTaosInterface.errStr(
self._connection._conn))
def executemany(self, operation, seq_of_parameters):
"""Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.
@ -142,27 +156,58 @@ class TDengineCursor(object):
def fetchmany(self):
pass
def istype(self, col, dataType):
if (dataType.upper() == "BOOL"):
if (self._description[col][1] == FieldType.C_BOOL):
return True
if (dataType.upper() == "TINYINT"):
if (self._description[col][1] == FieldType.C_TINYINT):
return True
if (dataType.upper() == "INT"):
if (self._description[col][1] == FieldType.C_INT):
return True
if (dataType.upper() == "BIGINT"):
if (self._description[col][1] == FieldType.C_INT):
return True
if (dataType.upper() == "FLOAT"):
if (self._description[col][1] == FieldType.C_FLOAT):
return True
if (dataType.upper() == "DOUBLE"):
if (self._description[col][1] == FieldType.C_DOUBLE):
return True
if (dataType.upper() == "BINARY"):
if (self._description[col][1] == FieldType.C_BINARY):
return True
if (dataType.upper() == "TIMESTAMP"):
if (self._description[col][1] == FieldType.C_TIMESTAMP):
return True
if (dataType.upper() == "NCHAR"):
if (self._description[col][1] == FieldType.C_NCHAR):
return True
return False
def fetchall(self):
"""Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation.
"""
if self._result is None or self._fields is None:
raise OperationalError("Invalid use of fetchall")
buffer = [[] for i in range(len(self._fields))]
self._rowcount = 0
while True:
block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
if num_of_fields == 0: break
block, num_of_fields = CTaosInterface.fetchBlock(
self._result, self._fields)
if num_of_fields == 0:
break
self._rowcount += num_of_fields
for i in range(len(self._fields)):
buffer[i].extend(block[i])
self._connection.clear_result_set()
return list(map(tuple, zip(*buffer)))
def nextset(self):
"""
"""
@ -185,12 +230,13 @@ class TDengineCursor(object):
self._block_rows = -1
self._block_iter = 0
self._affected_rows = 0
def _handle_result(self):
"""Handle the return result from query.
"""
self._description = []
for ele in self._fields:
self._description.append((ele['name'], ele['type'], None, None, None, None, False))
self._description.append(
(ele['name'], ele['type'], None, None, None, None, False))
return self._result

69
src/cq/src/cqMain.c
View File

@ -18,6 +18,7 @@
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <errno.h>
#include "taosdef.h"
#include "taosmsg.h"
#include "tglobal.h"
@ -40,6 +41,7 @@ typedef struct {
int num; // number of continuous streams
struct SCqObj *pHead;
void *dbConn;
int master;
pthread_mutex_t mutex;
} SCqContext;
@ -58,17 +60,22 @@ typedef struct SCqObj {
int cqDebugFlag = 135;
static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row);
static void cqCreateStream(SCqContext *pContext, SCqObj *pObj);
void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
SCqContext *pContext = calloc(sizeof(SCqContext), 1);
if (pContext == NULL) return NULL;
if (pContext == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
strcpy(pContext->user, pCfg->user);
strcpy(pContext->pass, pCfg->pass);
pContext->vgId = pCfg->vgId;
pContext->cqWrite = pCfg->cqWrite;
pContext->ahandle = ahandle;
tscEmbedded = 1;
pthread_mutex_init(&pContext->mutex, NULL);
@ -79,11 +86,14 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
void cqClose(void *handle) {
SCqContext *pContext = handle;
if (handle == NULL) return;
// stop all CQs
cqStop(pContext);
// free all resources
pthread_mutex_lock(&pContext->mutex);
SCqObj *pObj = pContext->pHead;
while (pObj) {
SCqObj *pTemp = pObj;
@ -91,6 +101,8 @@ void cqClose(void *handle) {
free(pTemp);
}
pthread_mutex_unlock(&pContext->mutex);
pthread_mutex_destroy(&pContext->mutex);
cTrace("vgId:%d, CQ is closed", pContext->vgId);
@ -99,29 +111,16 @@ void cqClose(void *handle) {
void cqStart(void *handle) {
SCqContext *pContext = handle;
cTrace("vgId:%d, start all CQs", pContext->vgId);
if (pContext->dbConn) return;
if (pContext->dbConn || pContext->master) return;
cTrace("vgId:%d, start all CQs", pContext->vgId);
pthread_mutex_lock(&pContext->mutex);
tscEmbedded = 1;
pContext->dbConn = taos_connect("localhost", pContext->user, pContext->pass, NULL, 0);
if (pContext->dbConn == NULL) {
cError("vgId:%d, failed to connect to TDengine(%s)", pContext->vgId, tstrerror(terrno));
pthread_mutex_unlock(&pContext->mutex);
return;
}
pContext->master = 1;
SCqObj *pObj = pContext->pHead;
while (pObj) {
int64_t lastKey = 0;
pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, lastKey, pObj, NULL);
if (pObj->pStream) {
pContext->num++;
cTrace("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
} else {
cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
}
cqCreateStream(pContext, pObj);
pObj = pObj->next;
}
@ -131,10 +130,11 @@ void cqStart(void *handle) {
void cqStop(void *handle) {
SCqContext *pContext = handle;
cTrace("vgId:%d, stop all CQs", pContext->vgId);
if (pContext->dbConn == NULL) return;
if (pContext->dbConn == NULL || pContext->master == 0) return;
pthread_mutex_lock(&pContext->mutex);
pContext->master = 0;
SCqObj *pObj = pContext->pHead;
while (pObj) {
if (pObj->pStream) {
@ -176,16 +176,7 @@ void *cqCreate(void *handle, int tid, char *sqlStr, SSchema *pSchema, int column
if (pContext->pHead) pContext->pHead->prev = pObj;
pContext->pHead = pObj;
if (pContext->dbConn) {
int64_t lastKey = 0;
pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, lastKey, pObj, NULL);
if (pObj->pStream) {
pContext->num++;
cTrace("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
} else {
cError("vgId:%d, id:%d CQ:%s, failed to launch", pContext->vgId, pObj->tid, pObj->sqlStr);
}
}
cqCreateStream(pContext, pObj);
pthread_mutex_unlock(&pContext->mutex);
@ -218,6 +209,26 @@ void cqDrop(void *handle) {
pthread_mutex_lock(&pContext->mutex);
}
static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {
if (pContext->dbConn == NULL) {
pContext->dbConn = taos_connect("localhost", pContext->user, pContext->pass, NULL, 0);
if (pContext->dbConn == NULL) {
cError("vgId:%d, failed to connect to TDengine(%s)", pContext->vgId, tstrerror(terrno));
}
return;
}
int64_t lastKey = 0;
pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, lastKey, pObj, NULL);
if (pObj->pStream) {
pContext->num++;
cTrace("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
} else {
cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
}
}
static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
SCqObj *pObj = (SCqObj *)param;
SCqContext *pContext = pObj->pContext;

2
src/dnode/CMakeLists.txt
View File

@ -16,7 +16,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
AUX_SOURCE_DIRECTORY(src SRC)
ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb twal vnode cJson lz4)
TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4)
IF (TD_ACCOUNT)
TARGET_LINK_LIBRARIES(taosd account)

1
src/dnode/src/dnodeMain.c
View File

@ -37,6 +37,7 @@ static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
int32_t dnodeInitSystem() {
dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE);
tscEmbedded = 1;
taosBlockSIGPIPE();
taosResolveCRC();
taosInitGlobalCfg();
taosReadGlobalLogCfg();

37
src/dnode/src/dnodeModule.c
View File

@ -20,6 +20,7 @@
#include "trpc.h"
#include "mnode.h"
#include "http.h"
#include "mqtt.h"
#include "monitor.h"
#include "dnodeInt.h"
#include "dnodeModule.h"
@ -62,6 +63,16 @@ static void dnodeAllocModules() {
dnodeSetModuleStatus(TSDB_MOD_HTTP);
}
tsModule[TSDB_MOD_MQTT].enable = (tsEnableMqttModule == 1);
tsModule[TSDB_MOD_MQTT].name = "mqtt";
tsModule[TSDB_MOD_MQTT].initFp = mqttInitSystem;
tsModule[TSDB_MOD_MQTT].cleanUpFp = mqttCleanUpSystem;
tsModule[TSDB_MOD_MQTT].startFp = mqttStartSystem;
tsModule[TSDB_MOD_MQTT].stopFp = mqttStopSystem;
if (tsEnableMqttModule) {
dnodeSetModuleStatus(TSDB_MOD_MQTT);
}
tsModule[TSDB_MOD_MONITOR].enable = (tsEnableMonitorModule == 1);
tsModule[TSDB_MOD_MONITOR].name = "monitor";
tsModule[TSDB_MOD_MONITOR].initFp = monitorInitSystem;
@ -114,18 +125,20 @@ void dnodeStartModules() {
}
void dnodeProcessModuleStatus(uint32_t moduleStatus) {
bool enableMgmtModule = moduleStatus & (1 << TSDB_MOD_MGMT);
if (!tsModule[TSDB_MOD_MGMT].enable && enableMgmtModule) {
dPrint("module status is received, start mgmt module", tsModuleStatus, moduleStatus);
tsModule[TSDB_MOD_MGMT].enable = true;
dnodeSetModuleStatus(TSDB_MOD_MGMT);
(*tsModule[TSDB_MOD_MGMT].startFp)();
}
for (int32_t module = TSDB_MOD_MGMT; module < TSDB_MOD_HTTP; ++module) {
bool enableModule = moduleStatus & (1 << module);
if (!tsModule[module].enable && enableModule) {
dPrint("module status:%u is received, start %s module", tsModuleStatus, tsModule[module].name);
tsModule[module].enable = true;
dnodeSetModuleStatus(module);
(*tsModule[module].startFp)();
}
if (tsModule[TSDB_MOD_MGMT].enable && !enableMgmtModule) {
dPrint("module status is received, stop mgmt module", tsModuleStatus, moduleStatus);
tsModule[TSDB_MOD_MGMT].enable = false;
dnodeUnSetModuleStatus(TSDB_MOD_MGMT);
(*tsModule[TSDB_MOD_MGMT].stopFp)();
if (tsModule[module].enable && !enableModule) {
dPrint("module status:%u is received, stop %s module", tsModuleStatus, tsModule[module].name);
tsModule[module].enable = false;
dnodeUnSetModuleStatus(module);
(*tsModule[module].stopFp)();
}
}
}

39
src/dnode/src/dnodeSystem.c
View File

@ -22,6 +22,7 @@
#include "dnodeMain.h"
static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
static sem_t exitSem;
int32_t main(int32_t argc, char *argv[]) {
// Set global configuration file
@ -65,6 +66,11 @@ int32_t main(int32_t argc, char *argv[]) {
#endif
}
if (sem_init(&exitSem, 0, 0) != 0) {
printf("failed to create exit semphore\n");
exit(EXIT_FAILURE);
}
/* Set termination handler. */
struct sigaction act = {{0}};
act.sa_flags = SA_SIGINFO;
@ -90,9 +96,19 @@ int32_t main(int32_t argc, char *argv[]) {
syslog(LOG_INFO, "Started TDengine service successfully.");
while (1) {
sleep(1000);
for (int res = sem_wait(&exitSem); res != 0; res = sem_wait(&exitSem)) {
if (res != EINTR) {
syslog(LOG_ERR, "failed to wait exit semphore: %d", res);
break;
}
}
dnodeCleanUpSystem();
// close the syslog
syslog(LOG_INFO, "Shut down TDengine service successfully");
dPrint("TDengine is shut down!");
closelog();
return EXIT_SUCCESS;
}
static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) {
@ -104,14 +120,21 @@ static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) {
taosCfgDynamicOptions("resetlog");
return;
}
syslog(LOG_INFO, "Shut down signal is %d", signum);
syslog(LOG_INFO, "Shutting down TDengine service...");
// clean the system.
dPrint("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid);
dnodeCleanUpSystem();
// close the syslog
syslog(LOG_INFO, "Shut down TDengine service successfully");
dPrint("TDengine is shut down!");
closelog();
exit(EXIT_SUCCESS);
// protect the application from receive another signal
struct sigaction act = {{0}};
act.sa_handler = SIG_IGN;
sigaction(SIGTERM, &act, NULL);
sigaction(SIGHUP, &act, NULL);
sigaction(SIGINT, &act, NULL);
sigaction(SIGUSR1, &act, NULL);
sigaction(SIGUSR2, &act, NULL);
// inform main thread to exit
sem_post(&exitSem);
}

15
src/dnode/src/dnodeVRead.c
View File

@ -17,6 +17,7 @@
#include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tutil.h"
#include "tqueue.h"
#include "trpc.h"
#include "twal.h"
@ -71,11 +72,16 @@ int32_t dnodeInitRead() {
}
void dnodeCleanupRead() {
for (int i=0; i < readPool.max; ++i) {
SReadWorker *pWorker = readPool.readWorker + i;
if (pWorker->thread) {
taosQsetThreadResume(readQset);
}
}
for (int i=0; i < readPool.max; ++i) {
SReadWorker *pWorker = readPool.readWorker + i;
if (pWorker->thread) {
pthread_cancel(pWorker->thread);
pthread_join(pWorker->thread, NULL);
}
}
@ -201,15 +207,14 @@ void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) {
}
static void *dnodeProcessReadQueue(void *param) {
SReadWorker *pWorker = param;
SReadMsg *pReadMsg;
int type;
void *pVnode;
while (1) {
if (taosReadQitemFromQset(readQset, &type, (void **)&pReadMsg, &pVnode) == 0) {
dnodeHandleIdleReadWorker(pWorker);
continue;
dTrace("dnodeProcessReadQueee: got no message from qset, exiting...");
break;
}
dTrace("%p, msg:%s will be processed", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]);
@ -221,6 +226,8 @@ static void *dnodeProcessReadQueue(void *param) {
return NULL;
}
UNUSED_FUNC
static void dnodeHandleIdleReadWorker(SReadWorker *pWorker) {
int32_t num = taosGetQueueNumber(readQset);

15
src/dnode/src/dnodeVWrite.c
View File

@ -17,6 +17,7 @@
#include "os.h"
#include "taosmsg.h"
#include "taoserror.h"
#include "tutil.h"
#include "tqueue.h"
#include "trpc.h"
#include "tsdb.h"
@ -67,11 +68,16 @@ int32_t dnodeInitWrite() {
}
void dnodeCleanupWrite() {
for (int32_t i = 0; i < wWorkerPool.max; ++i) {
SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
if (pWorker->thread) {
taosQsetThreadResume(pWorker->qset);
}
}
for (int32_t i = 0; i < wWorkerPool.max; ++i) {
SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
if (pWorker->thread) {
pthread_cancel(pWorker->thread);
pthread_join(pWorker->thread, NULL);
taosFreeQall(pWorker->qall);
taosCloseQset(pWorker->qset);
@ -186,9 +192,9 @@ static void *dnodeProcessWriteQueue(void *param) {
while (1) {
numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode);
if (numOfMsgs <=0) {
dnodeHandleIdleWorker(pWorker); // thread exit if no queues anymore
continue;
if (numOfMsgs ==0) {
dTrace("dnodeProcessWriteQueee: got no message from qset, exiting...");
break;
}
for (int32_t i = 0; i < numOfMsgs; ++i) {
@ -228,6 +234,7 @@ static void *dnodeProcessWriteQueue(void *param) {
return NULL;
}
UNUSED_FUNC
static void dnodeHandleIdleWorker(SWriteWorker *pWorker) {
int32_t num = taosGetQueueNumber(pWorker->qset);

35
src/inc/mqtt.h Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MQTT_H
#define TDENGINE_MQTT_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
int32_t mqttGetReqCount();
int32_t mqttInitSystem();
int32_t mqttStartSystem();
void mqttStopSystem();
void mqttCleanUpSystem();
#ifdef __cplusplus
}
#endif
#endif

22
src/inc/taosdef.h
View File

@ -32,7 +32,7 @@ extern "C" {
#define TSKEY int64_t
#endif
#define TSWINDOW_INITIALIZER {INT64_MIN, INT64_MAX};
#define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX})
#define TSKEY_INITIAL_VAL INT64_MIN
// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
@ -252,12 +252,12 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta
#define TSDB_MIN_CACHE_BLOCK_SIZE 1
#define TSDB_MAX_CACHE_BLOCK_SIZE 10240 // 10GB for each vnode
#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode
#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16
#define TSDB_MIN_TOTAL_BLOCKS 2
#define TSDB_MAX_TOTAL_BLOCKS 10000
#define TSDB_DEFAULT_TOTAL_BLOCKS 2
#define TSDB_DEFAULT_TOTAL_BLOCKS 4
#define TSDB_MIN_TABLES 4
#define TSDB_MAX_TABLES 200000
@ -340,13 +340,14 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_SESSIONS_PER_VNODE (300)
#define TSDB_SESSIONS_PER_DNODE (TSDB_SESSIONS_PER_VNODE * TSDB_MAX_VNODES)
#define TSDB_MAX_MNODES 5
#define TSDB_MAX_DNODES 10
#define TSDB_MAX_ACCOUNTS 10
#define TSDB_MAX_USERS 20
#define TSDB_MAX_DBS 100
#define TSDB_MAX_VGROUPS 1000
#define TSDB_MAX_SUPER_TABLES 100
#define TSDB_DEFAULT_MNODES_HASH_SIZE 5
#define TSDB_DEFAULT_DNODES_HASH_SIZE 10
#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10
#define TSDB_DEFAULT_USERS_HASH_SIZE 20
#define TSDB_DEFAULT_DBS_HASH_SIZE 100
#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100
#define TSDB_DEFAULT_STABLES_HASH_SIZE 100
#define TSDB_DEFAULT_CTABLES_HASH_SIZE 10000
#define TSDB_PORT_DNODESHELL 0
#define TSDB_PORT_DNODEDNODE 5
@ -375,6 +376,7 @@ typedef enum {
TSDB_MOD_MGMT,
TSDB_MOD_HTTP,
TSDB_MOD_MONITOR,
TSDB_MOD_MQTT,
TSDB_MOD_MAX
} EModuleType;

1
src/inc/taoserror.h
View File

@ -156,6 +156,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY, 0, 405, "server out of m
TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS, 0, 406, "no disk permissions")
TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, 0, 407, "file corrupted")
TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, 0, 408, "memory corrupted")
TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUCH_FILE_OR_DIR, 0, 409, "no such file or directory")
// client
TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION, 0, 451, "invalid client version")

47
src/inc/taosmsg.h
View File

@ -140,19 +140,19 @@ enum _mgmt_table {
TSDB_MGMT_TABLE_MAX,
};
#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1
#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2
#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1
#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2
#define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3
#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
#define TSDB_ALTER_TABLE_ADD_COLUMN 5
#define TSDB_ALTER_TABLE_DROP_COLUMN 6
#define TSDB_ALTER_TABLE_ADD_COLUMN 5
#define TSDB_ALTER_TABLE_DROP_COLUMN 6
#define TSDB_INTERPO_NONE 0
#define TSDB_INTERPO_NULL 1
#define TSDB_INTERPO_SET_VALUE 2
#define TSDB_INTERPO_LINEAR 3
#define TSDB_INTERPO_PREV 4
#define TSDB_FILL_NONE 0
#define TSDB_FILL_NULL 1
#define TSDB_FILL_SET_VALUE 2
#define TSDB_FILL_LINEAR 3
#define TSDB_FILL_PREV 4
#define TSDB_ALTER_USER_PASSWD 0x1
#define TSDB_ALTER_USER_PRIVILEGES 0x2
@ -164,8 +164,8 @@ enum _mgmt_table {
#define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS)
#define TSDB_COL_NORMAL 0x0u
#define TSDB_COL_TAG 0x1u
#define TSDB_COL_JOIN 0x2u
#define TSDB_COL_TAG 0x1u
#define TSDB_COL_JOIN 0x2u
extern char *taosMsg[];
@ -269,9 +269,11 @@ typedef struct {
char tableId[TSDB_TABLE_ID_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1];
int16_t type; /* operation type */
char tagVal[TSDB_MAX_BYTES_PER_ROW];
int8_t numOfCols; /* number of schema */
int16_t numOfCols; /* number of schema */
int32_t tagValLen;
SSchema schema[];
// tagVal is padded after schema
// char tagVal[];
} SCMAlterTableMsg;
typedef struct {
@ -440,7 +442,7 @@ typedef struct {
uint16_t queryType; // denote another query process
int16_t numOfOutput; // final output columns numbers
int16_t tagNameRelType; // relation of tag criteria and tbname criteria
int16_t interpoType; // interpolate type
int16_t fillType; // interpolate type
uint64_t defaultVal; // default value array list
int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed
int32_t tsLen; // total length of ts comp block
@ -485,20 +487,20 @@ typedef struct {
typedef struct {
char acct[TSDB_USER_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1];
int32_t maxTables;
int32_t cacheBlockSize; //MB
int32_t numOfBlocks;
int32_t totalBlocks;
int32_t maxTables;
int32_t daysPerFile;
int32_t daysToKeep;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t daysToKeep;
int32_t commitTime;
int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock;
int32_t commitTime;
uint8_t precision; // time resolution
int8_t compression;
int8_t walLevel;
int8_t replications;
uint8_t precision; // time resolution
int8_t ignoreExist;
} SCMCreateDbMsg, SCMAlterDbMsg;
@ -563,9 +565,9 @@ typedef struct {
typedef struct {
uint32_t vgId;
int32_t cfgVersion;
int32_t maxTables;
int32_t cacheBlockSize;
int32_t totalBlocks;
int32_t maxTables;
int32_t daysPerFile;
int32_t daysToKeep;
int32_t daysToKeep1;
@ -647,6 +649,7 @@ typedef struct SMultiTableMeta {
} SMultiTableMeta;
typedef struct {
int32_t dataLen;
char name[TSDB_TABLE_ID_LEN + 1];
char data[TSDB_MAX_TAGS_LEN];
} STagData;
@ -722,6 +725,8 @@ typedef struct {
typedef struct {
uint32_t queryId;
uint32_t streamId;
uint32_t totalDnodes;
uint32_t onlineDnodes;
int8_t killConnection;
SRpcIpSet ipList;
} SCMHeartBeatRsp;

6
src/inc/tsdb.h
View File

@ -167,12 +167,6 @@ typedef struct {
SArray *pGroupList;
} STableGroupInfo;
typedef struct {
} SFields;
#define TSDB_TS_GREATER_EQUAL 1
#define TSDB_TS_LESS_EQUAL 2
typedef struct SQueryRowCond {
int32_t rel;
TSKEY ts;

4
src/inc/tsync.h
View File

@ -94,8 +94,8 @@ typedef void* tsync_h;
tsync_h syncStart(const SSyncInfo *);
void syncStop(tsync_h shandle);
int syncReconfig(tsync_h shandle, const SSyncCfg *);
int syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle);
int32_t syncReconfig(tsync_h shandle, const SSyncCfg *);
int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype);
void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code);
void syncRecover(tsync_h shandle); // recover from other nodes:
int syncGetNodesRole(tsync_h shandle, SNodesRole *);

33
src/kit/taosdemo/taosdemo.c
View File

@ -362,6 +362,26 @@ int main(int argc, char *argv[]) {
time_t tTime = time(NULL);
struct tm tm = *localtime(&tTime);
printf("###################################################################\n");
printf("# Server IP: %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port);
printf("# User: %s\n", user);
printf("# Password: %s\n", pass);
printf("# Use metric: %s\n", use_metric ? "true" : "false");
printf("# Datatype of Columns: %s\n", dataString);
printf("# Binary Length(If applicable): %d\n",
(strcasestr(dataString, "BINARY") != NULL) ? len_of_binary : -1);
printf("# Number of Columns per record: %d\n", ncols_per_record);
printf("# Number of Connections: %d\n", nconnections);
printf("# Number of Tables: %d\n", ntables);
printf("# Number of Data per Table: %d\n", nrecords_per_table);
printf("# Records/Request: %d\n", nrecords_per_request);
printf("# Database name: %s\n", db_name);
printf("# Table prefix: %s\n", tb_prefix);
printf("# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
printf("###################################################################\n\n");
printf("Press enter key to continue");
getchar();
fprintf(fp, "###################################################################\n");
fprintf(fp, "# Server IP: %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port);
@ -858,15 +878,16 @@ void generateData(char *res, char **data_type, int num_of_cols, int64_t timestam
pstr += sprintf(pstr, ")");
}
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890";
void rand_string(char *str, int size) {
memset(str, 0, size);
const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890";
char *sptr = str;
if (size) {
str[0] = 0;
if (size > 0) {
--size;
for (size_t n = 0; n < size; n++) {
int n;
for (n = 0; n < size; n++) {
int key = rand() % (int)(sizeof charset - 1);
sptr += sprintf(sptr, "%c", charset[key]);
str[n] = charset[key];
}
str[n] = 0;
}
}

63
src/kit/taosdump/taosdump.c
View File

@ -37,6 +37,8 @@
#define COMMAND_SIZE 65536
#define DEFAULT_DUMP_FILE "taosdump.sql"
#define MAX_DBS 100
int converStringToReadable(char *str, int size, char *buf, int bufsize);
int convertNCharToReadable(char *str, int size, char *buf, int bufsize);
void taosDumpCharset(FILE *fp);
@ -359,7 +361,7 @@ int main(int argc, char *argv[]) {
void taosFreeDbInfos() {
if (dbInfos == NULL) return;
for (int i = 0; i < TSDB_MAX_DBS; i++) tfree(dbInfos[i]);
for (int i = 0; i < MAX_DBS; i++) tfree(dbInfos[i]);
tfree(dbInfos);
}
@ -437,7 +439,7 @@ int taosDumpOut(SDumpArguments *arguments) {
return -1;
}
dbInfos = (SDbInfo **)calloc(TSDB_MAX_DBS, sizeof(SDbInfo *));
dbInfos = (SDbInfo **)calloc(MAX_DBS, sizeof(SDbInfo *));
if (dbInfos == NULL) {
fprintf(stderr, "failed to allocate memory\n");
goto _exit_failure;
@ -531,7 +533,7 @@ int taosDumpOut(SDumpArguments *arguments) {
}
}
taos_free_result(result);
// taos_free_result(result);
if (count == 0) {
fprintf(stderr, "No databases valid to dump\n");
@ -720,6 +722,57 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols
count_temp = counter;
for (; counter < numOfCols; counter++) {
TAOS_ROW row = NULL;
sprintf(command, "select %s from %s limit 1", tableDes->cols[counter].field, tableDes->name);
if (taos_query(taos, command) != 0) {
fprintf(stderr, "failed to run command %s\n", command);
return;
}
result = taos_use_result(taos);
if (result == NULL) {
fprintf(stderr, "failed to use result\n");
return;
}
TAOS_FIELD *fields = taos_fetch_fields(result);
row = taos_fetch_row(result);
switch (fields[0].type) {
case TSDB_DATA_TYPE_BOOL:
sprintf(tableDes->cols[counter].note, "%d", ((((int)(*((char *)row[0]))) == 1) ? 1 : 0));
break;
case TSDB_DATA_TYPE_TINYINT:
sprintf(tableDes->cols[counter].note, "%d", (int)(*((char *)row[0])));
break;
case TSDB_DATA_TYPE_SMALLINT:
sprintf(tableDes->cols[counter].note, "%d", (int)(*((short *)row[0])));
break;
case TSDB_DATA_TYPE_INT:
sprintf(tableDes->cols[counter].note, "%d", *((int *)row[0]));
break;
case TSDB_DATA_TYPE_BIGINT:
sprintf(tableDes->cols[counter].note, "%" PRId64 "", *((int64_t *)row[0]));
break;
case TSDB_DATA_TYPE_FLOAT:
sprintf(tableDes->cols[counter].note, "%f", GET_FLOAT_VAL(row[0]));
break;
case TSDB_DATA_TYPE_DOUBLE:
sprintf(tableDes->cols[counter].note, "%f", GET_DOUBLE_VAL(row[0]));
break;
case TSDB_DATA_TYPE_TIMESTAMP:
sprintf(tableDes->cols[counter].note, "%" PRId64 "", *(int64_t *)row[0]);
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
default:
strncpy(tableDes->cols[counter].note, (char *)row[0], fields[0].bytes);
break;
}
taos_free_result(result);
if (counter != count_temp) {
if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
@ -941,7 +994,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) {
pstr += sprintf(pstr, "%d", *((int *)row[col]));
break;
case TSDB_DATA_TYPE_BIGINT:
pstr += sprintf(pstr, "%" PRId64 "", *((int64_t *)row[col]));
pstr += sprintf(pstr, "%" PRId64, *((int64_t *)row[col]));
break;
case TSDB_DATA_TYPE_FLOAT:
pstr += sprintf(pstr, "%f", GET_FLOAT_VAL(row[col]));
@ -960,7 +1013,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) {
pstr += sprintf(pstr, "\'%s\'", tbuf);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
pstr += sprintf(pstr, "%" PRId64 "", *(int64_t *)row[col]);
pstr += sprintf(pstr, "%" PRId64, *(int64_t *)row[col]);
break;
default:
break;

2
src/mnode/inc/mgmtDef.h
View File

@ -63,10 +63,8 @@ typedef struct SMnodeObj {
int8_t updateEnd[1];
int32_t refCount;
int8_t role;
SDnodeObj *pDnode;
} SMnodeObj;
// todo use dynamic length string
typedef struct {
char *tableId;
int8_t type;

1
src/mnode/inc/mgmtDnode.h
View File

@ -34,6 +34,7 @@ char* mgmtGetDnodeStatusStr(int32_t dnodeStatus);
void mgmtMonitorDnodeModule();
int32_t mgmtGetDnodesNum();
int32_t mgmtGetOnlinDnodesNum();
void * mgmtGetNextDnode(void *pIter, SDnodeObj **pDnode);
void mgmtIncDnodeRef(SDnodeObj *pDnode);
void mgmtDecDnodeRef(SDnodeObj *pDnode);

2
src/mnode/inc/mgmtMnode.h
View File

@ -44,7 +44,7 @@ void mgmtDecMnodeRef(struct SMnodeObj *pMnode);
char * mgmtGetMnodeRoleStr();
void mgmtGetMnodeIpSet(SRpcIpSet *ipSet);
void mgmtGetMnodeInfos(void *mnodes);
void mgmtUpdateMnodeIpSet();
#ifdef __cplusplus
}

1
src/mnode/inc/mgmtTable.h
View File

@ -31,6 +31,7 @@ void * mgmtGetNextChildTable(void *pIter, SChildTableObj **pTable);
void * mgmtGetNextSuperTable(void *pIter, SSuperTableObj **pTable);
void mgmtDropAllChildTables(SDbObj *pDropDb);
void mgmtDropAllSuperTables(SDbObj *pDropDb);
void mgmtDropAllChildTablesInVgroups(SVgObj *pVgroup);
#ifdef __cplusplus
}

1
src/mnode/inc/mgmtVgroup.h
View File

@ -29,6 +29,7 @@ void mgmtIncVgroupRef(SVgObj *pVgroup);
void mgmtDecVgroupRef(SVgObj *pVgroup);
void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg);
void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode);
void mgmtUpdateAllDbVgroups(SDbObj *pAlterDb);
void * mgmtGetNextVgroup(void *pIter, SVgObj **pVgroup);
void mgmtUpdateVgroup(SVgObj *pVgroup);

2
src/mnode/src/mgmtAcct.c
View File

@ -94,7 +94,7 @@ int32_t mgmtInitAccts() {
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_ACCOUNT,
.tableName = "accounts",
.hashSessions = TSDB_MAX_ACCOUNTS,
.hashSessions = TSDB_DEFAULT_ACCOUNTS_HASH_SIZE,
.maxRowSize = tsAcctUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_STRING,

117
src/mnode/src/mgmtDb.c
View File

@ -96,6 +96,7 @@ static int32_t mgmtDbActionUpdate(SSdbOper *pOper) {
memcpy(pSaved, pDb, pOper->rowSize);
free(pDb);
}
mgmtUpdateAllDbVgroups(pSaved);
mgmtDecDbRef(pSaved);
return TSDB_CODE_SUCCESS;
}
@ -127,7 +128,7 @@ int32_t mgmtInitDbs() {
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_DB,
.tableName = "dbs",
.hashSessions = TSDB_MAX_DBS,
.hashSessions = TSDB_DEFAULT_DBS_HASH_SIZE,
.maxRowSize = tsDbUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_STRING,
@ -187,11 +188,13 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) {
mError("invalid db option cacheBlockSize:%d valid range: [%d, %d]", pCfg->cacheBlockSize, TSDB_MIN_CACHE_BLOCK_SIZE,
TSDB_MAX_CACHE_BLOCK_SIZE);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) {
mError("invalid db option totalBlocks:%d valid range: [%d, %d]", pCfg->totalBlocks, TSDB_MIN_TOTAL_BLOCKS,
TSDB_MAX_TOTAL_BLOCKS);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) {
@ -206,18 +209,22 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
}
if (pCfg->daysToKeep < TSDB_MIN_KEEP || pCfg->daysToKeep > TSDB_MAX_KEEP) {
mError("invalid db option daysToKeep:%d", pCfg->daysToKeep);
mError("invalid db option daysToKeep:%d valid range: [%d, %d]", pCfg->daysToKeep, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->daysToKeep < pCfg->daysPerFile) {
mError("invalid db option daysToKeep:%d daysPerFile:%d", pCfg->daysToKeep, pCfg->daysPerFile);
mError("invalid db option daysToKeep:%d should larger than daysPerFile:%d", pCfg->daysToKeep, pCfg->daysPerFile);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) {
mError("invalid db option minRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->minRowsPerFileBlock,
TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > pCfg->daysToKeep) {
mError("invalid db option daysToKeep2:%d valid range: [%d, %d]", pCfg->daysToKeep, TSDB_MIN_KEEP, pCfg->daysToKeep);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) {
mError("invalid db option daysToKeep1:%d valid range: [%d, %d]", pCfg->daysToKeep1, TSDB_MIN_KEEP, pCfg->daysToKeep2);
return TSDB_CODE_INVALID_OPTION;
}
@ -227,9 +234,15 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) {
mError("invalid db option minRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->minRowsPerFileBlock,
TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) {
mError("invalid db option minRowsPerFileBlock:%d maxRowsPerFileBlock:%d", pCfg->minRowsPerFileBlock,
pCfg->maxRowsPerFileBlock);
mError("invalid db option minRowsPerFileBlock:%d should smaller than maxRowsPerFileBlock:%d",
pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock);
return TSDB_CODE_INVALID_OPTION;
}
@ -252,7 +265,7 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
}
if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) {
mError("invalid db option walLevel:%d, only 0-2 allowed", pCfg->walLevel);
mError("invalid db option walLevel:%d, valid range: [%d, %d]", pCfg->walLevel, TSDB_MIN_WAL_LEVEL, TSDB_MAX_WAL_LEVEL);
return TSDB_CODE_INVALID_OPTION;
}
@ -262,6 +275,11 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->replications > 1 && pCfg->walLevel <= TSDB_MIN_WAL_LEVEL) {
mError("invalid db option walLevel:%d must > 0, while replica:%d > 1", pCfg->walLevel, pCfg->replications);
return TSDB_CODE_INVALID_OPTION;
}
#ifndef _SYNC
if (pCfg->replications != 1) {
mError("invalid db option replications:%d can only be 1 in this version", pCfg->replications);
@ -314,13 +332,13 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
pDb->createdTime = taosGetTimestampMs();
pDb->cfg = (SDbCfg) {
.cacheBlockSize = pCreate->cacheBlockSize,
.totalBlocks = pCreate->numOfBlocks,
.totalBlocks = pCreate->totalBlocks,
.maxTables = pCreate->maxTables,
.daysPerFile = pCreate->daysPerFile,
.daysToKeep = pCreate->daysToKeep,
.daysToKeep1 = pCreate->daysToKeep1,
.daysToKeep2 = pCreate->daysToKeep2,
.minRowsPerFileBlock = pCreate->maxRowsPerFileBlock,
.minRowsPerFileBlock = pCreate->minRowsPerFileBlock,
.maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock,
.commitTime = pCreate->commitTime,
.precision = pCreate->precision,
@ -735,7 +753,7 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
pCreate->maxTables = htonl(pCreate->maxTables);
pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize);
pCreate->numOfBlocks = htonl(pCreate->numOfBlocks);
pCreate->totalBlocks = htonl(pCreate->totalBlocks);
pCreate->daysPerFile = htonl(pCreate->daysPerFile);
pCreate->daysToKeep = htonl(pCreate->daysToKeep);
pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1);
@ -763,37 +781,47 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
SDbCfg newCfg = pDb->cfg;
int32_t cacheBlockSize = htonl(pAlter->daysToKeep);
int32_t totalBlocks = htonl(pAlter->numOfBlocks);
int32_t maxTables = htonl(pAlter->maxTables);
int32_t daysToKeep = htonl(pAlter->daysToKeep);
int32_t daysToKeep1 = htonl(pAlter->daysToKeep1);
int32_t daysToKeep2 = htonl(pAlter->daysToKeep2);
int8_t compression = pAlter->compression;
int8_t replications = pAlter->replications;
int8_t walLevel = pAlter->walLevel;
int32_t maxTables = htonl(pAlter->maxTables);
int32_t cacheBlockSize = htonl(pAlter->cacheBlockSize);
int32_t totalBlocks = htonl(pAlter->totalBlocks);
int32_t daysPerFile = htonl(pAlter->daysPerFile);
int32_t daysToKeep = htonl(pAlter->daysToKeep);
int32_t daysToKeep1 = htonl(pAlter->daysToKeep1);
int32_t daysToKeep2 = htonl(pAlter->daysToKeep2);
int32_t minRows = htonl(pAlter->minRowsPerFileBlock);
int32_t maxRows = htonl(pAlter->maxRowsPerFileBlock);
int32_t commitTime = htonl(pAlter->commitTime);
int8_t compression = pAlter->compression;
int8_t walLevel = pAlter->walLevel;
int8_t replications = pAlter->replications;
int8_t precision = pAlter->precision;
terrno = TSDB_CODE_SUCCESS;
if (cacheBlockSize > 0 && cacheBlockSize != pDb->cfg.cacheBlockSize) {
mTrace("db:%s, cache:%d change to %d", pDb->name, pDb->cfg.cacheBlockSize, cacheBlockSize);
newCfg.cacheBlockSize = cacheBlockSize;
mError("db:%s, can't alter cache option", pDb->name);
terrno = TSDB_CODE_INVALID_OPTION;
}
if (totalBlocks > 0 && totalBlocks != pDb->cfg.totalBlocks) {
mTrace("db:%s, blocks:%d change to %d", pDb->name, pDb->cfg.totalBlocks, totalBlocks);
mPrint("db:%s, blocks:%d change to %d", pDb->name, pDb->cfg.totalBlocks, totalBlocks);
newCfg.totalBlocks = totalBlocks;
}
if (maxTables > 0 && maxTables != pDb->cfg.maxTables) {
mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables);
if (maxTables > 0) {
mPrint("db:%s, maxTables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables);
newCfg.maxTables = maxTables;
if (newCfg.maxTables < pDb->cfg.maxTables) {
mTrace("db:%s, tables:%d should larger than origin:%d", pDb->name, newCfg.maxTables, pDb->cfg.maxTables);
mError("db:%s, tables:%d should larger than origin:%d", pDb->name, newCfg.maxTables, pDb->cfg.maxTables);
terrno = TSDB_CODE_INVALID_OPTION;
}
}
if (daysPerFile > 0 && daysPerFile != pDb->cfg.daysPerFile) {
mError("db:%s, can't alter days option", pDb->name);
terrno = TSDB_CODE_INVALID_OPTION;
}
if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) {
mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep);
newCfg.daysToKeep = daysToKeep;
@ -809,15 +837,45 @@ static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
newCfg.daysToKeep2 = daysToKeep2;
}
if (minRows > 0 && minRows != pDb->cfg.minRowsPerFileBlock) {
mError("db:%s, can't alter minRows option", pDb->name);
terrno = TSDB_CODE_INVALID_OPTION;
}
if (maxRows > 0 && maxRows != pDb->cfg.maxRowsPerFileBlock) {
mError("db:%s, can't alter maxRows option", pDb->name);
terrno = TSDB_CODE_INVALID_OPTION;
}
if (commitTime > 0 && commitTime != pDb->cfg.commitTime) {
mError("db:%s, can't alter commitTime option", pDb->name);
terrno = TSDB_CODE_INVALID_OPTION;
}
if (precision > 0 && precision != pDb->cfg.precision) {
mError("db:%s, can't alter precision option", pDb->name);
terrno = TSDB_CODE_INVALID_OPTION;
}
if (compression >= 0 && compression != pDb->cfg.compression) {
mTrace("db:%s, compression:%d change to %d", pDb->name, pDb->cfg.compression, compression);
newCfg.compression = compression;
}
if (walLevel > 0 && walLevel != pDb->cfg.walLevel) {
mError("db:%s, can't alter walLevel option", pDb->name);
terrno = TSDB_CODE_INVALID_OPTION;
}
if (replications > 0 && replications != pDb->cfg.replications) {
mTrace("db:%s, replications:%d change to %d", pDb->name, pDb->cfg.replications, replications);
newCfg.replications = replications;
if (replications > 1 && pDb->cfg.walLevel <= TSDB_MIN_WAL_LEVEL) {
mError("db:%s, walLevel:%d must > 0, while replica:%d > 1", pDb->name, pDb->cfg.walLevel, replications);
terrno = TSDB_CODE_INVALID_OPTION;
}
if (replications > mgmtGetDnodesNum()) {
mError("db:%s, no enough dnode to change replica:%d", pDb->name, replications);
terrno = TSDB_CODE_NO_ENOUGH_DNODES;
@ -829,11 +887,6 @@ static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
}
}
if (walLevel >= 0 && (walLevel < TSDB_MIN_WAL_LEVEL || walLevel > TSDB_MAX_WAL_LEVEL)) {
mError("db:%s, wal level %d should be between 0-2, origin:%d", pDb->name, walLevel, pDb->cfg.walLevel);
terrno = TSDB_CODE_INVALID_OPTION;
}
return newCfg;
}

23
src/mnode/src/mgmtDnode.c
View File

@ -74,7 +74,6 @@ static int32_t mgmtDnodeActionDelete(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
#ifndef _SYNC
//TODO: drop dnode local
mgmtDropAllDnodeVgroups(pDnode);
#endif
mgmtDropMnodeLocal(pDnode->dnodeId);
@ -130,7 +129,7 @@ int32_t mgmtInitDnodes() {
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_DNODE,
.tableName = "dnodes",
.hashSessions = TSDB_MAX_DNODES,
.hashSessions = TSDB_DEFAULT_DNODES_HASH_SIZE,
.maxRowSize = tsDnodeUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_AUTO,
@ -179,6 +178,23 @@ int32_t mgmtGetDnodesNum() {
return sdbGetNumOfRows(tsDnodeSdb);
}
int32_t mgmtGetOnlinDnodesNum(char *ep) {
SDnodeObj *pDnode = NULL;
void * pIter = NULL;
int32_t onlineDnodes = 0;
while (1) {
pIter = mgmtGetNextDnode(pIter, &pDnode);
if (pDnode == NULL) break;
if (pDnode->status != TAOS_DN_STATUS_OFFLINE) onlineDnodes++;
mgmtDecDnodeRef(pDnode);
}
sdbFreeIter(pIter);
return onlineDnodes;
}
void *mgmtGetDnode(int32_t dnodeId) {
return sdbGetRow(tsDnodeSdb, &dnodeId);
}
@ -397,7 +413,6 @@ static int32_t mgmtCreateDnode(char *ep) {
return code;
}
//TODO drop others tables
int32_t mgmtDropDnode(SDnodeObj *pDnode) {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
@ -410,7 +425,7 @@ int32_t mgmtDropDnode(SDnodeObj *pDnode) {
code = TSDB_CODE_SDB_ERROR;
}
mLPrint("dnode:%d is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
mLPrint("dnode:%d, is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
return code;
}

116
src/mnode/src/mgmtMnode.c
View File

@ -36,6 +36,25 @@ static int32_t tsMnodeUpdateSize = 0;
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static SRpcIpSet tsMnodeRpcIpSet;
static SDMMnodeInfos tsMnodeInfos;
#if defined(LINUX)
static pthread_rwlock_t tsMnodeLock;
#define mgmtMnodeWrLock() pthread_rwlock_wrlock(&tsMnodeLock)
#define mgmtMnodeRdLock() pthread_rwlock_rdlock(&tsMnodeLock)
#define mgmtMnodeUnLock() pthread_rwlock_unlock(&tsMnodeLock)
#define mgmtMnodeInitLock() pthread_rwlock_init(&tsMnodeLock, NULL)
#define mgmtMnodeDestroyLock() pthread_rwlock_destroy(&tsMnodeLock)
#else
static pthread_mutex_t tsMnodeLock;
#define mgmtMnodeWrLock() pthread_mutex_lock(&tsMnodeLock)
#define mgmtMnodeRdLock() pthread_mutex_lock(&tsMnodeLock)
#define mgmtMnodeUnLock() pthread_mutex_unlock(&tsMnodeLock)
#define mgmtMnodeInitLock() pthread_mutex_init(&tsMnodeLock, NULL)
#define mgmtMnodeDestroyLock() pthread_mutex_destroy(&tsMnodeLock)
#endif
static int32_t mgmtMnodeActionDestroy(SSdbOper *pOper) {
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
@ -46,7 +65,6 @@ static int32_t mgmtMnodeActionInsert(SSdbOper *pOper) {
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
if (pDnode == NULL) return TSDB_CODE_DNODE_NOT_EXIST;
pMnode->pDnode = pDnode;
pDnode->isMgmt = true;
mgmtDecDnodeRef(pDnode);
@ -102,17 +120,22 @@ static int32_t mgmtMnodeActionRestored() {
}
sdbFreeIter(pIter);
}
mgmtUpdateMnodeIpSet();
return TSDB_CODE_SUCCESS;
}
int32_t mgmtInitMnodes() {
mgmtMnodeInitLock();
SMnodeObj tObj;
tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_MNODE,
.tableName = "mnodes",
.hashSessions = TSDB_MAX_MNODES,
.hashSessions = TSDB_DEFAULT_MNODES_HASH_SIZE,
.maxRowSize = tsMnodeUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_INT,
@ -140,6 +163,7 @@ int32_t mgmtInitMnodes() {
void mgmtCleanupMnodes() {
sdbCloseTable(tsMnodeSdb);
mgmtMnodeDestroyLock();
}
int32_t mgmtGetMnodesNum() {
@ -177,50 +201,65 @@ char *mgmtGetMnodeRoleStr(int32_t role) {
}
}
void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
void *pIter = NULL;
while (1) {
SMnodeObj *pMnode = NULL;
pIter = mgmtGetNextMnode(pIter, &pMnode);
if (pMnode == NULL) break;
void mgmtUpdateMnodeIpSet() {
SRpcIpSet *ipSet = &tsMnodeRpcIpSet;
SDMMnodeInfos *mnodes = &tsMnodeInfos;
strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn);
ipSet->port[ipSet->numOfIps] = htons(pMnode->pDnode->dnodePort);
mPrint("update mnodes ipset, numOfIps:%d ", mgmtGetMnodesNum());
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
ipSet->inUse = ipSet->numOfIps;
}
mgmtMnodeWrLock();
ipSet->numOfIps++;
mgmtDecMnodeRef(pMnode);
}
sdbFreeIter(pIter);
}
memset(ipSet, 0, sizeof(tsMnodeRpcIpSet));
memset(mnodes, 0, sizeof(SDMMnodeInfos));
void mgmtGetMnodeInfos(void *param) {
SDMMnodeInfos *mnodes = param;
mnodes->inUse = 0;
int32_t index = 0;
void *pIter = NULL;
void * pIter = NULL;
while (1) {
SMnodeObj *pMnode = NULL;
pIter = mgmtGetNextMnode(pIter, &pMnode);
if (pMnode == NULL) break;
mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
strcpy(mnodes->nodeInfos[index].nodeEp, pMnode->pDnode->dnodeEp);
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
mnodes->inUse = index;
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
if (pDnode != NULL) {
strcpy(ipSet->fqdn[ipSet->numOfIps], pDnode->dnodeFqdn);
ipSet->port[ipSet->numOfIps] = htons(pDnode->dnodePort);
mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
strcpy(mnodes->nodeInfos[index].nodeEp, pDnode->dnodeEp);
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
ipSet->inUse = ipSet->numOfIps;
mnodes->inUse = index;
}
mPrint("mnode:%d, ep:%s %s", index, pDnode->dnodeEp,
pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : "");
ipSet->numOfIps++;
index++;
}
index++;
mgmtDecDnodeRef(pDnode);
mgmtDecMnodeRef(pMnode);
}
sdbFreeIter(pIter);
mnodes->nodeNum = index;
sdbFreeIter(pIter);
mgmtMnodeUnLock();
}
void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
mgmtMnodeRdLock();
*ipSet = tsMnodeRpcIpSet;
mgmtMnodeUnLock();
}
void mgmtGetMnodeInfos(void *mnodeInfos) {
mgmtMnodeRdLock();
*(SDMMnodeInfos *)mnodeInfos = tsMnodeInfos;
mgmtMnodeUnLock();
}
int32_t mgmtAddMnode(int32_t dnodeId) {
@ -240,6 +279,8 @@ int32_t mgmtAddMnode(int32_t dnodeId) {
code = TSDB_CODE_SDB_ERROR;
}
mgmtUpdateMnodeIpSet();
return code;
}
@ -250,6 +291,8 @@ void mgmtDropMnodeLocal(int32_t dnodeId) {
sdbDeleteRow(&oper);
mgmtDecMnodeRef(pMnode);
}
mgmtUpdateMnodeIpSet();
}
int32_t mgmtDropMnode(int32_t dnodeId) {
@ -270,6 +313,9 @@ int32_t mgmtDropMnode(int32_t dnodeId) {
}
sdbDecRef(tsMnodeSdb, pMnode);
mgmtUpdateMnodeIpSet();
return code;
}
@ -343,7 +389,15 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pMnode->pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
if (pDnode != NULL) {
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
} else {
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, "invalid ep", pShow->bytes[cols] - VARSTR_HEADER_SIZE);
}
mgmtDecDnodeRef(pDnode);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

24
src/mnode/src/mgmtSdb.c
View File

@ -28,6 +28,7 @@
#include "mgmtDef.h"
#include "mgmtInt.h"
#include "mgmtMnode.h"
#include "mgmtDnode.h"
#include "mgmtSdb.h"
typedef enum {
@ -196,6 +197,8 @@ void sdbUpdateMnodeRoles() {
mgmtDecMnodeRef(pMnode);
}
}
mgmtUpdateMnodeIpSet();
}
static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion) {
@ -227,7 +230,7 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
static int32_t sdbForwardToPeer(SWalHead *pHead) {
if (tsSdbObj.sync == NULL) return TSDB_CODE_SUCCESS;
int32_t code = syncForwardToPeer(tsSdbObj.sync, pHead, (void*)pHead->version);
int32_t code = syncForwardToPeer(tsSdbObj.sync, pHead, (void*)pHead->version, TAOS_QTYPE_RPC);
if (code > 0) {
sdbTrace("forward request is sent, version:%" PRIu64 ", code:%d", pHead->version, code);
sem_wait(&tsSdbObj.sem);
@ -257,10 +260,15 @@ void sdbUpdateSync() {
if (pMnode == NULL) break;
syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId;
syncCfg.nodeInfo[index].nodePort = pMnode->pDnode->dnodePort + TSDB_PORT_SYNC;
strcpy(syncCfg.nodeInfo[index].nodeFqdn, pMnode->pDnode->dnodeEp);
index++;
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
if (pDnode != NULL) {
syncCfg.nodeInfo[index].nodePort = pDnode->dnodePort + TSDB_PORT_SYNC;
strcpy(syncCfg.nodeInfo[index].nodeFqdn, pDnode->dnodeEp);
index++;
}
mgmtDecDnodeRef(pDnode);
mgmtDecMnodeRef(pMnode);
}
sdbFreeIter(pIter);
@ -352,7 +360,7 @@ void sdbIncRef(void *handle, void *pObj) {
int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos);
atomic_add_fetch_32(pRefCount, 1);
if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) {
sdbTrace("table:%s, add ref to record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
sdbTrace("add ref to table:%s record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
}
}
@ -363,7 +371,7 @@ void sdbDecRef(void *handle, void *pObj) {
int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos);
int32_t refCount = atomic_sub_fetch_32(pRefCount, 1);
if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) {
sdbTrace("table:%s, def ref of record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
sdbTrace("def ref of table:%s record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
}
int8_t *updateEnd = pObj + pTable->refCountPos - 1;
@ -442,8 +450,8 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
pthread_mutex_unlock(&pTable->mutex);
sdbTrace("table:%s, insert record:%s to hash, numOfRows:%d version:%" PRIu64, pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
sdbTrace("table:%s, insert record:%s to hash, rowSize:%d vnumOfRows:%d version:%" PRIu64, pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion());
(*pTable->insertFp)(pOper);
return TSDB_CODE_SUCCESS;

4
src/mnode/src/mgmtShell.c
View File

@ -119,7 +119,7 @@ static void mgmtDoDealyedAddToShellQueue(void *param, void *tmrId) {
void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg) {
void *unUsed = NULL;
taosTmrReset(mgmtDoDealyedAddToShellQueue, 1000, queuedMsg, tsMgmtTmr, &unUsed);
taosTmrReset(mgmtDoDealyedAddToShellQueue, 300, queuedMsg, tsMgmtTmr, &unUsed);
}
void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
@ -325,6 +325,8 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
return;
}
pHBRsp->onlineDnodes = htonl(mgmtGetOnlinDnodesNum());
pHBRsp->totalDnodes = htonl(mgmtGetDnodesNum());
mgmtGetMnodeIpSet(&pHBRsp->ipList);
/*

133
src/mnode/src/mgmtTable.c
View File

@ -269,7 +269,6 @@ static int32_t mgmtChildTableActionRestored() {
SChildTableObj *pTable = NULL;
while (1) {
mgmtDecTableRef(pTable);
pIter = mgmtGetNextChildTable(pIter, &pTable);
if (pTable == NULL) break;
@ -278,6 +277,7 @@ static int32_t mgmtChildTableActionRestored() {
mError("ctable:%s, failed to get db, discard it", pTable->info.tableId);
SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
sdbDeleteRow(&desc);
mgmtDecTableRef(pTable);
continue;
}
mgmtDecDbRef(pDb);
@ -288,6 +288,7 @@ static int32_t mgmtChildTableActionRestored() {
pTable->vgId = 0;
SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
sdbDeleteRow(&desc);
mgmtDecTableRef(pTable);
continue;
}
mgmtDecVgroupRef(pVgroup);
@ -298,6 +299,7 @@ static int32_t mgmtChildTableActionRestored() {
pTable->vgId = 0;
SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
sdbDeleteRow(&desc);
mgmtDecTableRef(pTable);
continue;
}
@ -306,6 +308,7 @@ static int32_t mgmtChildTableActionRestored() {
pTable->vgId = 0;
SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
sdbDeleteRow(&desc);
mgmtDecTableRef(pTable);
continue;
}
@ -316,10 +319,13 @@ static int32_t mgmtChildTableActionRestored() {
pTable->vgId = 0;
SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
sdbDeleteRow(&desc);
mgmtDecTableRef(pTable);
continue;
}
mgmtDecTableRef(pSuperTable);
}
mgmtDecTableRef(pTable);
}
sdbFreeIter(pIter);
@ -334,7 +340,7 @@ static int32_t mgmtInitChildTables() {
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_CTABLE,
.tableName = "ctables",
.hashSessions = tsMaxTables,
.hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE,
.maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN + TSDB_CQ_SQL_SIZE,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_VAR_STRING,
@ -501,7 +507,7 @@ static int32_t mgmtInitSuperTables() {
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_STABLE,
.tableName = "stables",
.hashSessions = TSDB_MAX_SUPER_TABLES,
.hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE,
.maxRowSize = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_VAR_STRING,
@ -1133,22 +1139,23 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
prefixLen = strlen(prefix);
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
char stableName[TSDB_TABLE_NAME_LEN] = {0};
char stableName[TSDB_TABLE_NAME_LEN + 1] = {0};
while (numOfRows < rows) {
mgmtDecTableRef(pTable);
pShow->pIter = mgmtGetNextSuperTable(pShow->pIter, &pTable);
if (pTable == NULL) break;
if (strncmp(pTable->info.tableId, prefix, prefixLen)) {
mgmtDecTableRef(pTable);
continue;
}
memset(stableName, 0, tListLen(stableName));
mgmtExtractTableName(pTable->info.tableId, stableName);
if (pShow->payloadLen > 0 &&
patternMatch(pShow->payload, stableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH)
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, stableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) {
mgmtDecTableRef(pTable);
continue;
}
cols = 0;
@ -1178,6 +1185,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
cols++;
numOfRows++;
mgmtDecTableRef(pTable);
}
pShow->numOfReads += numOfRows;
@ -1252,7 +1260,7 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) {
pMeta->contLen = htons(pMeta->contLen);
rpcSendResponse(&rpcRsp);
mTrace("stable:%%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid);
mTrace("stable:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid);
}
static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
@ -1297,7 +1305,7 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
if (pDnode == NULL) break;
strncpy(pVgroupInfo->vgroups[vgSize].ipAddr[vn].fqdn, pDnode->dnodeFqdn, tListLen(pDnode->dnodeFqdn));
pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(tsDnodeShellPort);
pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(pDnode->dnodePort);
pVgroupInfo->vgroups[vgSize].numOfIps++;
}
@ -1326,13 +1334,13 @@ static void mgmtProcessDropSuperTableRsp(SRpcMsg *rpcMsg) {
}
static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) {
char * pTagData = NULL;
STagData * pTagData = NULL;
int32_t tagDataLen = 0;
int32_t totalCols = 0;
int32_t contLen = 0;
if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) {
pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1;
tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1;
pTagData = (STagData*)pMsg->schema;
tagDataLen = ntohl(pTagData->dataLen);
totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags;
contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen + pTable->sqlLen;
} else {
@ -1385,7 +1393,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb
}
if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) {
memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen);
memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData->data, tagDataLen);
memcpy(pCreate->data + totalCols * sizeof(SSchema) + tagDataLen, pTable->sql, pTable->sqlLen);
}
@ -1412,10 +1420,10 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj
pTable->vgId = pVgroup->vgId;
if (pTable->info.type == TSDB_CHILD_TABLE) {
char *pTagData = (char *) pCreate->schema; // it is a tag key
SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData);
STagData *pTagData = (STagData *) pCreate->schema; // it is a tag key
SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData->name);
if (pSuperTable == NULL) {
mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData);
mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData->name);
free(pTable);
terrno = TSDB_CODE_INVALID_TABLE;
return NULL;
@ -1475,7 +1483,7 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj
return NULL;
}
mTrace("table:%s, create table in vgroup, id:%d, uid:%" PRIu64 , pTable->info.tableId, pTable->sid, pTable->uid);
mTrace("table:%s, create table in vgroup:%d, id:%d, uid:%" PRIu64 , pTable->info.tableId, pVgroup->vgId, pTable->sid, pTable->uid);
return pTable;
}
@ -1530,7 +1538,7 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
newMsg->ahandle = pMsg->pTable;
newMsg->maxRetry = 5;
newMsg->maxRetry = 10;
SRpcMsg rpcMsg = {
.handle = newMsg,
.pCont = pMDCreate,
@ -1734,7 +1742,9 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) {
SCMTableInfoMsg *pInfo = pMsg->pCont;
int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData);
STagData* pTag = (STagData*)pInfo->tags;
int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + ntohl(pTag->dataLen);
SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
if (pCreateMsg == NULL) {
mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId);
@ -1748,18 +1758,13 @@ static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) {
pCreateMsg->getMeta = 1;
pCreateMsg->contLen = htonl(contLen);
contLen = sizeof(STagData);
if (contLen > pMsg->contLen - sizeof(SCMTableInfoMsg)) {
contLen = pMsg->contLen - sizeof(SCMTableInfoMsg);
}
memcpy(pCreateMsg->schema, pInfo->tags, contLen);
memcpy(pCreateMsg->schema, pInfo->tags, contLen - sizeof(SCMCreateTableMsg));
SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
pMsg->pCont = newMsg->pCont;
newMsg->msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE;
newMsg->pCont = pCreateMsg;
mTrace("table:%s, start to create on demand", pInfo->tableId);
mTrace("table:%s, start to create on demand, stable:%s", pInfo->tableId, pInfo->tags);
mgmtAddToShellQueue(newMsg);
}
@ -1782,6 +1787,34 @@ static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) {
rpcSendResponse(&rpcRsp);
}
void mgmtDropAllChildTablesInVgroups(SVgObj *pVgroup) {
void * pIter = NULL;
int32_t numOfTables = 0;
SChildTableObj *pTable = NULL;
mPrint("vgId:%d, all child tables will be dropped from sdb", pVgroup->vgId);
while (1) {
pIter = mgmtGetNextChildTable(pIter, &pTable);
if (pTable == NULL) break;
if (pTable->vgId == pVgroup->vgId) {
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsChildTableSdb,
.pObj = pTable,
};
sdbDeleteRow(&oper);
numOfTables++;
}
mgmtDecTableRef(pTable);
}
sdbFreeIter(pIter);
mPrint("vgId:%d, all child tables is dropped from sdb", pVgroup->vgId);
}
void mgmtDropAllChildTables(SDbObj *pDropDb) {
void * pIter = NULL;
int32_t numOfTables = 0;
@ -1991,7 +2024,7 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) {
SCMMultiTableInfoMsg *pInfo = pMsg->pCont;
pInfo->numOfTables = htonl(pInfo->numOfTables);
int32_t totalMallocLen = 4*1024*1024; // first malloc 4 MB, subsequent reallocation as twice
int32_t totalMallocLen = 4 * 1024 * 1024; // first malloc 4 MB, subsequent reallocation as twice
SMultiTableMeta *pMultiMeta = rpcMallocCont(totalMallocLen);
if (pMultiMeta == NULL) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
@ -2001,26 +2034,30 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) {
pMultiMeta->contLen = sizeof(SMultiTableMeta);
pMultiMeta->numOfTables = 0;
for (int t = 0; t < pInfo->numOfTables; ++t) {
char *tableId = (char*)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN);
for (int32_t t = 0; t < pInfo->numOfTables; ++t) {
char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN + 1);
SChildTableObj *pTable = mgmtGetChildTable(tableId);
if (pTable == NULL) continue;
if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(tableId);
if (pMsg->pDb == NULL) continue;
if (pMsg->pDb == NULL) {
mgmtDecTableRef(pTable);
continue;
}
int availLen = totalMallocLen - pMultiMeta->contLen;
if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)) {
//TODO realloc
//totalMallocLen *= 2;
//pMultiMeta = rpcReMalloc(pMultiMeta, totalMallocLen);
//if (pMultiMeta == NULL) {
/// rpcSendResponse(ahandle, TSDB_CODE_SERV_OUT_OF_MEMORY, NULL, 0);
// return TSDB_CODE_SERV_OUT_OF_MEMORY;
//} else {
// t--;
// continue;
//}
totalMallocLen *= 2;
pMultiMeta = rpcReallocCont(pMultiMeta, totalMallocLen);
if (pMultiMeta == NULL) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
mgmtDecTableRef(pTable);
return;
} else {
t--;
mgmtDecTableRef(pTable);
continue;
}
}
STableMetaMsg *pMeta = (STableMetaMsg *)(pMultiMeta->metas + pMultiMeta->contLen);
@ -2029,6 +2066,8 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) {
pMultiMeta->numOfTables ++;
pMultiMeta->contLen += pMeta->contLen;
}
mgmtDecTableRef(pTable);
}
SRpcMsg rpcRsp = {0};
@ -2106,22 +2145,22 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows,
int32_t prefixLen = strlen(prefix);
while (numOfRows < rows) {
mgmtDecTableRef(pTable);
pShow->pIter = mgmtGetNextChildTable(pShow->pIter, &pTable);
if (pTable == NULL) break;
// not belong to current db
if (strncmp(pTable->info.tableId, prefix, prefixLen)) {
mgmtDecTableRef(pTable);
continue;
}
char tableName[TSDB_TABLE_NAME_LEN] = {0};
char tableName[TSDB_TABLE_NAME_LEN + 1] = {0};
// pattern compare for table name
mgmtExtractTableName(pTable->info.tableId, tableName);
if (pShow->payloadLen > 0 &&
patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) {
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) {
mgmtDecTableRef(pTable);
continue;
}
@ -2156,6 +2195,7 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows,
cols++;
numOfRows++;
mgmtDecTableRef(pTable);
}
pShow->numOfReads += numOfRows;
@ -2192,6 +2232,8 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
}
pAlter->type = htons(pAlter->type);
pAlter->numOfCols = htons(pAlter->numOfCols);
pAlter->tagValLen = htonl(pAlter->tagValLen);
if (pAlter->numOfCols > 2) {
mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols);
@ -2223,7 +2265,8 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
mTrace("table:%s, start to alter ctable", pAlter->tableId);
SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) {
code = mgmtModifyChildTableTagValue(pTable, pAlter->schema[0].name, pAlter->tagVal);
char *tagVal = (char*)(pAlter->schema + pAlter->numOfCols);
code = mgmtModifyChildTableTagValue(pTable, pAlter->schema[0].name, tagVal);
} else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) {
code = mgmtAddNormalTableColumn(pMsg->pDb, pTable, pAlter->schema, 1);
} else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) {

2
src/mnode/src/mgmtUser.c
View File

@ -117,7 +117,7 @@ int32_t mgmtInitUsers() {
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_USER,
.tableName = "users",
.hashSessions = TSDB_MAX_USERS,
.hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE,
.maxRowSize = tsUserUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_STRING,

57
src/mnode/src/mgmtVgroup.c
View File

@ -121,6 +121,20 @@ static int32_t mgmtVgroupActionDelete(SSdbOper *pOper) {
return TSDB_CODE_SUCCESS;
}
static void mgmtVgroupUpdateIdPool(SVgObj *pVgroup) {
int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool);
SDbObj *pDb = pVgroup->pDb;
if (pDb != NULL) {
if (pDb->cfg.maxTables != oldTables) {
mPrint("vgId:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
memset(pVgroup->tableList + oldTables, 0, (pDb->cfg.maxTables - oldTables) * sizeof(SChildTableObj **));
}
}
}
static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
SVgObj *pNew = pOper->pObj;
SVgObj *pVgroup = mgmtGetVgroup(pNew->vgId);
@ -146,20 +160,11 @@ static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
}
}
int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool);
SDbObj *pDb = pVgroup->pDb;
if (pDb != NULL) {
if (pDb->cfg.maxTables != oldTables) {
mPrint("vgId:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
}
}
mgmtVgroupUpdateIdPool(pVgroup);
mgmtDecVgroupRef(pVgroup);
mTrace("vgId:%d, is updated, numOfVnode:%d tables:%d", pVgroup->vgId, pVgroup->numOfVnodes, pDb == NULL ? 0 : pDb->cfg.maxTables);
mTrace("vgId:%d, is updated, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes);
return TSDB_CODE_SUCCESS;
}
@ -196,7 +201,7 @@ int32_t mgmtInitVgroups() {
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_VGROUP,
.tableName = "vgroups",
.hashSessions = TSDB_MAX_VGROUPS,
.hashSessions = TSDB_DEFAULT_VGROUPS_HASH_SIZE,
.maxRowSize = tsVgUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_AUTO,
@ -742,11 +747,14 @@ void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
SVgObj *pVgroup = NULL;
int32_t numOfVgroups = 0;
mPrint("dnode:%d, all vgroups will be dropped from sdb", pDropDnode->dnodeId);
while (1) {
pIter = mgmtGetNextVgroup(pIter, &pVgroup);
if (pVgroup == NULL) break;
if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) {
mgmtDropAllChildTablesInVgroups(pVgroup);
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsVgroupSdb,
@ -754,12 +762,35 @@ void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
};
sdbDeleteRow(&oper);
numOfVgroups++;
continue;
}
mgmtDecVgroupRef(pVgroup);
}
sdbFreeIter(pIter);
mPrint("dnode:%d, all vgroups is dropped from sdb", pDropDnode->dnodeId);
}
void mgmtUpdateAllDbVgroups(SDbObj *pAlterDb) {
void * pIter = NULL;
SVgObj *pVgroup = NULL;
mPrint("db:%s, all vgroups will be update in sdb", pAlterDb->name);
while (1) {
pIter = mgmtGetNextVgroup(pIter, &pVgroup);
if (pVgroup == NULL) break;
if (pVgroup->pDb == pAlterDb) {
mgmtVgroupUpdateIdPool(pVgroup);
}
mgmtDecVgroupRef(pVgroup);
}
sdbFreeIter(pIter);
mPrint("db:%s, all vgroups is updated in sdb", pAlterDb->name);
}
void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {

1
src/os/linux/inc/os.h
View File

@ -53,6 +53,7 @@ extern "C" {
#include <string.h>
#include <strings.h>
#include <sys/epoll.h>
#include <sys/eventfd.h>
#include <sys/file.h>
#include <sys/ioctl.h>
#include <sys/mman.h>

10
src/os/linux/src/linuxPlatform.c
View File

@ -148,6 +148,10 @@ static void taosDeleteTimer(void *tharg) {
timer_delete(*pTimer);
}
static pthread_t timerThread;
static timer_t timerId;
static volatile bool stopTimer = false;
void *taosProcessAlarmSignal(void *tharg) {
// Block the signal
sigset_t sigset;
@ -156,7 +160,6 @@ void *taosProcessAlarmSignal(void *tharg) {
sigprocmask(SIG_BLOCK, &sigset, NULL);
void (*callback)(int) = tharg;
static timer_t timerId;
struct sigevent sevent = {{0}};
#ifdef _ALPINE
@ -187,7 +190,7 @@ void *taosProcessAlarmSignal(void *tharg) {
}
int signo;
while (1) {
while (!stopTimer) {
if (sigwait(&sigset, &signo)) {
uError("Failed to wait signal: number %d", signo);
continue;
@ -202,7 +205,6 @@ void *taosProcessAlarmSignal(void *tharg) {
return NULL;
}
static pthread_t timerThread;
int taosInitTimer(void (*callback)(int), int ms) {
pthread_attr_t tattr;
@ -217,7 +219,7 @@ int taosInitTimer(void (*callback)(int), int ms) {
}
void taosUninitTimer() {
pthread_cancel(timerThread);
stopTimer = true;
pthread_join(timerThread, NULL);
}

1
src/plugins/CMakeLists.txt
View File

@ -3,3 +3,4 @@ PROJECT(TDengine)
ADD_SUBDIRECTORY(monitor)
ADD_SUBDIRECTORY(http)
ADD_SUBDIRECTORY(mqtt)

5
src/plugins/http/inc/httpHandle.h
View File

@ -199,7 +199,7 @@ typedef struct HttpThread {
pthread_t thread;
HttpContext * pHead;
pthread_mutex_t threadMutex;
pthread_cond_t fdReady;
bool stop;
int pollFd;
int numOfFds;
int threadId;
@ -212,6 +212,8 @@ typedef struct HttpServer {
char label[HTTP_LABEL_SIZE];
uint32_t serverIp;
uint16_t serverPort;
bool online;
int fd;
int cacheContext;
int sessionExpire;
int numOfThreads;
@ -226,7 +228,6 @@ typedef struct HttpServer {
bool (*processData)(HttpContext *pContext);
int requestNum;
void *timerHandle;
bool online;
} HttpServer;
// http util method

90
src/plugins/http/src/httpServer.c
View File

@ -258,28 +258,47 @@ void httpCloseContextByServerForExpired(void *param, void *tmrId) {
httpCloseContextByServer(pContext->pThread, pContext);
}
void httpCleanUpConnect(HttpServer *pServer) {
int i;
HttpThread *pThread;
static void httpStopThread(HttpThread* pThread) {
pThread->stop = true;
// signal the thread to stop, try graceful method first,
// and use pthread_cancel when failed
struct epoll_event event = { .events = EPOLLIN };
eventfd_t fd = eventfd(1, 0);
if (fd == -1) {
httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno));
pthread_cancel(pThread->thread);
} else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno));
pthread_cancel(pThread->thread);
}
pthread_join(pThread->thread, NULL);
if (fd != -1) {
close(fd);
}
close(pThread->pollFd);
pthread_mutex_destroy(&(pThread->threadMutex));
//while (pThread->pHead) {
// httpCleanUpContext(pThread->pHead, 0);
//}
}
void httpCleanUpConnect(HttpServer *pServer) {
if (pServer == NULL) return;
pthread_cancel(pServer->thread);
shutdown(pServer->fd, SHUT_RD);
pthread_join(pServer->thread, NULL);
for (i = 0; i < pServer->numOfThreads; ++i) {
pThread = pServer->pThreads + i;
if (pThread == NULL) continue;
//taosCloseSocket(pThread->pollFd);
//while (pThread->pHead) {
// httpCleanUpContext(pThread->pHead, 0);
//}
pthread_cancel(pThread->thread);
pthread_join(pThread->thread, NULL);
pthread_cond_destroy(&(pThread->fdReady));
pthread_mutex_destroy(&(pThread->threadMutex));
for (int i = 0; i < pServer->numOfThreads; ++i) {
HttpThread* pThread = pServer->pThreads + i;
if (pThread != NULL) {
httpStopThread(pThread);
}
}
tfree(pServer->pThreads);
@ -412,15 +431,13 @@ void httpProcessHttpData(void *param) {
pthread_sigmask(SIG_SETMASK, &set, NULL);
while (1) {
pthread_mutex_lock(&pThread->threadMutex);
if (pThread->numOfFds < 1) {
pthread_cond_wait(&pThread->fdReady, &pThread->threadMutex);
}
pthread_mutex_unlock(&pThread->threadMutex);
struct epoll_event events[HTTP_MAX_EVENTS];
//-1 means uncertainty, 0-nowait, 1-wait 1 ms, set it from -1 to 1
fdNum = epoll_wait(pThread->pollFd, events, HTTP_MAX_EVENTS, 1);
if (pThread->stop) {
httpTrace("%p, http thread get stop event, exiting...", pThread);
break;
}
if (fdNum <= 0) continue;
for (int i = 0; i < fdNum; ++i) {
@ -485,10 +502,9 @@ void httpProcessHttpData(void *param) {
}
}
void httpAcceptHttpConnection(void *arg) {
void* httpAcceptHttpConnection(void *arg) {
int connFd = -1;
struct sockaddr_in clientAddr;
int sockFd;
int threadId = 0;
HttpThread * pThread;
HttpServer * pServer;
@ -502,12 +518,12 @@ void httpAcceptHttpConnection(void *arg) {
sigaddset(&set, SIGPIPE);
pthread_sigmask(SIG_SETMASK, &set, NULL);
sockFd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort);
pServer->fd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort);
if (sockFd < 0) {
if (pServer->fd < 0) {
httpError("http server:%s, failed to open http socket, ip:%s:%u error:%s", pServer->label, taosIpStr(pServer->serverIp),
pServer->serverPort, strerror(errno));
return;
return NULL;
} else {
httpPrint("http service init success at %u", pServer->serverPort);
pServer->online = true;
@ -515,9 +531,12 @@ void httpAcceptHttpConnection(void *arg) {
while (1) {
socklen_t addrlen = sizeof(clientAddr);
connFd = (int)accept(sockFd, (struct sockaddr *)&clientAddr, &addrlen);
if (connFd < 3) {
connFd = (int)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen);
if (connFd == -1) {
if (errno == EINVAL) {
httpTrace("%s HTTP server socket was shutdown, exiting...", pServer->label);
break;
}
httpError("http server:%s, accept connect failure, errno:%d, reason:%s", pServer->label, errno, strerror(errno));
continue;
}
@ -579,7 +598,6 @@ void httpAcceptHttpConnection(void *arg) {
pThread->pHead = pContext;
pThread->numOfFds++;
pthread_cond_signal(&pThread->fdReady);
pthread_mutex_unlock(&(pThread->threadMutex));
@ -587,6 +605,9 @@ void httpAcceptHttpConnection(void *arg) {
threadId++;
threadId = threadId % pServer->numOfThreads;
}
close(pServer->fd);
return NULL;
}
bool httpInitConnect(HttpServer *pServer) {
@ -612,11 +633,6 @@ bool httpInitConnect(HttpServer *pServer) {
return false;
}
if (pthread_cond_init(&(pThread->fdReady), NULL) != 0) {
httpError("http thread:%s, init HTTP condition variable failed, reason:%s", pThread->label, strerror(errno));
return false;
}
pThread->pollFd = epoll_create(HTTP_MAX_EVENTS); // size does not matter
if (pThread->pollFd < 0) {
httpError("http thread:%s, failed to create HTTP epoll", pThread->label);

22
src/plugins/mqtt/CMakeLists.txt Normal file
View File

@ -0,0 +1,22 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(mqtt ${SRC})
TARGET_LINK_LIBRARIES(mqtt taos_static z)
IF (TD_ADMIN)
TARGET_LINK_LIBRARIES(mqtt admin)
ENDIF ()
ENDIF ()

42
src/plugins/mqtt/inc/mqttLog.h Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MQTT_LOG_H
#define TDENGINE_MQTT_LOG_H
#include "tlog.h"
extern int32_t mqttDebugFlag;
#define mqttError(...) \
if (mqttDebugFlag & DEBUG_ERROR) { \
taosPrintLog("ERROR MQT ", 255, __VA_ARGS__); \
}
#define mqttWarn(...) \
if ( mqttDebugFlag & DEBUG_WARN) { \
taosPrintLog("WARN MQT ", mqttDebugFlag, __VA_ARGS__); \
}
#define mqttTrace(...) \
if ( mqttDebugFlag & DEBUG_TRACE) { \
taosPrintLog("MQT ", mqttDebugFlag, __VA_ARGS__); \
}
#define mqttDump(...) \
if ( mqttDebugFlag & DEBUG_TRACE) { \
taosPrintLongString("MQT ", mqttDebugFlag, __VA_ARGS__); \
}
#define mqttPrint(...) \
{ taosPrintLog("MQT ", 255, __VA_ARGS__); }
#endif

34
src/plugins/mqtt/inc/mqttSystem.h Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MQTT_SYSTEM_H
#define TDENGINE_MQTT_SYSTEM_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
int32_t mqttGetReqCount();
int32_t mqttInitSystem();
int32_t mqttStartSystem();
void mqttStopSystem();
void mqttCleanUpSystem();
#ifdef __cplusplus
}
#endif
#endif

43
src/plugins/mqtt/src/mqttSystem.c Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "mqttSystem.h"
#include "mqtt.h"
#include "mqttLog.h"
#include "os.h"
#include "taos.h"
#include "tglobal.h"
#include "tsocket.h"
#include "ttimer.h"
int32_t mqttGetReqCount() { return 0; }
int mqttInitSystem() {
mqttPrint("mqttInitSystem");
return 0;
}
int mqttStartSystem() {
mqttPrint("mqttStartSystem");
return 0;
}
void mqttStopSystem() {
mqttPrint("mqttStopSystem");
}
void mqttCleanUpSystem() {
mqttPrint("mqttCleanUpSystem");
}

24
src/query/inc/queryExecutor.h → src/query/inc/qExecutor.h
View File

@ -18,20 +18,20 @@
#include "os.h"
#include "hash.h"
#include "tsdb.h"
#include "qinterpolation.h"
#include "qfill.h"
#include "qresultBuf.h"
#include "qsqlparser.h"
#include "qtsbuf.h"
#include "taosdef.h"
#include "tref.h"
#include "tsqlfunction.h"
#include "tarray.h"
#include "tref.h"
#include "tsdb.h"
#include "tsqlfunction.h"
typedef struct SData {
int32_t num;
char data[];
} SData;
//typedef struct tFilePage {
// int64_t num;
// char data[];
//} tFilePage;
struct SColumnFilterElem;
typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
@ -129,7 +129,7 @@ typedef struct SQuery {
char slidingTimeUnit; // interval data type, used for daytime revise
int8_t precision;
int16_t numOfOutput;
int16_t interpoType;
int16_t fillType;
int16_t checkBuffer; // check if the buffer is full during scan each block
SLimitVal limit;
int32_t rowSize;
@ -139,11 +139,10 @@ typedef struct SQuery {
SColumnInfo* tagColList;
int32_t numOfFilterCols;
int64_t* defaultVal;
// TSKEY lastKey;
uint32_t status; // query status
SResultRec rec;
int32_t pos;
SData** sdata;
tFilePage** sdata;
STableQueryInfo* current;
SSingleColumnFilterInfo* pFilterInfo;
} SQuery;
@ -151,12 +150,11 @@ typedef struct SQuery {
typedef struct SQueryRuntimeEnv {
SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo
SQuery* pQuery;
SData** pInterpoBuf;
SQLFunctionCtx* pCtx;
int16_t numOfRowsPerPage;
int16_t offset[TSDB_MAX_COLUMNS];
uint16_t scanFlag; // denotes reversed scan of data or not
SInterpolationInfo interpoInfo;
SFillInfo* pFillInfo;
SWindowResInfo windowResInfo;
STSBuf* pTSBuf;
STSCursor cur;

0
src/query/inc/queryUtil.h → src/query/inc/qUtil.h
View File

2
src/query/inc/qextbuffer.h
View File

@ -68,7 +68,7 @@ typedef struct SExtFileInfo {
} SExtFileInfo;
typedef struct tFilePage {
uint64_t numOfElems;
uint64_t num;
char data[];
} tFilePage;

92
src/query/inc/qfill.h Normal file
View File

@ -0,0 +1,92 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QFILL_H
#define TDENGINE_QFILL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "taosdef.h"
#include "qextbuffer.h"
typedef struct {
STColumn col; // column info
int16_t functionId; // sql function id
int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN
union {int64_t i; double d;} defaultVal;
} SFillColInfo;
typedef struct SFillInfo {
TSKEY start; // start timestamp
TSKEY endKey; // endKey for fill
int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC]
int32_t fillType; // fill type
int32_t numOfRows; // number of rows in the input data block
int32_t rowIdx; // rowIdx
int32_t numOfTotal; // number of filled rows in one round
int32_t numOfCurrent; // number of filled rows in current results
int32_t numOfTags; // number of tags
int32_t numOfCols; // number of columns, including the tags columns
int32_t rowSize; // size of each row
char ** pTags; // tags value for current interpolation
int64_t slidingTime; // sliding value to determine the number of result for a given time window
char * prevValues; // previous row of data, to generate the interpolation results
char * nextValues; // next row of data
SFillColInfo* pFillCol; // column info for fill operations
char** pData; // original result data block involved in filling data
} SFillInfo;
typedef struct SPoint {
int64_t key;
void * val;
} SPoint;
int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, char timeUnit, int16_t precision);
SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity,
int32_t numOfCols, int64_t slidingTime, int32_t fillType, SFillColInfo* pFillCol);
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
void taosDestoryFillInfo(SFillInfo *pFillInfo);
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput);
void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput);
TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision);
int32_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows);
int32_t taosNumOfRemainRows(SFillInfo *pFillInfo);
int32_t taosDoInterpoResult(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfRows, int32_t outputRows, char** srcData);
int taosDoLinearInterpolation(int32_t type, SPoint *point1, SPoint *point2, SPoint *point);
void taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int64_t* outputRows, int32_t capacity);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_QFILL_H

94
src/query/inc/qinterpolation.h
View File

@ -1,94 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TINTERPOLATION_H
#define TDENGINE_TINTERPOLATION_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "taosdef.h"
#include "qextbuffer.h"
typedef struct SInterpolationInfo {
int64_t startTimestamp;
int32_t order; // order [asc/desc]
int32_t numOfRawDataInRows; // number of points in pQuery->sdata
int32_t rowIdx; // rowIdx in pQuery->sdata
int32_t numOfTotalInterpo; // number of interpolated rows in one round
int32_t numOfCurrentInterpo; // number of interpolated rows in current results
char * prevValues; // previous row of data
char * nextValues; // next row of data
int32_t numOfTags;
char ** pTags; // tags value for current interoplation
} SInterpolationInfo;
typedef struct SPoint {
int64_t key;
void * val;
} SPoint;
int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t timeRange, char intervalTimeUnit, int16_t precision);
void taosInitInterpoInfo(SInterpolationInfo *pInterpoInfo, int32_t order, int64_t startTimeStamp, int32_t numOfTags,
int32_t rowSize);
void taosDestoryInterpoInfo(SInterpolationInfo *pInterpoInfo);
void taosInterpoSetStartInfo(SInterpolationInfo *pInterpoInfo, int32_t numOfRawDataInRows, int32_t type);
TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int32_t timeInterval, int8_t intervalTimeUnit, int8_t precision);
/**
*
* @param pInterpoInfo
* @param pPrimaryKeyArray
* @param numOfRows
* @param nInterval
* @param ekey
* @param maxNumOfRows
* @return
*/
int32_t taosGetNumOfResultWithInterpo(SInterpolationInfo *pInterpoInfo, int64_t *pPrimaryKeyArray, int32_t numOfRows,
int64_t nInterval, int64_t ekey, int32_t maxNumOfRows);
int32_t taosGetNumOfResWithoutLimit(SInterpolationInfo *pInterpoInfo, int64_t *pPrimaryKeyArray,
int32_t numOfRawDataInRows, int64_t nInterval, int64_t ekey);
/**
*
* @param pInterpoInfo
* @return
*/
bool taosHasRemainsDataForInterpolation(SInterpolationInfo *pInterpoInfo);
int32_t taosNumOfRemainPoints(SInterpolationInfo *pInterpoInfo);
/**
*
*/
int32_t taosDoInterpoResult(SInterpolationInfo *pInterpoInfo, int16_t interpoType, tFilePage **data,
int32_t numOfRawDataInRows, int32_t outputRows, int64_t nInterval,
const int64_t *pPrimaryKeyArray, SColumnModel *pModel, char **srcData, int64_t *defaultVal,
const int32_t *functionIDs, int32_t bufSize);
int taosDoLinearInterpolation(int32_t type, SPoint *point1, SPoint *point2, SPoint *point);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TINTERPOLATION_H

25
src/query/inc/queryLog.h
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_RPC_LOG_H
#define TDENGINE_RPC_LOG_H
#ifndef TDENGINE_QUERY_LOG_H
#define TDENGINE_QUERY_LOG_H
#ifdef __cplusplus
extern "C" {
@ -24,22 +24,23 @@ extern "C" {
extern int32_t qDebugFlag;
#define qTrace(...) \
if (qDebugFlag & DEBUG_TRACE) { \
taosPrintLog("DND QRY ", qDebugFlag, __VA_ARGS__); \
#define qTrace(...) \
if (qDebugFlag & DEBUG_TRACE) { \
taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); \
}
#define qError(...) \
if (qDebugFlag & DEBUG_ERROR) { \
#define qError(...) \
if (qDebugFlag & DEBUG_ERROR) { \
taosPrintLog("ERROR QRY ", qDebugFlag, __VA_ARGS__); \
}
#define qWarn(...) \
if (qDebugFlag & DEBUG_WARN) { \
taosPrintLog("WARN QRY ", qDebugFlag, __VA_ARGS__); \
}
#define qWarn(...) \
if (qDebugFlag & DEBUG_WARN) { \
taosPrintLog("WARN QRY ", qDebugFlag, __VA_ARGS__); \
}
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_RPC_CACHE_H
#endif // TDENGINE_QUERY_CACHE_H

2
src/query/inc/tlosertree.h
View File

@ -32,7 +32,7 @@ typedef struct SLoserTreeNode {
typedef struct SLoserTreeInfo {
int32_t numOfEntries;
int32_t totalEntries;
__merge_compare_fn_t comparaFn;
__merge_compare_fn_t comparFn;
void * param;
SLoserTreeNode *pNode;

577
src/query/src/queryExecutor.c → src/query/src/qExecutor.c
View File

File diff suppressed because it is too large Load Diff

4
src/query/src/queryFilterFunc.c → src/query/src/qFilterFunc.c
View File

@ -16,10 +16,10 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "qExecutor.h"
#include "taosmsg.h"
#include "tsqlfunction.h"
#include "queryExecutor.h"
#include "tcompare.h"
#include "tsqlfunction.h"
bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval < pFilter->filterInfo.upperBndi);

13
src/query/src/queryUtil.c → src/query/src/qUtil.c
View File

@ -20,11 +20,11 @@
#include "qextbuffer.h"
#include "ttime.h"
#include "qinterpolation.h"
#include "qfill.h"
#include "ttime.h"
#include "queryExecutor.h"
#include "queryUtil.h"
#include "qExecutor.h"
#include "qUtil.h"
int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size,
int32_t threshold, int16_t type) {
@ -37,7 +37,8 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun
pWindowResInfo->hashList = taosHashInit(threshold, fn, false);
pWindowResInfo->curIndex = -1;
pWindowResInfo->size = 0;
pWindowResInfo->size = 0;
pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
// use the pointer arraylist
pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult));
@ -96,8 +97,8 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR
_hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type);
pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, false);
pWindowResInfo->startTime = 0;
pWindowResInfo->prevSKey = 0;
pWindowResInfo->startTime = TSKEY_INITIAL_VAL;
pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
}
void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {

42
src/query/src/qast.c
View File

@ -476,44 +476,6 @@ typedef struct {
SEndPoint* end;
} SQueryCond;
//static void setInitialValueForRangeQueryCondition(tSKipListQueryCond *q, int8_t type) {
// q->lowerBndRelOptr = TSDB_RELATION_GREATER;
// q->upperBndRelOptr = TSDB_RELATION_LESS;
//
// switch (type) {
// case TSDB_DATA_TYPE_BOOL:
// case TSDB_DATA_TYPE_TINYINT:
// case TSDB_DATA_TYPE_SMALLINT:
// case TSDB_DATA_TYPE_INT:
// case TSDB_DATA_TYPE_BIGINT: {
// q->upperBnd.nType = TSDB_DATA_TYPE_BIGINT;
// q->lowerBnd.nType = TSDB_DATA_TYPE_BIGINT;
//
// q->upperBnd.i64Key = INT64_MAX;
// q->lowerBnd.i64Key = INT64_MIN;
// break;
// };
// case TSDB_DATA_TYPE_FLOAT:
// case TSDB_DATA_TYPE_DOUBLE: {
// q->upperBnd.nType = TSDB_DATA_TYPE_DOUBLE;
// q->lowerBnd.nType = TSDB_DATA_TYPE_DOUBLE;
// q->upperBnd.dKey = DBL_MAX;
// q->lowerBnd.dKey = -DBL_MIN;
// break;
// };
// case TSDB_DATA_TYPE_NCHAR:
// case TSDB_DATA_TYPE_BINARY: {
// q->upperBnd.nType = type;
// q->upperBnd.pz = NULL;
// q->upperBnd.nLen = -1;
//
// q->lowerBnd.nType = type;
// q->lowerBnd.pz = NULL;
// q->lowerBnd.nLen = -1;
// }
// }
//}
// todo check for malloc failure
static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
int32_t optr = queryColInfo->optr;
@ -788,7 +750,6 @@ static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTravers
taosArrayCopy(pResult, array);
}
static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SExprTraverseSupp *param ) {
SSkipListIterator* iter = tSkipListCreateIter(pSkipList);
@ -834,8 +795,6 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
tSkipListDestroyIter(iter);
}
// post-root order traverse syntax tree
void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param) {
if (pExpr == NULL) {
@ -1100,7 +1059,6 @@ static char* exception_strdup(const char* str) {
return p;
}
static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
int32_t anchor = CLEANUP_GET_ANCHOR();

28
src/query/src/qextbuffer.c
View File

@ -136,7 +136,7 @@ static bool tExtMemBufferAlloc(tExtMemBuffer *pMemBuffer) {
}
item->pNext = NULL;
item->item.numOfElems = 0;
item->item.num = 0;
if (pMemBuffer->pTail != NULL) {
pMemBuffer->pTail->pNext = item;
@ -167,13 +167,13 @@ int16_t tExtMemBufferPut(tExtMemBuffer *pMemBuffer, void *data, int32_t numOfRow
pLast = pMemBuffer->pTail;
}
if (pLast->item.numOfElems + numOfRows <= pMemBuffer->numOfElemsPerPage) { // enough space for records
if (pLast->item.num + numOfRows <= pMemBuffer->numOfElemsPerPage) { // enough space for records
tColModelAppend(pMemBuffer->pColumnModel, &pLast->item, data, 0, numOfRows, numOfRows);
pMemBuffer->numOfElemsInBuffer += numOfRows;
pMemBuffer->numOfTotalElems += numOfRows;
} else {
int32_t numOfRemainEntries = pMemBuffer->numOfElemsPerPage - pLast->item.numOfElems;
int32_t numOfRemainEntries = pMemBuffer->numOfElemsPerPage - pLast->item.num;
tColModelAppend(pMemBuffer->pColumnModel, &pLast->item, data, 0, numOfRemainEntries, numOfRows);
pMemBuffer->numOfElemsInBuffer += numOfRemainEntries;
@ -271,7 +271,7 @@ bool tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) {
ret = false;
}
pMemBuffer->fileMeta.numOfElemsInFile += first->item.numOfElems;
pMemBuffer->fileMeta.numOfElemsInFile += first->item.num;
pMemBuffer->fileMeta.nFileSize += 1;
tFilePagesItem *ptmp = first;
@ -985,16 +985,16 @@ void tColModelDisplayEx(SColumnModel *pModel, void *pData, int32_t numOfRows, in
////////////////////////////////////////////////////////////////////////////////////////////
void tColModelCompact(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxElemsCapacity) {
if (inputBuffer->numOfElems == 0 || maxElemsCapacity == inputBuffer->numOfElems) {
if (inputBuffer->num == 0 || maxElemsCapacity == inputBuffer->num) {
return;
}
/* start from the second column */
for (int32_t i = 1; i < pModel->numOfCols; ++i) {
SSchemaEx* pSchemaEx = &pModel->pFields[i];
memmove(inputBuffer->data + pSchemaEx->offset * inputBuffer->numOfElems,
memmove(inputBuffer->data + pSchemaEx->offset * inputBuffer->num,
inputBuffer->data + pSchemaEx->offset * maxElemsCapacity,
pSchemaEx->field.bytes * inputBuffer->numOfElems);
pSchemaEx->field.bytes * inputBuffer->num);
}
}
@ -1009,13 +1009,13 @@ int16_t getColumnModelOffset(SColumnModel *pColumnModel, int32_t index) {
}
void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t blockCapacity, int32_t s, int32_t e) {
if (inputBuffer->numOfElems == 0 || (e - s + 1) <= 0) {
if (inputBuffer->num == 0 || (e - s + 1) <= 0) {
return;
}
int32_t removed = e - s + 1;
int32_t remain = inputBuffer->numOfElems - removed;
int32_t secPart = inputBuffer->numOfElems - e - 1;
int32_t remain = inputBuffer->num - removed;
int32_t secPart = inputBuffer->num - e - 1;
/* start from the second column */
for (int32_t i = 0; i < pModel->numOfCols; ++i) {
@ -1028,7 +1028,7 @@ void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t blockC
memmove(startPos, endPos, pSchema->bytes * secPart);
}
inputBuffer->numOfElems = remain;
inputBuffer->num = remain;
}
/*
@ -1040,16 +1040,16 @@ void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t blockC
*/
void tColModelAppend(SColumnModel *dstModel, tFilePage *dstPage, void *srcData, int32_t start, int32_t numOfRows,
int32_t srcCapacity) {
assert(dstPage->numOfElems + numOfRows <= dstModel->capacity);
assert(dstPage->num + numOfRows <= dstModel->capacity);
for (int32_t col = 0; col < dstModel->numOfCols; ++col) {
char *dst = COLMODEL_GET_VAL(dstPage->data, dstModel, dstModel->capacity, dstPage->numOfElems, col);
char *dst = COLMODEL_GET_VAL(dstPage->data, dstModel, dstModel->capacity, dstPage->num, col);
char *src = COLMODEL_GET_VAL((char *)srcData, dstModel, srcCapacity, start, col);
memmove(dst, src, dstModel->pFields[col].field.bytes * numOfRows);
}
dstPage->numOfElems += numOfRows;
dstPage->num += numOfRows;
}
tOrderDescriptor *tOrderDesCreate(const int32_t *orderColIdx, int32_t numOfOrderCols, SColumnModel *pModel,

470
src/query/src/qfill.c Normal file
View File

@ -0,0 +1,470 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "qfill.h"
#include "os.h"
#include "qextbuffer.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tsqlfunction.h"
#define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC)
int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, char timeUnit, int16_t precision) {
if (slidingTime == 0) {
return startTime;
}
if (timeUnit == 'a' || timeUnit == 'm' || timeUnit == 's' || timeUnit == 'h') {
return (startTime / slidingTime) * slidingTime;
} else {
/*
* here we revised the start time of day according to the local time zone,
* but in case of DST, the start time of one day need to be dynamically decided.
*
* TODO dynamically decide the start time of a day
*/
// todo refactor to extract function that is available for Linux/Windows/Mac platform
#if defined(WINDOWS) && _MSC_VER >= 1900
// see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019
int64_t timezone = _timezone;
int32_t daylight = _daylight;
char** tzname = _tzname;
#endif
int64_t t = (precision == TSDB_TIME_PRECISION_MILLI) ? MILLISECOND_PER_SECOND : MILLISECOND_PER_SECOND * 1000L;
int64_t revStartime = (startTime / slidingTime) * slidingTime + timezone * t;
int64_t revEndtime = revStartime + slidingTime - 1;
if (revEndtime < startTime) {
revStartime += slidingTime;
}
return revStartime;
}
}
SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int32_t fillType, SFillColInfo* pFillCol) {
if (fillType == TSDB_FILL_NONE) {
return NULL;
}
SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo));
taosResetFillInfo(pFillInfo, skey);
pFillInfo->order = order;
pFillInfo->fillType = fillType;
pFillInfo->pFillCol = pFillCol;
pFillInfo->numOfTags = numOfTags;
pFillInfo->numOfCols = numOfCols;
pFillInfo->slidingTime = slidingTime;
pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
int32_t rowsize = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
int32_t bytes = pFillInfo->pFillCol[i].col.bytes;
pFillInfo->pData[i] = calloc(1, sizeof(tFilePage) + bytes * capacity);
rowsize += bytes;
}
if (numOfTags > 0) {
pFillInfo->pTags = calloc(1, pFillInfo->numOfTags * POINTER_BYTES + rowsize);
}
pFillInfo->rowSize = rowsize;
return pFillInfo;
}
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
pFillInfo->start = startTimestamp;
pFillInfo->rowIdx = -1;
pFillInfo->numOfRows = 0;
pFillInfo->numOfCurrent = 0;
pFillInfo->numOfTotal = 0;
}
void taosDestoryFillInfo(SFillInfo* pFillInfo) {
if (pFillInfo == NULL) {
return;
}
tfree(pFillInfo->prevValues);
tfree(pFillInfo->nextValues);
tfree(pFillInfo->pTags);
tfree(pFillInfo);
}
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
if (pFillInfo->fillType == TSDB_FILL_NONE) {
return;
}
pFillInfo->rowIdx = 0;
pFillInfo->numOfRows = numOfRows;
pFillInfo->endKey = endKey;
}
void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput) {
// copy the data into source data buffer
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
memcpy(pFillInfo->pData[i], pInput[i]->data, pFillInfo->numOfRows * pFillInfo->pFillCol[i].col.bytes);
}
}
void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput) {
assert(pFillInfo->numOfRows == pInput->num);
for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* s = pInput->data + pCol->col.offset * pInput->num;
memcpy(pFillInfo->pData[i], s, pInput->num * pCol->col.bytes);
if (pCol->flag == TSDB_COL_TAG) { // copy the tag value
memcpy(pFillInfo->pTags[i], pFillInfo->pData[i], pCol->col.bytes);
}
}
}
TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision) {
if (order == TSDB_ORDER_ASC) {
return ekey;
} else {
return taosGetIntervalStartTimestamp(ekey, timeInterval, slidingTimeUnit, precision);
}
}
static int32_t taosGetTotalNumOfFilledRes(SFillInfo* pFillInfo, const TSKEY* tsArray, int32_t remain,
int64_t nInterval, int64_t ekey) {
if (remain > 0) { // still fill gap within current data block, not generating data after the result set.
TSKEY lastKey = tsArray[pFillInfo->numOfRows - 1];
int32_t total = (int32_t)(labs(lastKey - pFillInfo->start) / nInterval) + 1;
assert(total >= remain);
return total;
} else { // reach the end of data
if ((ekey < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) ||
(ekey > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) {
return 0;
} else {
return (int32_t)(labs(ekey - pFillInfo->start) / nInterval) + 1;
}
}
}
int32_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows) {
int32_t numOfRes = taosGetTotalNumOfFilledRes(pFillInfo, (int64_t*) pFillInfo->pData[0], numOfRows,
pFillInfo->slidingTime, ekey);
return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
}
int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) {
if (pFillInfo->rowIdx == -1 || pFillInfo->numOfRows == 0) {
return 0;
}
return FILL_IS_ASC_FILL(pFillInfo) ? (pFillInfo->numOfRows - pFillInfo->rowIdx)
: pFillInfo->rowIdx + 1;
}
// todo: refactor
static double linearInterpolationImpl(double v1, double v2, double k1, double k2, double k) {
return v1 + (v2 - v1) * (k - k1) / (k2 - k1);
}
int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) {
switch (type) {
case TSDB_DATA_TYPE_INT: {
*(int32_t*)point->val = linearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, point1->key,
point2->key, point->key);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
*(float*)point->val =
linearInterpolationImpl(*(float*)point1->val, *(float*)point2->val, point1->key, point2->key, point->key);
break;
};
case TSDB_DATA_TYPE_DOUBLE: {
*(double*)point->val =
linearInterpolationImpl(*(double*)point1->val, *(double*)point2->val, point1->key, point2->key, point->key);
break;
};
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT: {
*(int64_t*)point->val = linearInterpolationImpl(*(int64_t*)point1->val, *(int64_t*)point2->val, point1->key,
point2->key, point->key);
break;
};
case TSDB_DATA_TYPE_SMALLINT: {
*(int16_t*)point->val = linearInterpolationImpl(*(int16_t*)point1->val, *(int16_t*)point2->val, point1->key,
point2->key, point->key);
break;
};
case TSDB_DATA_TYPE_TINYINT: {
*(int8_t*)point->val =
linearInterpolationImpl(*(int8_t*)point1->val, *(int8_t*)point2->val, point1->key, point2->key, point->key);
break;
};
default: {
// TODO: Deal with interpolation with bool and strings and timestamp
return -1;
}
}
return 0;
}
static void setTagsValue(SFillInfo* pColInfo, tFilePage** data, char** pTags, int32_t start, int32_t num) {
for (int32_t j = 0, i = start; i < pColInfo->numOfCols + pColInfo->numOfTags; ++i, ++j) {
SFillColInfo* pCol = &pColInfo->pFillCol[i];
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, num);
assignVal(val1, pTags[j], pCol->col.bytes, pCol->col.type);
}
}
static void doInterpoResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* num, char** srcData,
int64_t ts, char** pTags, bool outOfBound) {
char** prevValues = &pFillInfo->prevValues;
char** nextValues = &pFillInfo->nextValues;
SPoint point1, point2, point;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
char* val = elePtrAt(data[0]->data, TSDB_KEYSIZE, *num);
*(TSKEY*) val = pFillInfo->start;
int32_t numOfValCols = pFillInfo->numOfCols - pFillInfo->numOfTags;
// set the other values
if (pFillInfo->fillType == TSDB_FILL_PREV) {
char* pInterpolationData = FILL_IS_ASC_FILL(pFillInfo) ? *prevValues : *nextValues;
if (pInterpolationData != NULL) {
for (int32_t i = 1; i < numOfValCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
if (isNull(pInterpolationData + pCol->col.offset, pCol->col.type)) {
setNull(val1, pCol->col.type, pCol->col.bytes);
} else {
assignVal(val1, pInterpolationData + pCol->col.offset, pCol->col.bytes, pCol->col.type);
}
}
} else { // no prev value yet, set the value for NULL
for (int32_t i = 1; i < numOfValCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
setNull(val1, pCol->col.type, pCol->col.bytes);
}
}
setTagsValue(pFillInfo, data, pTags, numOfValCols, *num);
} else if (pFillInfo->fillType == TSDB_FILL_LINEAR) {
// TODO : linear interpolation supports NULL value
if (*prevValues != NULL && !outOfBound) {
for (int32_t i = 1; i < numOfValCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
int16_t type = pCol->col.type;
int16_t bytes = pCol->col.bytes;
char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
setNull(val1, pCol->col.type, bytes);
continue;
}
point1 = (SPoint){.key = *(TSKEY*)(*prevValues), .val = *prevValues + pCol->col.offset};
point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->rowIdx * bytes};
point = (SPoint){.key = pFillInfo->start, .val = val1};
taosDoLinearInterpolation(type, &point1, &point2, &point);
}
setTagsValue(pFillInfo, data, pTags, numOfValCols, *num);
} else {
for (int32_t i = 1; i < numOfValCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
setNull(val1, pCol->col.type, pCol->col.bytes);
}
setTagsValue(pFillInfo, data, pTags, numOfValCols, *num);
}
} else { /* default value interpolation */
for (int32_t i = 1; i < numOfValCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
assignVal(val1, (char*)&pCol->defaultVal.i, pCol->col.bytes, pCol->col.type);
}
setTagsValue(pFillInfo, data, pTags, numOfValCols, *num);
}
pFillInfo->start += (pFillInfo->slidingTime * step);
pFillInfo->numOfCurrent++;
(*num) += 1;
}
static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** nextValues) {
if (*nextValues != NULL) {
return;
}
*nextValues = calloc(1, pFillInfo->rowSize);
for (int i = 1; i < pFillInfo->numOfCols; i++) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
setNull(*nextValues + pCol->col.offset, pCol->col.type, pCol->col.bytes);
}
}
int32_t taosDoInterpoResult(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfRows, int32_t outputRows, char** srcData) {
int32_t num = 0;
pFillInfo->numOfCurrent = 0;
char** prevValues = &pFillInfo->prevValues;
char** nextValues = &pFillInfo->nextValues;
int32_t numOfTags = pFillInfo->numOfTags;
char** pTags = pFillInfo->pTags;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
if (numOfRows == 0) {
/*
* we need to rebuild whole result set
* NOTE:we need to keep the last saved data, to generated the filled data
*/
while (num < outputRows) {
doInterpoResultImpl(pFillInfo, data, &num, srcData, pFillInfo->start, pTags, true);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return outputRows;
} else {
while (1) {
int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->rowIdx];
if ((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
/* set the next value for interpolation */
initBeforeAfterDataBuf(pFillInfo, nextValues);
int32_t offset = pFillInfo->rowIdx;
for (int32_t i = 0; i < pFillInfo->numOfCols - numOfTags; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
memcpy(*nextValues + pCol->col.offset, srcData[i] + offset * pCol->col.bytes, pCol->col.bytes);
}
}
if (((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) && num < outputRows) {
while (((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) && num < outputRows) {
doInterpoResultImpl(pFillInfo, data, &num, srcData, pFillInfo->start, pTags, false);
}
/* output buffer is full, abort */
if ((num == outputRows && FILL_IS_ASC_FILL(pFillInfo)) ||
(num < 0 && !FILL_IS_ASC_FILL(pFillInfo))) {
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return outputRows;
}
} else {
assert(pFillInfo->start == ts);
initBeforeAfterDataBuf(pFillInfo, prevValues);
// assign rows to dst buffer
int32_t i = 0;
for (; i < pFillInfo->numOfCols - numOfTags; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, num);
char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->rowIdx);
if (i == 0 ||
(pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) ||
(pCol->functionId == TSDB_FUNC_COUNT && GET_INT64_VAL(src) != 0)) {
assignVal(val1, src, pCol->col.bytes, pCol->col.type);
memcpy(*prevValues + pCol->col.offset, src, pCol->col.bytes);
} else { // i > 0 and data is null , do interpolation
if (pFillInfo->fillType == TSDB_FILL_PREV) {
assignVal(val1, *prevValues + pCol->col.offset, pCol->col.bytes, pCol->col.type);
} else if (pFillInfo->fillType == TSDB_FILL_LINEAR) {
assignVal(val1, src, pCol->col.bytes, pCol->col.type);
memcpy(*prevValues + pCol->col.offset, src, pCol->col.bytes);
} else {
assignVal(val1, (char*) &pCol->defaultVal.i, pCol->col.bytes, pCol->col.type);
}
}
}
// set the tag value for final result
setTagsValue(pFillInfo, data, pTags, pFillInfo->numOfCols - numOfTags, num);
pFillInfo->start += (pFillInfo->slidingTime * step);
pFillInfo->rowIdx += 1;
num += 1;
}
if ((pFillInfo->rowIdx >= pFillInfo->numOfRows && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->rowIdx < 0 && !FILL_IS_ASC_FILL(pFillInfo)) || num >= outputRows) {
if (pFillInfo->rowIdx >= pFillInfo->numOfRows || pFillInfo->rowIdx < 0) {
pFillInfo->rowIdx = -1;
pFillInfo->numOfRows = 0;
/* the raw data block is exhausted, next value does not exists */
tfree(*nextValues);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return num;
}
}
}
}
void taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int64_t* outputRows, int32_t capacity) {
int32_t remain = taosNumOfRemainRows(pFillInfo); // todo use iterator?
// TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime,
// pQuery->slidingTimeUnit, pQuery->precision);
// if (QUERY_IS_ASC_QUERY(pQuery)) {
// assert(ekey >= pQuery->window.ekey);
// } else {
// assert(ekey <= pQuery->window.ekey);
// }
int32_t rows = taosGetNumOfResultWithFill(pFillInfo, remain, pFillInfo->endKey, capacity);
int32_t numOfRes = taosDoInterpoResult(pFillInfo, output, remain, rows, pFillInfo->pData);
*outputRows = rows;
assert(numOfRes == rows);
}

429
src/query/src/qinterpolation.c
View File

@ -1,429 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "qinterpolation.h"
#include "os.h"
#include "qextbuffer.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tsqlfunction.h"
#define INTERPOL_IS_ASC_INTERPOL(interp) ((interp)->order == TSDB_ORDER_ASC)
int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t timeRange, char intervalTimeUnit, int16_t precision) {
if (timeRange == 0) {
return startTime;
}
if (intervalTimeUnit == 'a' || intervalTimeUnit == 'm' || intervalTimeUnit == 's' || intervalTimeUnit == 'h') {
return (startTime / timeRange) * timeRange;
} else {
/*
* here we revised the start time of day according to the local time zone,
* but in case of DST, the start time of one day need to be dynamically decided.
*
* TODO dynamically decide the start time of a day
*/
#if defined(WINDOWS) && _MSC_VER >= 1900
// see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019
int64_t timezone = _timezone;
int32_t daylight = _daylight;
char** tzname = _tzname;
#endif
int64_t t = (precision == TSDB_TIME_PRECISION_MILLI) ? MILLISECOND_PER_SECOND : MILLISECOND_PER_SECOND * 1000L;
int64_t revStartime = (startTime / timeRange) * timeRange + timezone * t;
int64_t revEndtime = revStartime + timeRange - 1;
if (revEndtime < startTime) {
revStartime += timeRange;
}
return revStartime;
}
}
void taosInitInterpoInfo(SInterpolationInfo* pInterpoInfo, int32_t order, int64_t startTimestamp,
int32_t numOfGroupbyTags, int32_t rowSize) {
pInterpoInfo->startTimestamp = startTimestamp;
pInterpoInfo->rowIdx = -1;
pInterpoInfo->numOfRawDataInRows = 0;
pInterpoInfo->numOfCurrentInterpo = 0;
pInterpoInfo->numOfTotalInterpo = 0;
pInterpoInfo->order = order;
pInterpoInfo->numOfTags = numOfGroupbyTags;
if (pInterpoInfo->pTags == NULL && numOfGroupbyTags > 0) {
pInterpoInfo->pTags = calloc(1, numOfGroupbyTags * POINTER_BYTES + rowSize);
}
// set the previous value to be null
tfree(pInterpoInfo->prevValues);
}
// the SInterpolationInfo itself will not be released
void taosDestoryInterpoInfo(SInterpolationInfo* pInterpoInfo) {
if (pInterpoInfo == NULL) {
return;
}
tfree(pInterpoInfo->prevValues);
tfree(pInterpoInfo->nextValues);
tfree(pInterpoInfo->pTags);
}
void taosInterpoSetStartInfo(SInterpolationInfo* pInterpoInfo, int32_t numOfRawDataInRows, int32_t type) {
if (type == TSDB_INTERPO_NONE) {
return;
}
pInterpoInfo->rowIdx = 0;
pInterpoInfo->numOfRawDataInRows = numOfRawDataInRows;
}
TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int32_t timeInterval, int8_t intervalTimeUnit, int8_t precision) {
if (order == TSDB_ORDER_ASC) {
return ekey;
} else {
return taosGetIntervalStartTimestamp(ekey, timeInterval, intervalTimeUnit, precision);
}
}
int32_t taosGetNumOfResultWithInterpo(SInterpolationInfo* pInterpoInfo, TSKEY* pPrimaryKeyArray,
int32_t numOfRawDataInRows, int64_t nInterval, int64_t ekey,
int32_t maxNumOfRows) {
int32_t numOfRes = taosGetNumOfResWithoutLimit(pInterpoInfo, pPrimaryKeyArray, numOfRawDataInRows, nInterval, ekey);
return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
}
int32_t taosGetNumOfResWithoutLimit(SInterpolationInfo* pInterpoInfo, int64_t* pPrimaryKeyArray,
int32_t numOfAvailRawData, int64_t nInterval, int64_t ekey) {
if (numOfAvailRawData > 0) {
int32_t finalNumOfResult = 0;
// get last timestamp, calculate the result size
int64_t lastKey = pPrimaryKeyArray[pInterpoInfo->numOfRawDataInRows - 1];
finalNumOfResult = (int32_t)(labs(lastKey - pInterpoInfo->startTimestamp) / nInterval) + 1;
assert(finalNumOfResult >= numOfAvailRawData);
return finalNumOfResult;
} else {
/* reach the end of data */
if ((ekey < pInterpoInfo->startTimestamp && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
(ekey > pInterpoInfo->startTimestamp && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) {
return 0;
} else {
return (int32_t)(labs(ekey - pInterpoInfo->startTimestamp) / nInterval) + 1;
}
}
}
bool taosHasRemainsDataForInterpolation(SInterpolationInfo* pInterpoInfo) {
return taosNumOfRemainPoints(pInterpoInfo) > 0;
}
int32_t taosNumOfRemainPoints(SInterpolationInfo* pInterpoInfo) {
if (pInterpoInfo->rowIdx == -1 || pInterpoInfo->numOfRawDataInRows == 0) {
return 0;
}
return INTERPOL_IS_ASC_INTERPOL(pInterpoInfo) ? (pInterpoInfo->numOfRawDataInRows - pInterpoInfo->rowIdx)
: pInterpoInfo->rowIdx + 1;
}
static double doLinearInterpolationImpl(double v1, double v2, double k1, double k2, double k) {
return v1 + (v2 - v1) * (k - k1) / (k2 - k1);
}
int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) {
switch (type) {
case TSDB_DATA_TYPE_INT: {
*(int32_t*)point->val = doLinearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, point1->key,
point2->key, point->key);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
*(float*)point->val =
doLinearInterpolationImpl(*(float*)point1->val, *(float*)point2->val, point1->key, point2->key, point->key);
break;
};
case TSDB_DATA_TYPE_DOUBLE: {
*(double*)point->val =
doLinearInterpolationImpl(*(double*)point1->val, *(double*)point2->val, point1->key, point2->key, point->key);
break;
};
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT: {
*(int64_t*)point->val = doLinearInterpolationImpl(*(int64_t*)point1->val, *(int64_t*)point2->val, point1->key,
point2->key, point->key);
break;
};
case TSDB_DATA_TYPE_SMALLINT: {
*(int16_t*)point->val = doLinearInterpolationImpl(*(int16_t*)point1->val, *(int16_t*)point2->val, point1->key,
point2->key, point->key);
break;
};
case TSDB_DATA_TYPE_TINYINT: {
*(int8_t*)point->val =
doLinearInterpolationImpl(*(int8_t*)point1->val, *(int8_t*)point2->val, point1->key, point2->key, point->key);
break;
};
default: {
// TODO: Deal with interpolation with bool and strings and timestamp
return -1;
}
}
return 0;
}
static char* getPos(char* data, int32_t bytes, int32_t index) { return data + index * bytes; }
static void setTagsValueInInterpolation(tFilePage** data, char** pTags, SColumnModel* pModel, int32_t order,
int32_t start, int32_t capacity, int32_t num) {
for (int32_t j = 0, i = start; i < pModel->numOfCols; ++i, ++j) {
SSchema* pSchema = getColumnModelSchema(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, num);
assignVal(val1, pTags[j], pSchema->bytes, pSchema->type);
}
}
static void doInterpoResultImpl(SInterpolationInfo* pInterpoInfo, int16_t interpoType, tFilePage** data,
SColumnModel* pModel, int32_t* num, char** srcData, int64_t nInterval,
int64_t* defaultVal, int64_t currentTimestamp, int32_t capacity, int32_t numOfTags,
char** pTags, bool outOfBound) {
char** prevValues = &pInterpoInfo->prevValues;
char** nextValues = &pInterpoInfo->nextValues;
SPoint point1, point2, point;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pInterpoInfo->order);
char* val = getPos(data[0]->data, TSDB_KEYSIZE, *num);
*(TSKEY*)val = pInterpoInfo->startTimestamp;
int32_t numOfValCols = pModel->numOfCols - numOfTags;
// set the other values
if (interpoType == TSDB_INTERPO_PREV) {
char* pInterpolationData = INTERPOL_IS_ASC_INTERPOL(pInterpoInfo) ? *prevValues : *nextValues;
if (pInterpolationData != NULL) {
for (int32_t i = 1; i < numOfValCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i);
int16_t offset = getColumnModelOffset(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
if (isNull(pInterpolationData + offset, pSchema->type)) {
setNull(val1, pSchema->type, pSchema->bytes);
} else {
assignVal(val1, pInterpolationData + offset, pSchema->bytes, pSchema->type);
}
}
} else { /* no prev value yet, set the value for null */
for (int32_t i = 1; i < numOfValCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
setNull(val1, pSchema->type, pSchema->bytes);
}
}
setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, numOfValCols, capacity, *num);
} else if (interpoType == TSDB_INTERPO_LINEAR) {
// TODO : linear interpolation supports NULL value
if (*prevValues != NULL && !outOfBound) {
for (int32_t i = 1; i < numOfValCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i);
int16_t offset = getColumnModelOffset(pModel, i);
int16_t type = pSchema->type;
char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
setNull(val1, type, pSchema->bytes);
continue;
}
point1 = (SPoint){.key = *(TSKEY*)(*prevValues), .val = *prevValues + offset};
point2 = (SPoint){.key = currentTimestamp, .val = srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes};
point = (SPoint){.key = pInterpoInfo->startTimestamp, .val = val1};
taosDoLinearInterpolation(type, &point1, &point2, &point);
}
setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, numOfValCols, capacity, *num);
} else {
for (int32_t i = 1; i < numOfValCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
setNull(val1, pSchema->type, pSchema->bytes);
}
setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, numOfValCols, capacity, *num);
}
} else { /* default value interpolation */
for (int32_t i = 1; i < numOfValCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
assignVal(val1, (char*)&defaultVal[i], pSchema->bytes, pSchema->type);
}
setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, numOfValCols, capacity, *num);
}
pInterpoInfo->startTimestamp += (nInterval * step);
pInterpoInfo->numOfCurrentInterpo++;
(*num) += 1;
}
static void initBeforeAfterDataBuf(SColumnModel* pModel, char** nextValues) {
if (*nextValues != NULL) {
return;
}
*nextValues = calloc(1, pModel->rowSize);
for (int i = 1; i < pModel->numOfCols; i++) {
int16_t offset = getColumnModelOffset(pModel, i);
SSchema* pSchema = getColumnModelSchema(pModel, i);
setNull(*nextValues + offset, pSchema->type, pSchema->bytes);
}
}
int32_t taosDoInterpoResult(SInterpolationInfo* pInterpoInfo, int16_t interpoType, tFilePage** data,
int32_t numOfRawDataInRows, int32_t outputRows, int64_t nInterval,
const int64_t* pPrimaryKeyArray, SColumnModel* pModel, char** srcData, int64_t* defaultVal,
const int32_t* functionIDs, int32_t bufSize) {
int32_t num = 0;
pInterpoInfo->numOfCurrentInterpo = 0;
char** prevValues = &pInterpoInfo->prevValues;
char** nextValues = &pInterpoInfo->nextValues;
int32_t numOfTags = pInterpoInfo->numOfTags;
char** pTags = pInterpoInfo->pTags;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pInterpoInfo->order);
if (numOfRawDataInRows == 0) {
/*
* we need to rebuild whole data
* NOTE:we need to keep the last saved data, to satisfy the interpolation
*/
while (num < outputRows) {
doInterpoResultImpl(pInterpoInfo, interpoType, data, pModel, &num, srcData, nInterval, defaultVal,
pInterpoInfo->startTimestamp, bufSize, numOfTags, pTags, true);
}
pInterpoInfo->numOfTotalInterpo += pInterpoInfo->numOfCurrentInterpo;
return outputRows;
} else {
while (1) {
int64_t currentTimestamp = pPrimaryKeyArray[pInterpoInfo->rowIdx];
if ((pInterpoInfo->startTimestamp < currentTimestamp && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
(pInterpoInfo->startTimestamp > currentTimestamp && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) {
/* set the next value for interpolation */
initBeforeAfterDataBuf(pModel, nextValues);
int32_t offset = pInterpoInfo->rowIdx;
for (int32_t tlen = 0, i = 0; i < pModel->numOfCols - numOfTags; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i);
memcpy(*nextValues + tlen, srcData[i] + offset * pSchema->bytes, pSchema->bytes);
tlen += pSchema->bytes;
}
}
if (((pInterpoInfo->startTimestamp < currentTimestamp && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
(pInterpoInfo->startTimestamp > currentTimestamp && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) &&
num < outputRows) {
while (((pInterpoInfo->startTimestamp < currentTimestamp && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
(pInterpoInfo->startTimestamp > currentTimestamp && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) &&
num < outputRows) {
doInterpoResultImpl(pInterpoInfo, interpoType, data, pModel, &num, srcData, nInterval, defaultVal,
currentTimestamp, bufSize, numOfTags, pTags, false);
}
/* output buffer is full, abort */
if ((num == outputRows && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
(num < 0 && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) {
pInterpoInfo->numOfTotalInterpo += pInterpoInfo->numOfCurrentInterpo;
return outputRows;
}
} else {
assert(pInterpoInfo->startTimestamp == currentTimestamp);
initBeforeAfterDataBuf(pModel, prevValues);
// assign rows to dst buffer
int32_t i = 0;
for (int32_t tlen = 0; i < pModel->numOfCols - numOfTags; ++i) {
int16_t offset = getColumnModelOffset(pModel, i);
SSchema* pSchema = getColumnModelSchema(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, num);
char* src = srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes;
if (i == 0 ||
(functionIDs[i] != TSDB_FUNC_COUNT && !isNull(src, pSchema->type)) ||
(functionIDs[i] == TSDB_FUNC_COUNT && *(int64_t*)(src) != 0)) {
assignVal(val1, src, pSchema->bytes, pSchema->type);
memcpy(*prevValues + tlen, src, pSchema->bytes);
} else { // i > 0 and data is null , do interpolation
if (interpoType == TSDB_INTERPO_PREV) {
assignVal(val1, *prevValues + offset, pSchema->bytes, pSchema->type);
} else if (interpoType == TSDB_INTERPO_LINEAR) {
assignVal(val1, src, pSchema->bytes, pSchema->type);
memcpy(*prevValues + tlen, src, pSchema->bytes);
} else {
assignVal(val1, (char*)&defaultVal[i], pSchema->bytes, pSchema->type);
}
}
tlen += pSchema->bytes;
}
/* set the tag value for final result */
setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, pModel->numOfCols - numOfTags, bufSize,
num);
pInterpoInfo->startTimestamp += (nInterval * step);
pInterpoInfo->rowIdx += 1;
num += 1;
}
if ((pInterpoInfo->rowIdx >= pInterpoInfo->numOfRawDataInRows && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
(pInterpoInfo->rowIdx < 0 && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) || num >= outputRows) {
if (pInterpoInfo->rowIdx >= pInterpoInfo->numOfRawDataInRows || pInterpoInfo->rowIdx < 0) {
pInterpoInfo->rowIdx = -1;
pInterpoInfo->numOfRawDataInRows = 0;
/* the raw data block is exhausted, next value does not exists */
tfree(*nextValues);
}
pInterpoInfo->numOfTotalInterpo += pInterpoInfo->numOfCurrentInterpo;
return num;
}
}
}
}

16
src/query/src/qpercentile.c
View File

@ -64,26 +64,26 @@ static tFilePage *loadIntoBucketFromDisk(tMemBucket *pMemBucket, int32_t segIdx,
for (uint32_t j = 0; j < pFlushInfo->numOfPages; ++j) {
ret = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
UNUSED(ret);
assert(pPage->numOfElems > 0);
assert(pPage->num > 0);
tColModelAppend(pDesc->pColumnModel, buffer, pPage->data, 0, pPage->numOfElems, pPage->numOfElems);
printf("id: %d count: %" PRIu64 "\n", j, buffer->numOfElems);
tColModelAppend(pDesc->pColumnModel, buffer, pPage->data, 0, pPage->num, pPage->num);
printf("id: %d count: %" PRIu64 "\n", j, buffer->num);
}
}
tfree(pPage);
assert(buffer->numOfElems == pMemBuffer->fileMeta.numOfElemsInFile);
assert(buffer->num == pMemBuffer->fileMeta.numOfElemsInFile);
}
// load data in pMemBuffer to buffer
tFilePagesItem *pListItem = pMemBuffer->pHead;
while (pListItem != NULL) {
tColModelAppend(pDesc->pColumnModel, buffer, pListItem->item.data, 0, pListItem->item.numOfElems,
pListItem->item.numOfElems);
tColModelAppend(pDesc->pColumnModel, buffer, pListItem->item.data, 0, pListItem->item.num,
pListItem->item.num);
pListItem = pListItem->pNext;
}
tColDataQSort(pDesc, buffer->numOfElems, 0, buffer->numOfElems - 1, buffer->data, TSDB_ORDER_ASC);
tColDataQSort(pDesc, buffer->num, 0, buffer->num - 1, buffer->data, TSDB_ORDER_ASC);
pDesc->pColumnModel->capacity = oldCapacity; // restore value
return buffer;
@ -881,7 +881,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
for (uint32_t jx = 0; jx < pFlushInfo->numOfPages; ++jx) {
ret = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
UNUSED(ret);
tMemBucketPut(pMemBucket, pPage->data, pPage->numOfElems);
tMemBucketPut(pMemBucket, pPage->data, pPage->num);
}
fclose(pMemBuffer->file);

143
src/query/src/qtokenizer.c
View File

@ -282,11 +282,7 @@ int tSQLKeywordCode(const char* z, int n) {
}
SKeyword** pKey = (SKeyword**)taosHashGet(KeywordHashTable, key, n);
if (pKey != NULL) {
return (*pKey)->type;
} else {
return TK_ID;
}
return (pKey != NULL)? (*pKey)->type:TK_ID;
}
/*
@ -594,31 +590,28 @@ SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn
while (1) {
*i += t0.n;
bool hasComma = false;
while ((str[*i] == ' ' || str[*i] == '\n' || str[*i] == '\r' || str[*i] == '\t' || str[*i] == '\f')
|| str[*i] == ',') {
if (str[*i] == ',') {
if (false == hasComma) {
hasComma = true;
} else { // comma only allowed once
t0.n = 0;
return t0;
}
int32_t numOfComma = 0;
char t = str[*i];
while (t == ' ' || t == '\n' || t == '\r' || t == '\t' || t == '\f' || t == ',') {
if (t == ',' && (++numOfComma > 1)) { // comma only allowed once
t0.n = 0;
return t0;
}
(*i)++;
t = str[++(*i)];
}
t0.n = tSQLGetToken(&str[*i], &t0.type);
bool ignoreFlag = false;
bool ignore = false;
for (uint32_t k = 0; k < numOfIgnoreToken; k++) {
if (t0.type == ignoreTokenTypes[k]) {
ignoreFlag = true;
ignore = true;
break;
}
}
if (!ignoreFlag) {
if (!ignore) {
break;
}
}
@ -662,114 +655,4 @@ SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn
return t0;
}
FORCE_INLINE bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); }
FORCE_INLINE bool isNumber(const SSQLToken* pToken) {
return (pToken->type == TK_INTEGER || pToken->type == TK_FLOAT || pToken->type == TK_HEX || pToken->type == TK_BIN);
}
int32_t isValidNumber(const SSQLToken* pToken) {
const char* z = pToken->z;
int32_t type = TK_ILLEGAL;
int32_t i = 0;
for(; i < pToken->n; ++i) {
switch (z[i]) {
case '+':
case '-': {
break;
}
case '.': {
/*
* handle the the float number with out integer part
* .123
* .123e4
*/
if (!isdigit(z[i+1])) {
return TK_ILLEGAL;
}
for (i += 2; isdigit(z[i]); i++) {
}
if ((z[i] == 'e' || z[i] == 'E') &&
(isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) {
i += 2;
while (isdigit(z[i])) {
i++;
}
}
type = TK_FLOAT;
goto _end;
}
case '0': {
char next = z[i + 1];
if (next == 'b') { // bin number
type = TK_BIN;
for (i += 2; (z[i] == '0' || z[i] == '1'); ++i) {
}
goto _end;
} else if (next == 'x') { //hex number
type = TK_HEX;
for (i += 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) {
}
goto _end;
}
}
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
type = TK_INTEGER;
for (; isdigit(z[i]); i++) {
}
int32_t seg = 0;
while (z[i] == '.' && isdigit(z[i + 1])) {
i += 2;
while (isdigit(z[i])) {
i++;
}
seg++;
type = TK_FLOAT;
}
if (seg > 1) {
return TK_ILLEGAL;
}
if ((z[i] == 'e' || z[i] == 'E') &&
(isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) {
i += 2;
while (isdigit(z[i])) {
i++;
}
type = TK_FLOAT;
}
goto _end;
}
default:
return TK_ILLEGAL;
}
}
_end:
if (i < pToken->n) {
return TK_ILLEGAL;
} else {
return type;
}
}
bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); }

4
src/query/src/tlosertree.c
View File

@ -54,7 +54,7 @@ uint32_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* pa
(*pTree)->numOfEntries = numOfEntries;
(*pTree)->totalEntries = totalEntries;
(*pTree)->param = param;
(*pTree)->comparaFn = compareFn;
(*pTree)->comparFn = compareFn;
// set initial value for loser tree
tLoserTreeInit(*pTree);
@ -95,7 +95,7 @@ void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) {
return;
}
int32_t ret = pTree->comparaFn(&pTree->pNode[parentId], &kLeaf, pTree->param);
int32_t ret = pTree->comparFn(&pTree->pNode[parentId], &kLeaf, pTree->param);
if (ret < 0) {
SLoserTreeNode t = pTree->pNode[parentId];
pTree->pNode[parentId] = kLeaf;

11
src/rpc/src/rpcMain.c
View File

@ -867,9 +867,8 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
// underlying UDP layer does not know it is server or client
pRecv->connType = pRecv->connType | pRpc->connType;
if (pRecv->ip==0 && pConn) {
rpcProcessBrokenLink(pConn);
tfree(pRecv->msg);
if (pRecv->ip == 0 && pConn) {
rpcProcessBrokenLink(pConn);
return NULL;
}
@ -889,12 +888,12 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
rpcSendErrorMsgToPeer(pRecv, code);
tTrace("%s %p %p, %s is sent with error code:%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
}
} else { // parsing OK
} else { // msg is passed to app only parsing is ok
rpcProcessIncomingMsg(pConn, pHead);
}
}
if (code) rpcFreeMsg(pRecv->msg);
if (code) rpcFreeMsg(pRecv->msg); // parsing failed, msg shall be freed
return pConn;
}
@ -989,6 +988,7 @@ static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) {
pHead->sourceId = pConn->ownId;
pHead->destId = pConn->peerId;
pHead->linkUid = pConn->linkUid;
pHead->ahandle = (uint64_t)pConn->ahandle;
memcpy(pHead->user, pConn->user, tListLen(pHead->user));
pHead->code = htonl(code);
@ -1011,6 +1011,7 @@ static void rpcSendReqHead(SRpcConn *pConn) {
pHead->sourceId = pConn->ownId;
pHead->destId = pConn->peerId;
pHead->linkUid = pConn->linkUid;
pHead->ahandle = (uint64_t)pConn->ahandle;
memcpy(pHead->user, pConn->user, tListLen(pHead->user));
pHead->code = 1;

115
src/rpc/src/rpcTcp.c
View File

@ -39,8 +39,8 @@ typedef struct SThreadObj {
pthread_t thread;
SFdObj * pHead;
pthread_mutex_t mutex;
pthread_cond_t fdReady;
uint32_t ip;
bool stop;
int pollFd;
int numOfFds;
int threadId;
@ -50,6 +50,7 @@ typedef struct SThreadObj {
} SThreadObj;
typedef struct {
int fd;
uint32_t ip;
uint16_t port;
char label[12];
@ -63,7 +64,7 @@ static void *taosProcessTcpData(void *param);
static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd);
static void taosFreeFdObj(SFdObj *pFdObj);
static void taosReportBrokenLink(SFdObj *pFdObj);
static void taosAcceptTcpConnection(void *arg);
static void* taosAcceptTcpConnection(void *arg);
void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
SServerObj *pServerObj;
@ -95,12 +96,6 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
break;;
}
code = pthread_cond_init(&(pThreadObj->fdReady), NULL);
if (code != 0) {
tError("%s init TCP condition variable failed(%s)", label, strerror(errno));
break;
}
pThreadObj->pollFd = epoll_create(10); // size does not matter
if (pThreadObj->pollFd < 0) {
tError("%s failed to create TCP epoll", label);
@ -144,28 +139,47 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
return (void *)pServerObj;
}
static void taosStopTcpThread(SThreadObj* pThreadObj) {
pThreadObj->stop = true;
// signal the thread to stop, try graceful method first,
// and use pthread_cancel when failed
struct epoll_event event = { .events = EPOLLIN };
eventfd_t fd = eventfd(1, 0);
if (fd == -1) {
tError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThreadObj->label, strerror(errno));
pthread_cancel(pThreadObj->thread);
} else if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
tError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThreadObj->label, strerror(errno));
pthread_cancel(pThreadObj->thread);
}
pthread_join(pThreadObj->thread, NULL);
close(pThreadObj->pollFd);
if (fd != -1) {
close(fd);
}
while (pThreadObj->pHead) {
SFdObj *pFdObj = pThreadObj->pHead;
pThreadObj->pHead = pFdObj->next;
taosFreeFdObj(pFdObj);
}
}
void taosCleanUpTcpServer(void *handle) {
SServerObj *pServerObj = handle;
SThreadObj *pThreadObj;
if (pServerObj == NULL) return;
pthread_cancel(pServerObj->thread);
shutdown(pServerObj->fd, SHUT_RD);
pthread_join(pServerObj->thread, NULL);
for (int i = 0; i < pServerObj->numOfThreads; ++i) {
pThreadObj = pServerObj->pThreadObj + i;
while (pThreadObj->pHead) {
SFdObj *pFdObj = pThreadObj->pHead;
pThreadObj->pHead = pFdObj->next;
taosFreeFdObj(pFdObj);
}
close(pThreadObj->pollFd);
pthread_cancel(pThreadObj->thread);
pthread_join(pThreadObj->thread, NULL);
pthread_cond_destroy(&(pThreadObj->fdReady));
taosStopTcpThread(pThreadObj);
pthread_mutex_destroy(&(pThreadObj->mutex));
}
@ -175,31 +189,32 @@ void taosCleanUpTcpServer(void *handle) {
tfree(pServerObj);
}
static void taosAcceptTcpConnection(void *arg) {
static void* taosAcceptTcpConnection(void *arg) {
int connFd = -1;
struct sockaddr_in caddr;
int sockFd;
int threadId = 0;
SThreadObj *pThreadObj;
SServerObj *pServerObj;
pServerObj = (SServerObj *)arg;
sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
if (sockFd < 0) return;
pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
if (pServerObj->fd < 0) return NULL;
tTrace("%s TCP server is ready, ip:0x%x:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
while (1) {
socklen_t addrlen = sizeof(caddr);
connFd = accept(sockFd, (struct sockaddr *)&caddr, &addrlen);
if (connFd < 0) {
connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen);
if (connFd == -1) {
if (errno == EINVAL) {
tTrace("%s TCP server socket was shutdown, exiting...", pServerObj->label);
break;
}
tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno));
continue;
}
tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(caddr.sin_addr), caddr.sin_port);
taosKeepTcpAlive(connFd);
// pick up the thread to handle this connection
@ -213,13 +228,17 @@ static void taosAcceptTcpConnection(void *arg) {
inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds);
} else {
close(connFd);
tError("%s failed to malloc FdObj(%s)", pServerObj->label, strerror(errno));
tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno),
inet_ntoa(caddr.sin_addr), caddr.sin_port);
}
// pick up next thread for next connection
threadId++;
threadId = threadId % pServerObj->numOfThreads;
}
close(pServerObj->fd);
return NULL;
}
void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle) {
@ -237,11 +256,6 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *
return NULL;
}
if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) {
tError("%s init TCP condition variable failed(%s)", label, strerror(errno));
return NULL;
}
pThreadObj->pollFd = epoll_create(10); // size does not matter
if (pThreadObj->pollFd < 0) {
tError("%s failed to create TCP client epoll", label);
@ -268,17 +282,7 @@ void taosCleanUpTcpClient(void *chandle) {
SThreadObj *pThreadObj = chandle;
if (pThreadObj == NULL) return;
while (pThreadObj->pHead) {
SFdObj *pFdObj = pThreadObj->pHead;
pThreadObj->pHead = pFdObj->next;
taosFreeFdObj(pFdObj);
}
close(pThreadObj->pollFd);
pthread_cancel(pThreadObj->thread);
pthread_join(pThreadObj->thread, NULL);
taosStopTcpThread(pThreadObj);
tTrace (":%s, all connections are cleaned up", pThreadObj->label);
tfree(pThreadObj);
@ -337,7 +341,9 @@ static void taosReportBrokenLink(SFdObj *pFdObj) {
recvInfo.chandle = NULL;
recvInfo.connType = RPC_CONN_TCP;
(*(pThreadObj->processData))(&recvInfo);
}
} else {
taosFreeFdObj(pFdObj);
}
}
#define maxEvents 10
@ -348,15 +354,13 @@ static void *taosProcessTcpData(void *param) {
struct epoll_event events[maxEvents];
SRecvInfo recvInfo;
SRpcHead rpcHead;
while (1) {
pthread_mutex_lock(&pThreadObj->mutex);
if (pThreadObj->numOfFds < 1) {
pthread_cond_wait(&pThreadObj->fdReady, &pThreadObj->mutex);
}
pthread_mutex_unlock(&pThreadObj->mutex);
int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1);
if (pThreadObj->stop) {
tTrace("%s, tcp thread get stop event, exiting...", pThreadObj->label);
break;
}
if (fdNum < 0) continue;
for (int i = 0; i < fdNum; ++i) {
@ -444,7 +448,6 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd) {
if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj;
pThreadObj->pHead = pFdObj;
pThreadObj->numOfFds++;
pthread_cond_signal(&pThreadObj->fdReady);
pthread_mutex_unlock(&(pThreadObj->mutex));
return pFdObj;
@ -465,7 +468,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pFdObj->signature = NULL;
epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
close(pFdObj->fd);
taosCloseSocket(pFdObj->fd);
pThreadObj->numOfFds--;
@ -492,5 +495,3 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
tfree(pFdObj);
}

12
src/rpc/src/rpcUdp.c
View File

@ -135,14 +135,15 @@ void taosCleanUpUdpConnection(void *handle) {
for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i;
pConn->signature = NULL;
free(pConn->buffer);
pthread_cancel(pConn->thread);
taosCloseSocket(pConn->fd);
// shutdown to signal the thread to exit
shutdown(pConn->fd, SHUT_RD);
}
for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i;
pthread_join(pConn->thread, NULL);
free(pConn->buffer);
taosCloseSocket(pConn->fd);
tTrace("chandle:%p is closed", pConn);
}
@ -177,6 +178,11 @@ static void *taosRecvUdpData(void *param) {
while (1) {
dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen);
if(dataLen == 0) {
tTrace("data length is 0, socket was closed, exiting");
break;
}
port = ntohs(sourceAdd.sin_port);
if (dataLen < sizeof(SRpcHead)) {

2
src/rpc/test/rserver.c
View File

@ -127,6 +127,8 @@ int main(int argc, char *argv[]) {
SRpcInit rpcInit;
char dataName[20] = "server.data";
taosBlockSIGPIPE();
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 7000;
rpcInit.label = "SER";

11
src/tsdb/inc/tsdbMain.h
View File

@ -495,11 +495,12 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo);
int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError);
// --------- For read operations
int tsdbLoadCompIdx(SRWHelper *pHelper, void *target);
int tsdbLoadCompInfo(SRWHelper *pHelper, void *target);
int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target);
int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds);
int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target);
int tsdbLoadCompIdx(SRWHelper *pHelper, void *target);
int tsdbLoadCompInfo(SRWHelper *pHelper, void *target);
int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target);
int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds);
int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target);
void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols);
// --------- For write operations
int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols);

1
src/tsdb/src/tsdbCache.c
View File

@ -162,6 +162,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
pCache->totalCacheBlocks = totalBlocks;
tsdbAdjustCacheBlocks(pCache);
}
pRepo->config.totalBlocks = totalBlocks;
tsdbUnLockRepo((TsdbRepoT *)pRepo);
tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks);

32
src/tsdb/src/tsdbMain.c
View File

@ -9,8 +9,6 @@
#include "ttime.h"
#include <sys/stat.h>
int tsdbDebugFlag = 135;
#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI // default precision
#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_PRECISION_MILLI) && ((precision) <= TSDB_PRECISION_NANO))
#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP
@ -39,7 +37,8 @@ static TSKEY tsdbNextIterKey(SSkipListIterator *pIter);
static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey);
static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression);
static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep);
static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables);
static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables);
static int32_t tsdbSaveConfig(STsdbRepo *pRepo);
#define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid]
#define TSDB_GET_TABLE_BY_NAME(pRepo, name)
@ -321,10 +320,25 @@ int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg) {
ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock);
ASSERT(pRCfg->precision == pCfg->precision);
if (pRCfg->compression != pCfg->compression) tsdbAlterCompression(pRepo, pCfg->compression);
if (pRCfg->keep != pCfg->keep) tsdbAlterKeep(pRepo, pCfg->keep);
if (pRCfg->totalBlocks != pCfg->totalBlocks) tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks);
if (pRCfg->maxTables != pCfg->maxTables) tsdbAlterMaxTables(pRepo, pCfg->maxTables);
bool configChanged = false;
if (pRCfg->compression != pCfg->compression) {
configChanged = true;
tsdbAlterCompression(pRepo, pCfg->compression);
}
if (pRCfg->keep != pCfg->keep) {
configChanged = true;
tsdbAlterKeep(pRepo, pCfg->keep);
}
if (pRCfg->totalBlocks != pCfg->totalBlocks) {
configChanged = true;
tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks);
}
if (pRCfg->maxTables != pCfg->maxTables) {
configChanged = true;
tsdbAlterMaxTables(pRepo, pCfg->maxTables);
}
if (configChanged) tsdbSaveConfig(pRepo);
return TSDB_CODE_SUCCESS;
}
@ -1136,8 +1150,10 @@ static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) {
int maxFiles = keep / pCfg->maxTables + 3;
if (pRepo->config.keep > keep) {
pRepo->config.keep = keep;
pRepo->tsdbFileH->maxFGroups = maxFiles;
} else {
pRepo->config.keep = keep;
pRepo->tsdbFileH->fGroup = realloc(pRepo->tsdbFileH->fGroup, sizeof(SFileGroup));
if (pRepo->tsdbFileH->fGroup == NULL) {
// TODO: deal with the error
@ -1157,6 +1173,8 @@ static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) {
pMeta->maxTables = maxTables;
pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *));
memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables-oldMaxTables));
pRepo->config.maxTables = maxTables;
tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables);
}

39
src/tsdb/src/tsdbMeta.c
View File

@ -16,7 +16,7 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx)
static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable);
static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable);
static int tsdbEstimateTableEncodeSize(STable *pTable);
static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable);
static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable, bool rmFromIdx);
/**
* Encode a TSDB table object as a binary content
@ -127,7 +127,7 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
if (pTable->type == TSDB_SUPER_TABLE) {
STColumn* pColSchema = schemaColAt(pTable->tagSchema, 0);
pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes,
1, 0, 0, getTagIndexKey);
1, 0, 1, getTagIndexKey);
}
tsdbAddTableToMeta(pMeta, pTable, false);
@ -154,6 +154,7 @@ STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables) {
STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta));
if (pMeta == NULL) return NULL;
pMeta->maxTables = maxTables;
pMeta->nTables = 0;
pMeta->superList = NULL;
pMeta->tables = (STable **)calloc(maxTables, sizeof(STable *));
@ -288,6 +289,13 @@ int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
if (tsdbCheckTableCfg(pCfg) < 0) return -1;
STable *pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid);
if (pTable != NULL) {
tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name),
pTable->tableId.tid, pTable->tableId.uid);
return TSDB_CODE_TABLE_ALREADY_EXIST;
}
STable *super = NULL;
int newSuper = 0;
@ -315,7 +323,7 @@ int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
// index the first tag column
STColumn* pColSchema = schemaColAt(super->tagSchema, 0);
super->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes,
1, 0, 0, getTagIndexKey); // Allow duplicate key, no lock
1, 0, 1, getTagIndexKey); // Allow duplicate key, no lock
if (super->pIndex == NULL) {
tdFreeSchema(super->schema);
@ -410,7 +418,7 @@ int tsdbDropTable(TsdbRepoT *repo, STableId tableId) {
tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name),
tableId.tid, tableId.uid);
if (tsdbRemoveTableFromMeta(pMeta, pTable) < 0) return -1;
if (tsdbRemoveTableFromMeta(pMeta, pTable, true) < 0) return -1;
return 0;
@ -439,6 +447,7 @@ static int tsdbFreeTable(STable *pTable) {
// Free content
if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) {
tdFreeSchema(pTable->tagSchema);
tSkipListDestroy(pTable->pIndex);
}
@ -499,7 +508,7 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) {
return 0;
}
static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable, bool rmFromIdx) {
if (pTable->type == TSDB_SUPER_TABLE) {
SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex);
while (tSkipListIterNext(pIter)) {
@ -508,7 +517,7 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
ASSERT(tTable != NULL && tTable->type == TSDB_CHILD_TABLE);
tsdbRemoveTableFromMeta(pMeta, tTable);
tsdbRemoveTableFromMeta(pMeta, tTable, false);
}
tSkipListDestroyIter(pIter);
@ -524,7 +533,7 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
}
} else {
pMeta->tables[pTable->tableId.tid] = NULL;
if (pTable->type == TSDB_CHILD_TABLE) {
if (pTable->type == TSDB_CHILD_TABLE && rmFromIdx) {
tsdbRemoveTableFromIndex(pMeta, pTable);
}
@ -571,9 +580,21 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN];
char* key = tdGetRowDataOfCol(pTable->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
bool ret = tSkipListRemove(pSTable->pIndex, key);
SArray* res = tSkipListGet(pSTable->pIndex, key);
assert(ret);
size_t size = taosArrayGetSize(res);
assert(size > 0);
for(int32_t i = 0; i < size; ++i) {
SSkipListNode* pNode = taosArrayGetP(res, i);
STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode);
if (pElem->pTable == pTable) { // this is the exact what we need
tSkipListRemoveNode(pSTable->pIndex, pNode);
}
}
taosArrayDestroy(res);
return 0;
}

33
src/tsdb/src/tsdbRWHelper.c
View File

@ -409,7 +409,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
if (pIdx->offset > 0) {
pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
if (pIdx->offset < 0) return -1;
ASSERT(pIdx->offset >= tsizeof(pHelper->pCompIdx));
ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) return -1;
}
@ -489,6 +489,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
}
ASSERT(((char *)ptr - (char *)pHelper->pBuffer) == (pFile->info.len - sizeof(TSCKSUM)));
if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
}
}
@ -542,6 +543,34 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) {
return 0;
}
void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols) {
SCompData *pCompData = pHelper->pCompData;
for (int i = 0, j = 0; i < numOfCols;) {
if (j >= pCompData->numOfCols) {
pStatis[i].numOfNull = -1;
i++;
continue;
}
if (pStatis[i].colId == pCompData->cols[j].colId) {
pStatis[i].sum = pCompData->cols[j].sum;
pStatis[i].max = pCompData->cols[j].max;
pStatis[i].min = pCompData->cols[j].min;
pStatis[i].maxIndex = pCompData->cols[j].maxIndex;
pStatis[i].minIndex = pCompData->cols[j].minIndex;
pStatis[i].numOfNull = pCompData->cols[j].numOfNull;
i++;
j++;
} else if (pStatis[i].colId < pCompData->cols[j].colId) {
pStatis[i].numOfNull = -1;
i++;
} else {
j++;
}
}
}
static int comparColIdCompCol(const void *arg1, const void *arg2) {
return (*(int16_t *)arg1) - ((SCompCol *)arg2)->colId;
}
@ -747,7 +776,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
pCompCol->colId = pDataCol->colId;
pCompCol->type = pDataCol->type;
if (tDataTypeDesc[pDataCol->type].getStatisFunc) {
if (tDataTypeDesc[pDataCol->type].getStatisFunc && ncol != 0) {
(*tDataTypeDesc[pDataCol->type].getStatisFunc)(
(TSKEY *)(pDataCols->cols[0].pData), pDataCol->pData, rowsToWrite, &(pCompCol->min), &(pCompCol->max),
&(pCompCol->sum), &(pCompCol->minIndex), &(pCompCol->maxIndex), &(pCompCol->numOfNull));

774
src/tsdb/src/tsdbRead.c
View File

File diff suppressed because it is too large Load Diff

2
src/util/inc/talgo.h
View File

@ -26,6 +26,8 @@ extern "C" {
#define TD_GE (TD_EQ | TD_GT)
#define TD_LE (TD_EQ | TD_LT)
#define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx))
typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, const void *param);
/**

1
src/util/inc/tqueue.h
View File

@ -39,6 +39,7 @@ void taosResetQitems(taos_qall);
taos_qset taosOpenQset();
void taosCloseQset();
void taosQsetThreadResume(taos_qset param);
int taosAddIntoQset(taos_qset, taos_queue, void *ahandle);
void taosRemoveFromQset(taos_qset, taos_queue);
int taosGetQueueNumber(taos_qset);

1
src/util/inc/tsocket.h
View File

@ -31,7 +31,6 @@ int taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
int taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
int taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
int taosKeepTcpAlive(int sockFd);
void taosCloseTcpSocket(int sockFd);
int taosGetFqdn(char *);
uint32_t taosGetIpFromFqdn(const char *);

113
src/util/inc/tstoken.h
View File

@ -21,6 +21,8 @@ extern "C" {
#endif
#include "os.h"
#include "tutil.h"
#include "ttokendef.h"
#define TK_SPACE 200
#define TK_COMMENT 201
@ -31,7 +33,7 @@ extern "C" {
#define TK_FILE 206
#define TK_QUESTION 207 // denoting the placeholder of "?",when invoking statement bind query
#define TSQL_TBNAME "TBNAME"
#define TSQL_TBNAME "TBNAME"
#define TSQL_TBNAME_L "tbname"
// used to denote the minimum unite in sql parsing
@ -74,14 +76,117 @@ bool isKeyWord(const char *z, int32_t len);
* @param pToken
* @return
*/
bool isNumber(const SSQLToken *pToken);
#define isNumber(tk) \
((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN)
/**
* check if it is a token or not
* @param pToken
* @return token type, if it is not a number, TK_ILLEGAL will return
* @return token type, if it is not a number, TK_ILLEGAL will return
*/
int32_t isValidNumber(const SSQLToken* pToken);
static FORCE_INLINE int32_t isValidNumber(const SSQLToken* pToken) {
const char* z = pToken->z;
int32_t type = TK_ILLEGAL;
int32_t i = 0;
for(; i < pToken->n; ++i) {
switch (z[i]) {
case '+':
case '-': {
break;
}
case '.': {
/*
* handle the the float number with out integer part
* .123
* .123e4
*/
if (!isdigit(z[i+1])) {
return TK_ILLEGAL;
}
for (i += 2; isdigit(z[i]); i++) {
}
if ((z[i] == 'e' || z[i] == 'E') &&
(isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) {
i += 2;
while (isdigit(z[i])) {
i++;
}
}
type = TK_FLOAT;
goto _end;
}
case '0': {
char next = z[i + 1];
if (next == 'b') { // bin number
type = TK_BIN;
for (i += 2; (z[i] == '0' || z[i] == '1'); ++i) {
}
goto _end;
} else if (next == 'x') { //hex number
type = TK_HEX;
for (i += 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) {
}
goto _end;
}
}
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
type = TK_INTEGER;
for (; isdigit(z[i]); i++) {
}
int32_t seg = 0;
while (z[i] == '.' && isdigit(z[i + 1])) {
i += 2;
while (isdigit(z[i])) {
i++;
}
seg++;
type = TK_FLOAT;
}
if (seg > 1) {
return TK_ILLEGAL;
}
if ((z[i] == 'e' || z[i] == 'E') &&
(isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) {
i += 2;
while (isdigit(z[i])) {
i++;
}
type = TK_FLOAT;
}
goto _end;
}
default:
return TK_ILLEGAL;
}
}
_end:
return (i < pToken->n)? TK_ILLEGAL:type;
}
#ifdef __cplusplus
}

54
src/util/src/hash.c
View File

@ -102,7 +102,32 @@ static void doUpdateHashTable(SHashObj *pHashObj, SHashNode *pNode);
* @param hashVal hash value by hash function
* @return
*/
static SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal);
FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal) {
uint32_t hash = (*pHashObj->hashFp)(key, keyLen);
int32_t slot = HASH_INDEX(hash, pHashObj->capacity);
SHashEntry *pEntry = pHashObj->hashList[slot];
SHashNode *pNode = pEntry->next;
while (pNode) {
if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
break;
}
pNode = pNode->next;
}
if (pNode) {
assert(HASH_INDEX(pNode->hashVal, pHashObj->capacity) == slot);
}
// return the calculated hash value, to avoid calculating it again in other functions
if (hashVal != NULL) {
*hashVal = hash;
}
return pNode;
}
/**
* Resize the hash list if the threshold is reached
@ -438,33 +463,6 @@ void doUpdateHashTable(SHashObj *pHashObj, SHashNode *pNode) {
}
}
SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal) {
uint32_t hash = (*pHashObj->hashFp)(key, keyLen);
int32_t slot = HASH_INDEX(hash, pHashObj->capacity);
SHashEntry *pEntry = pHashObj->hashList[slot];
SHashNode *pNode = pEntry->next;
while (pNode) {
if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
break;
}
pNode = pNode->next;
}
if (pNode) {
assert(HASH_INDEX(pNode->hashVal, pHashObj->capacity) == slot);
}
// return the calculated hash value, to avoid calculating it again in other functions
if (hashVal != NULL) {
*hashVal = hash;
}
return pNode;
}
void taosHashTableResize(SHashObj *pHashObj) {
if (pHashObj->size < pHashObj->capacity * HASH_DEFAULT_LOAD_FACTOR) {
return;

2
src/util/src/talgo.c
View File

@ -23,8 +23,6 @@
memcpy((__right), (__buf), (__size));\
} while (0);
#define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx))
static void median(void *src, size_t size, size_t s, size_t e, const void *param, __ext_compar_fn_t comparFn, void* buf) {
int32_t mid = ((e - s) >> 1u) + s;

20
src/util/src/tcompare.c
View File

@ -216,14 +216,14 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
const char* pattern = pRight;
const char* str = pLeft;
int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo);
char pattern[128] = {0};
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
assert(varDataLen(pRight) < 128);
int32_t ret = patternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft), &pInfo);
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
}
@ -232,14 +232,14 @@ static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) {
return taosArraySearchString(arr, pLeft) == NULL ? 0 : 1;
}
static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
const wchar_t* pattern = pRight;
const wchar_t* str = pLeft;
int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo);
wchar_t pattern[128] = {0};
memcpy(pattern, varDataVal(pRight), varDataLen(pRight)/TSDB_NCHAR_SIZE);
assert(varDataLen(pRight) < 128);
int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo);
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
}

29
src/util/src/thashutil.c
View File

@ -10,7 +10,7 @@
#include "hashfunc.h"
#include "tutil.h"
#define ROTL32(x, r) ((x) << (r) | (x) >> (32 - (r)))
#define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r)))
#define FMIX32(h) \
do { \
@ -20,12 +20,12 @@
(h) *= 0xc2b2ae35; \
(h) ^= (h) >> 16; \
} while (0)
static void MurmurHash3_32_s(const void *key, int len, uint32_t seed, void *out) {
uint32_t MurmurHash3_32(const char *key, uint32_t len) {
const uint8_t *data = (const uint8_t *)key;
const int nblocks = len / 4;
const int nblocks = len >> 2u;
uint32_t h1 = seed;
uint32_t h1 = 0x12345678;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
@ -36,11 +36,11 @@ static void MurmurHash3_32_s(const void *key, int len, uint32_t seed, void *out)
uint32_t k1 = blocks[i];
k1 *= c1;
k1 = ROTL32(k1, 15);
k1 = ROTL32(k1, 15u);
k1 *= c2;
h1 ^= k1;
h1 = ROTL32(h1, 13);
h1 = ROTL32(h1, 13u);
h1 = h1 * 5 + 0xe6546b64;
}
@ -48,7 +48,7 @@ static void MurmurHash3_32_s(const void *key, int len, uint32_t seed, void *out)
uint32_t k1 = 0;
switch (len & 3) {
switch (len & 3u) {
case 3:
k1 ^= tail[2] << 16;
case 2:
@ -56,7 +56,7 @@ static void MurmurHash3_32_s(const void *key, int len, uint32_t seed, void *out)
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = ROTL32(k1, 15);
k1 = ROTL32(k1, 15u);
k1 *= c2;
h1 ^= k1;
};
@ -65,16 +65,7 @@ static void MurmurHash3_32_s(const void *key, int len, uint32_t seed, void *out)
FMIX32(h1);
*(uint32_t *)out = h1;
}
uint32_t MurmurHash3_32(const char *key, uint32_t len) {
const int32_t hashSeed = 0x12345678;
uint32_t val = 0;
MurmurHash3_32_s(key, len, hashSeed, &val);
return val;
return h1;
}
uint32_t taosIntHash_32(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint32_t *)key; }

10
src/util/src/tqueue.c
View File

@ -65,6 +65,7 @@ taos_queue taosOpenQueue() {
}
void taosCloseQueue(taos_queue param) {
if (param == NULL) return;
STaosQueue *queue = (STaosQueue *)param;
STaosQnode *pTemp;
STaosQnode *pNode = queue->head;
@ -224,12 +225,21 @@ taos_qset taosOpenQset() {
}
void taosCloseQset(taos_qset param) {
if (param == NULL) return;
STaosQset *qset = (STaosQset *)param;
pthread_mutex_destroy(&qset->mutex);
tsem_destroy(&qset->sem);
free(qset);
}
// tsem_post 'qset->sem', so that reader threads waiting for it
// resumes execution and return, should only be used to signal the
// thread to exit.
void taosQsetThreadResume(taos_qset param) {
STaosQset *qset = (STaosQset *)param;
tsem_post(&qset->sem);
}
int taosAddIntoQset(taos_qset p1, taos_queue p2, void *ahandle) {
STaosQueue *queue = (STaosQueue *)p2;
STaosQset *qset = (STaosQset *)p1;

Some files were not shown because too many files have changed in this diff Show More