xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '2.0' into feature/2.0tsdb

This commit is contained in:
hzcheng 2020-03-29 17:21:10 +08:00
parent a944461d57 113f164b9e
commit d127890a1d
25 changed files with 2739 additions and 615 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/client/src/tscAsync.c
View File

@ -514,7 +514,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
tscTansformSQLFunctionForSTableQuery(pQueryInfo);
tscIncStreamExecutionCount(pSql->pStream);
} else {
tscTrace("%p get tableMeta/metricMeta successfully", pSql);
tscTrace("%p get tableMeta successfully", pSql);
}
tscDoQuery(pSql);

11
src/client/src/tscSecondaryMerge.c
View File

@ -605,7 +605,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
// (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pTableMetaInfo->pMetricMeta->numOfVnodes);
(*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * 1);
if (*pMemBuffer == NULL) {
tscError("%p failed to allocate memory", pSql);
pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
@ -636,10 +636,11 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity);
// for (int32_t i = 0; i < pTableMetaInfo->pMetricMeta->numOfVnodes; ++i) {
// (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel);
// (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL;
// }
size_t numOfSubs = taosArrayGetSize(pTableMetaInfo->vgroupIdList);
for (int32_t i = 0; i < numOfSubs; ++i) {
(*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel);
(*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL;
}
if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) {
pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;

22
src/client/src/tscServer.c
View File

@ -509,31 +509,31 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSubmitMsg *pShellMsg;
char * pMsg, *pStart;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta;
pStart = pSql->cmd.payload + tsRpcHeadSize;
pMsg = pStart;
char* pMsg = pSql->cmd.payload + tsRpcHeadSize;
// NOTE: shell message size should not include SMsgDesc
int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc);
SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg;
pMsgDesc->numOfVnodes = htonl(1); //set the number of vnodes
pMsgDesc->numOfVnodes = htonl(1); //todo set the right number of vnodes
pMsg += sizeof(SMsgDesc);
pShellMsg = (SSubmitMsg *)pMsg;
SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg;
pShellMsg->header.vgId = htonl(pTableMeta->vgId);
pShellMsg->header.contLen = htonl(pSql->cmd.payloadLen);
pShellMsg->header.contLen = htonl(size);
pShellMsg->length = pShellMsg->header.contLen;
pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted
// pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here
// pSql->cmd.payloadLen is set during copying data into paylaod
pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htons(pMsgDesc->numOfVnodes));
// tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pTableMeta->index].ip),
// htons(pShellMsg->vnode));
return TSDB_CODE_SUCCESS;
}

187
src/client/src/tscSubquery.c
View File

@ -1006,13 +1006,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
pRes->qhandle = 1; // hack the qhandle check
const uint32_t nBufferSize = (1 << 16); // 64KB
const uint32_t nBufferSize = (1u << 16); // 64KB
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
int32_t numOfSubQueries = 0;
// int32_t numOfSubQueries = pTableMetaInfo->pMetricMeta->numOfVnodes;
int32_t numOfSubQueries = taosArrayGetSize(pTableMetaInfo->vgroupIdList);
assert(numOfSubQueries > 0);
int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize);
@ -1118,7 +1117,7 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) {
tfree(trsupport);
}
static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows);
static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows);
static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t errCode) {
// set no disk space error info
@ -1140,7 +1139,7 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES
pthread_mutex_unlock(&trsupport->queryMutex);
tscRetrieveFromVnodeCallBack(trsupport, tres, trsupport->pState->code);
tscRetrieveFromDnodeCallBack(trsupport, tres, trsupport->pState->code);
}
static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) {
@ -1235,7 +1234,86 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq
}
}
static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) {
int32_t idx = trsupport->subqueryIndex;
SSqlObj * pPObj = trsupport->pParentSqlObj;
tOrderDescriptor *pDesc = trsupport->pOrderDescriptor;
SSubqueryState* pState = trsupport->pState;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
// data in from current vnode is stored in cache and disk
uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
// tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip,
// pSvd->vnode, numOfRowsFromSubquery, idx);
tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
#ifdef _DEBUG_VIEW
printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems);
SSrcColumnInfo colInfo[256] = {0};
tscGetSrcColumnInfo(colInfo, pQueryInfo);
tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems,
trsupport->localBuffer->numOfElems, colInfo);
#endif
if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) {
tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql,
tsAvailTmpDirGB, tsMinimalTmpDirGB);
tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE);
return;
}
// each result for a vnode is ordered as an independant list,
// then used as an input of loser tree for disk-based merge routine
int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer,
pQueryInfo->groupbyExpr.orderType);
if (ret != 0) { // set no disk space error info, and abort retry
return tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE);
}
// keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion
// increases the finished value up to pState->numOfTotal value, which means all subqueries are completed.
// In this case, the comparsion between finished value and released pState->numOfTotal is not safe.
int32_t numOfTotal = pState->numOfTotal;
int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1);
if (finished < numOfTotal) {
tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished);
return tscFreeSubSqlObj(trsupport, pSql);
}
// all sub-queries are returned, start to local merge process
pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage;
tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj,
pState->numOfTotal, pState->numOfRetrievedRows);
SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0);
tscClearInterpInfo(pPQueryInfo);
tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel,
&pPObj->cmd, &pPObj->res);
tscTrace("%p build loser tree completed", pPObj);
pPObj->res.precision = pSql->res.precision;
pPObj->res.numOfRows = 0;
pPObj->res.row = 0;
// only free once
tfree(trsupport->pState);
tscFreeSubSqlObj(trsupport, pSql);
// set the command flag must be after the semaphore been correctly set.
pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC;
if (pPObj->res.code == TSDB_CODE_SUCCESS) {
(*pPObj->fp)(pPObj->param, pPObj, 0);
} else {
tscQueueAsyncRes(pPObj);
}
}
static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
SRetrieveSupport *trsupport = (SRetrieveSupport *)param;
int32_t idx = trsupport->subqueryIndex;
SSqlObj * pPObj = trsupport->pParentSqlObj;
@ -1264,15 +1342,15 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
// SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx);
SVnodeSidList *vnodeInfo = 0;
SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index];
// SVnodeSidList *vnodeInfo = 0;
// SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index];
if (numOfRows > 0) {
assert(pRes->numOfRows == numOfRows);
int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows);
tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql,
pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx);
// tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql,
// pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx);
if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) {
tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId64 " , current:%" PRId64,
@ -1298,85 +1376,16 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR
int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data,
pRes->numOfRows, pQueryInfo->groupbyExpr.orderType);
if (ret < 0) {
// set no disk space error info, and abort retry
if (ret < 0) { // set no disk space error info, and abort retry
tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE);
} else {
} else if (pRes->completed) {
tscAllDataRetrievedFromDnode(trsupport, pSql);
} else { // continue fetch data from dnode
pthread_mutex_unlock(&trsupport->queryMutex);
taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param);
}
} else { // all data has been retrieved to client
/* data in from current vnode is stored in cache and disk */
uint32_t numOfRowsFromVnode = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip,
pSvd->vnode, numOfRowsFromVnode, idx);
tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
#ifdef _DEBUG_VIEW
printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems);
SSrcColumnInfo colInfo[256] = {0};
tscGetSrcColumnInfo(colInfo, pQueryInfo);
tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems,
trsupport->localBuffer->numOfElems, colInfo);
#endif
if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) {
tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql,
tsAvailTmpDirGB, tsMinimalTmpDirGB);
tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE);
return;
}
// each result for a vnode is ordered as an independant list,
// then used as an input of loser tree for disk-based merge routine
int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer,
pQueryInfo->groupbyExpr.orderType);
if (ret != 0) {
/* set no disk space error info, and abort retry */
return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE);
}
// keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion
// increases the finished value up to pState->numOfTotal value, which means all subqueries are completed.
// In this case, the comparsion between finished value and released pState->numOfTotal is not safe.
int32_t numOfTotal = pState->numOfTotal;
int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1);
if (finished < numOfTotal) {
tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished);
return tscFreeSubSqlObj(trsupport, pSql);
}
// all sub-queries are returned, start to local merge process
pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage;
tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj,
pState->numOfTotal, pState->numOfRetrievedRows);
SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0);
tscClearInterpInfo(pPQueryInfo);
tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel,
&pPObj->cmd, &pPObj->res);
tscTrace("%p build loser tree completed", pPObj);
pPObj->res.precision = pSql->res.precision;
pPObj->res.numOfRows = 0;
pPObj->res.row = 0;
// only free once
tfree(trsupport->pState);
tscFreeSubSqlObj(trsupport, pSql);
// set the command flag must be after the semaphore been correctly set.
pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC;
if (pPObj->res.code == TSDB_CODE_SUCCESS) {
(*pPObj->fp)(pPObj->param, pPObj, 0);
} else {
tscQueueAsyncRes(pPObj);
taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param);
}
} else { // all data has been retrieved to client
tscAllDataRetrievedFromDnode(trsupport, pSql);
}
}
@ -1436,7 +1445,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
/*
* if a query on a vnode is failed, all retrieve operations from vnode that occurs later
* than this one are actually not necessary, we simply call the tscRetrieveFromVnodeCallBack
* than this one are actually not necessary, we simply call the tscRetrieveFromDnodeCallBack
* function to abort current and remain retrieve process.
*
* NOTE: threadsafe is required.
@ -1474,7 +1483,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
trsupport->subqueryIndex, pState->code);
}
tscRetrieveFromVnodeCallBack(param, tres, pState->code);
tscRetrieveFromDnodeCallBack(param, tres, pState->code);
} else { // success, proceed to retrieve data from dnode
if (vnodeInfo != NULL) {
tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql,
@ -1485,7 +1494,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
trsupport->subqueryIndex);
}
taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param);
taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param);
}
}

2
src/client/src/tscUtil.c
View File

@ -608,7 +608,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
* the payloadLen should be actual message body size
* the old value of payloadLen is the allocated payload size
*/
pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize - sizeof(SMsgDesc);
pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize;
assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + 100 && pCmd->payloadLen > 0);
return TSDB_CODE_SUCCESS;

167
src/client/tests/timeParseTest.cpp Normal file
View File

@ -0,0 +1,167 @@
#include <gtest/gtest.h>
#include <cassert>
#include <iostream>
#include "taos.h"
#include "tstoken.h"
#include "ttime.h"
#include "tutil.h"
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
extern void deltaToUtcInitOnce();
/* test parse time function */
TEST(testCase, parse_time) {
taos_options(TSDB_OPTION_TIMEZONE, "GMT-8");
deltaToUtcInitOnce();
char t1[] = "2018-1-1 1:1:1.952798";
char t13[] = "1970-1-1 0:0:0";
int64_t time = 0, time1 = 0;
taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952);
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND);
char t2[] = "2018-1-1T1:1:1.952Z";
taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952 + 28800000);
char t3[] = "2018-1-1 1:01:01.952";
taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952);
char t4[] = "2018-1-1 1:01:01.9";
char t5[] = "2018-1-1 1:01:1.900";
char t6[] = "2018-01-01 1:1:1.90";
char t7[] = "2018-01-01 01:01:01.9";
char t8[] = "2018-01-01 01:01:01.9007865";
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t9[] = "2017-4-3 1:1:2.980";
char t10[] = "2017-4-3T2:1:2.98+9:00";
taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t11[] = "2017-4-3T2:1:2.98+09:00";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t12[] = "2017-4-3T2:1:2.98+0900";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taos_options(TSDB_OPTION_TIMEZONE, "UTC");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 0);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
deltaToUtcInitOnce();
char t14[] = "1970-1-1T0:0:0Z";
taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 0);
char t40[] = "1970-1-1 0:0:0.999999999";
taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND);
char t41[] = "1997-1-1 0:0:0.999999999";
taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 852048000999);
int64_t k = timezone;
char t42[] = "1997-1-1T0:0:0.999999999Z";
taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND);
////////////////////////////////////////////////////////////////////
// illegal timestamp format
char t15[] = "2017-12-33 0:0:0";
EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1);
char t16[] = "2017-12-31 99:0:0";
EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1);
char t17[] = "2017-12-31T9:0:0";
EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1);
char t18[] = "2017-12-31T9:0:0.Z";
EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1);
char t19[] = "2017-12-31 9:0:0.-1";
EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1);
char t20[] = "2017-12-31 9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0);
EXPECT_EQ(time, 1514682000100);
char t21[] = "2017-12-31T9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1);
char t22[] = "2017-12-31 9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0);
char t23[] = "2017-12-31T9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0);
//======================== add some case ============================//
char b1[] = "9999-12-31 23:59:59.999";
taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 253402271999999);
char b2[] = "2020-01-01 01:01:01.321";
taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1577811661321);
taos_options(TSDB_OPTION_TIMEZONE, "America/New_York");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND);
}

9
src/inc/taosmsg.h
View File

@ -188,15 +188,6 @@ extern char *taosMsg[];
#pragma pack(push, 1)
// typedef struct {
// int32_t vnode;
// int32_t sid;
// int32_t sversion;
// uint64_t uid;
// int16_t numOfRows;
// char payLoad[];
//} SShellSubmitBlock;
typedef struct {
int32_t numOfVnodes;
} SMsgDesc;

4
src/query/CMakeLists.txt
View File

@ -12,4 +12,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(query ${SRC})
TARGET_LINK_LIBRARIES(query tsdb tutil m rt)
ENDIF ()
ENDIF ()
ADD_SUBDIRECTORY(tests)

8
src/query/inc/qtsbuf.h
View File

@ -48,10 +48,10 @@ typedef struct STSElem {
} STSElem;
typedef struct STSCursor {
int32_t vnodeIndex;
int32_t blockIndex;
int32_t tsIndex;
int32_t order;
int32_t vnodeIndex;
int32_t blockIndex;
int32_t tsIndex;
uint32_t order;
} STSCursor;
typedef struct STSBlock {

24
src/query/inc/queryExecutor.h
View File

@ -32,12 +32,6 @@ typedef struct SData {
char data[];
} SData;
enum {
// ST_QUERY_KILLED = 0, // query killed
ST_QUERY_PAUSED = 1, // query paused, due to full of the response buffer
ST_QUERY_COMPLETED = 2, // query completed
};
struct SColumnFilterElem;
typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
@ -60,18 +54,20 @@ typedef struct SWindowStatus {
} SWindowStatus;
typedef struct SWindowResult {
uint16_t numOfRows;
uint16_t numOfRows; // number of rows of current time window
SPosInfo pos; // Position of current result in disk-based output buffer
SResultInfo* resultInfo; // For each result column, there is a resultInfo
STimeWindow window; // The time window that current result covers.
SWindowStatus status;
SWindowStatus status; // this result status: closed or opened
} SWindowResult;
typedef struct SResultRec {
int64_t total;
int64_t size;
int64_t capacity;
int32_t threshold; // the threshold size, when the number of rows in result buffer, return to client
int64_t total; // total generated result size in rows
int64_t size; // current result set size in rows
int64_t capacity; // capacity of current result output buffer
// result size threshold in rows. If the result buffer is larger than this, pause query and return to client
int32_t threshold;
} SResultRec;
typedef struct SWindowResInfo {
@ -99,7 +95,6 @@ typedef struct SSingleColumnFilterInfo {
void* pData;
} SSingleColumnFilterInfo;
/* intermediate pos during multimeter query involves interval */
typedef struct STableQueryInfo {
int64_t lastKey;
STimeWindow win;
@ -107,7 +102,7 @@ typedef struct STableQueryInfo {
int16_t queryRangeSet; // denote if the query range is set, only available for interval query
int64_t tag;
STSCursor cur;
int32_t sid; // for retrieve the page id list
int32_t tid; // for retrieve the page id list
SWindowResInfo windowResInfo;
} STableQueryInfo;
@ -116,7 +111,6 @@ typedef struct STableDataInfo {
int32_t numOfBlocks;
int32_t start; // start block index
int32_t tableIndex;
void* pMeterObj;
int32_t groupIdx; // group id in table list
STableQueryInfo* pTableQInfo;
} STableDataInfo;

867
src/query/src/queryExecutor.c
View File

File diff suppressed because it is too large Load Diff

15
src/query/tests/CMakeLists.txt Normal file
View File

@ -0,0 +1,15 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest)
FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib)
IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
MESSAGE(STATUS "gTest library found, build unit test")
INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(queryTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread)
ENDIF()

121
src/query/tests/histogramTest.cpp Normal file
View File

@ -0,0 +1,121 @@
#include <gtest/gtest.h>
#include <sys/time.h>
#include <cassert>
#include <iostream>
#include "taos.h"
#include "tsdb.h"
#include "tstoken.h"
#include "tutil.h"
#include "qhistogram.h"
/* test validate the names for table/database */
TEST(testCase, histogram_binary_search) {
SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN);
pHisto->numOfEntries = 10;
for (int32_t i = 0; i < 10; ++i) {
pHisto->elems[i].num = 1;
pHisto->elems[i].val = i;
}
int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1);
assert(idx == 1);
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9);
assert(idx == 9);
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20);
assert(idx == 10);
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1);
assert(idx == 0);
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9);
assert(idx == 4);
free(pHisto);
}
TEST(testCase, histogram_add) {
SHistogramInfo* pHisto = NULL;
/**
* use arrayList, elapsed time is:
* before:
* 10,000,000 45sec, bin:1000 (-O0) / 17sec. bin:1000, (-O3)
*
* after:
*
*/
struct timeval systemTime;
gettimeofday(&systemTime, NULL);
int64_t st =
(int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000;
for (int32_t i = 0; i < 10000; ++i) {
tHistogramAdd(&pHisto, i);
// tHistogramPrint(pHisto);
}
//
gettimeofday(&systemTime, NULL);
int64_t et =
(int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000;
printf("total elapsed time: %ld\n", et - st);
printf("elements: %d, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries);
tHistogramPrint(pHisto);
printf("%ld\n", tHistogramSum(pHisto, 1.5));
printf("%ld\n", tHistogramSum(pHisto, 2));
printf("%ld\n", tHistogramSum(pHisto, 3));
printf("%ld\n", tHistogramSum(pHisto, 4));
printf("%ld\n", tHistogramSum(pHisto, 5));
printf("%ld\n", tHistogramSum(pHisto, 6));
for (int32_t i = 399; i < 400; ++i) {
printf("val:%d, %ld\n", i, tHistogramSum(pHisto, i));
}
double ratio[] = {0 / 100, 20.0 / 100, 88.0 / 100, 100 / 100};
double* res = tHistogramUniform(pHisto, ratio, 4);
for (int32_t i = 0; i < 4; ++i) {
printf("%f\n", res[i]);
}
SHistogramInfo* pHisto1 = NULL;
for (int32_t i = (90000 - 1); i >= 80000; --i) {
tHistogramAdd(&pHisto1, i);
}
tHistogramPrint(pHisto1);
SHistogramInfo* pRes = tHistogramMerge(pHisto1, pHisto, MAX_HISTOGRAM_BIN);
assert(pRes->numOfElems == pHisto->numOfElems + pHisto1->numOfElems);
tHistogramPrint(pRes);
tHistogramDestroy(&pHisto);
tHistogramDestroy(&pHisto1);
tHistogramDestroy(&pRes);
free(res);
}
TEST(testCase, heapsort) {
// int32_t num = 20;
//
// SHeapEntry* pEntry = tHeapCreate(num);
//
// for(int32_t i=0; i<num; ++i) {
// pEntry[i].val = num - 1 - i;
// }
//
// tHeapSort(pEntry, num);
//
// for(int32_t i=0; i<num; ++i) {
// printf("%lf, ", pEntry[i].val);
// }
//
// printf("\n");
//
// free(pEntry);
}

74
src/query/tests/patternMatchTest.cpp Normal file
View File

@ -0,0 +1,74 @@
#include <gtest/gtest.h>
#include <sys/time.h>
#include <cassert>
#include <iostream>
#include "tsqlfunction.h"
TEST(testCase, patternMatchTest) {
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
const char* str = "abcdef";
int32_t ret = patternMatch("a%b%", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "tm01";
ret = patternMatch("tm__", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "tkm1";
ret = patternMatch("t%m1", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "tkm1";
ret = patternMatch("%m1", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "";
ret = patternMatch("%_", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
str = "1";
ret = patternMatch("%__", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
str = "";
ret = patternMatch("%", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = " ";
ret = patternMatch("_", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "!";
ret = patternMatch("%_", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefg";
ret = patternMatch("abc%fg", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("abc%fg", str, 7, &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("abc%f_", str, 6, &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
str = "abcdefgabcdeju";
ret = patternMatch("abc%f_", str, 1, &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH);
str = "abcdefgabcdeju";
ret = patternMatch("ab", str, 2, &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("a%", str, 2, &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("a__", str, 2, &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH);
}

35
src/query/tests/resultBufferTest.cpp Normal file
View File

@ -0,0 +1,35 @@
#include <gtest/gtest.h>
#include <cassert>
#include <iostream>
#include "taos.h"
#include "qresultBuf.h"
#include "tsdb.h"
namespace {
// simple test
void simpleTest() {
SDiskbasedResultBuf* pResultBuf = NULL;
int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64);
int32_t pageId = 0;
int32_t groupId = 0;
tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
ASSERT_TRUE(pBufPage != NULL);
ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64);
ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L);
SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
ASSERT_EQ(list.size, 1);
ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
destroyResultBuf(pResultBuf);
}
} // namespace
TEST(testCase, resultBufferTest) {
simpleTest();
}

451
src/query/tests/tsBufTest.cpp Normal file
View File

@ -0,0 +1,451 @@
#include <gtest/gtest.h>
#include <cassert>
#include <iostream>
#include "taos.h"
#include "tsdb.h"
#include "tstoken.h"
#include "ttime.h"
#include "tutil.h"
#include "qtsbuf.h"
namespace {
/**
*
* @param num total number
* @param step gap between two consecutive ts
* @return
*/
int64_t* createTsList(int32_t num, int64_t start, int32_t step) {
int64_t* pList = (int64_t*)malloc(num * sizeof(int64_t));
for (int64_t i = 0; i < num; ++i) {
pList[i] = start + i * step;
}
return pList;
}
// simple test
void simpleTest() {
STSBuf* pTSBuf = tsBufCreate(true);
// write 10 ts points
int32_t num = 10;
int64_t tag = 1;
int64_t* list = createTsList(10, 10000000, 30);
tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num);
EXPECT_EQ(pTSBuf->block.tag, tag);
EXPECT_EQ(pTSBuf->numOfVnodes, 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf);
}
// one large list of ts, the ts list need to be split into several small blocks
void largeTSTest() {
STSBuf* pTSBuf = tsBufCreate(true);
// write 10 ts points
int32_t num = 1000000;
int64_t tag = 1;
int64_t* list = createTsList(num, 10000000, 30);
tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t));
// the data has been flush to disk, no data in cache
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.tag, tag);
EXPECT_EQ(pTSBuf->numOfVnodes, 1);
EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf);
}
void multiTagsTest() {
STSBuf* pTSBuf = tsBufCreate(true);
int32_t num = 10000;
int64_t tag = 1;
int64_t start = 10000000;
int32_t numOfTags = 50;
int32_t step = 30;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
EXPECT_EQ(pTSBuf->numOfVnodes, 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf);
}
void multiVnodeTagsTest() {
STSBuf* pTSBuf = tsBufCreate(true);
int32_t num = 10000;
int64_t start = 10000000;
int32_t numOfTags = 50;
int32_t step = 30;
// 2000 vnodes
for (int32_t j = 0; j < 20; ++j) {
// vnodeId:0
start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
}
EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestory(pTSBuf);
}
void loadDataTest() {
STSBuf* pTSBuf = tsBufCreate(true);
int32_t num = 10000;
int64_t oldStart = 10000000;
int32_t numOfTags = 50;
int32_t step = 30;
int32_t numOfVnode = 200;
// 10000 vnodes
for (int32_t j = 0; j < numOfVnode; ++j) {
// vnodeId:0
int64_t start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
printf("%d - %lld\n", i, list[0]);
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
}
EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
// create from exists file
STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false);
EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder);
EXPECT_EQ(pNewBuf->numOfVnodes, numOfVnode);
EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize);
EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset);
EXPECT_EQ(pNewBuf->pData[0].info.numOfBlocks, pTSBuf->pData[0].info.numOfBlocks);
EXPECT_EQ(pNewBuf->pData[0].info.compLen, pTSBuf->pData[0].info.compLen);
EXPECT_STREQ(pNewBuf->path, pTSBuf->path);
tsBufResetPos(pNewBuf);
int64_t s = taosGetTimestampUs();
printf("start:%lld\n", s);
int32_t x = 0;
while (tsBufNextPos(pNewBuf)) {
STSElem elem = tsBufGetElem(pNewBuf);
if (++x == 100000000) {
break;
}
// printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
}
int64_t e = taosGetTimestampUs();
printf("end:%lld, elapsed:%lld, total obj:%d\n", e, e - s, x);
}
void randomIncTsTest() {}
void TSTraverse() {
// 10000 vnodes
int32_t num = 200000;
int64_t oldStart = 10000000;
int32_t numOfTags = 3;
int32_t step = 30;
int32_t numOfVnode = 2;
STSBuf* pTSBuf = tsBufCreate(true);
for (int32_t j = 0; j < numOfVnode; ++j) {
// vnodeId:0
int64_t start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]);
free(list);
start += step * num;
list = createTsList(num, start, step);
tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]);
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
}
tsBufResetPos(pTSBuf);
////////////////////////////////////////////////////////////////////////////////////////
// reverse traverse
int64_t s = taosGetTimestampUs();
printf("start:%lld\n", s);
pTSBuf->cur.order = TSQL_SO_DESC;
// complete reverse traverse
int32_t x = 0;
while (tsBufNextPos(pTSBuf)) {
STSElem elem = tsBufGetElem(pTSBuf);
// printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
}
// specify the data block with vnode and tags value
tsBufResetPos(pTSBuf);
pTSBuf->cur.order = TSQL_SO_DESC;
int32_t startVnode = 1;
int32_t startTag = 2;
tsBufGetElemStartPos(pTSBuf, startVnode, startTag);
int32_t totalOutput = 10;
while (1) {
STSElem elem = tsBufGetElem(pTSBuf);
printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
if (!tsBufNextPos(pTSBuf)) {
break;
}
if (--totalOutput <= 0) {
totalOutput = 10;
tsBufGetElemStartPos(pTSBuf, startVnode, --startTag);
if (startTag == 0) {
startVnode -= 1;
startTag = 3;
}
if (startVnode < 0) {
break;
}
}
}
/////////////////////////////////////////////////////////////////////////////////
// traverse
pTSBuf->cur.order = TSQL_SO_ASC;
tsBufResetPos(pTSBuf);
// complete forwards traverse
while (tsBufNextPos(pTSBuf)) {
STSElem elem = tsBufGetElem(pTSBuf);
// printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
}
// specify the data block with vnode and tags value
tsBufResetPos(pTSBuf);
pTSBuf->cur.order = TSQL_SO_ASC;
startVnode = 1;
startTag = 2;
tsBufGetElemStartPos(pTSBuf, startVnode, startTag);
totalOutput = 10;
while (1) {
STSElem elem = tsBufGetElem(pTSBuf);
printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
if (!tsBufNextPos(pTSBuf)) {
break;
}
if (--totalOutput <= 0) {
totalOutput = 10;
tsBufGetElemStartPos(pTSBuf, startVnode, --startTag);
if (startTag < 0) {
startVnode -= 1;
startTag = 3;
}
if (startVnode < 0) {
break;
}
}
}
}
void performanceTest() {}
void emptyTagTest() {}
void invalidFileTest() {
const char* cmd = "touch /tmp/test";
// create empty file
system(cmd);
STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true);
EXPECT_TRUE(pNewBuf == NULL);
pNewBuf = tsBufCreateFromFile("/tmp/911", true);
EXPECT_TRUE(pNewBuf == NULL);
}
void mergeDiffVnodeBufferTest() {
STSBuf* pTSBuf1 = tsBufCreate(true);
STSBuf* pTSBuf2 = tsBufCreate(true);
int32_t step = 30;
int32_t num = 1000;
int32_t numOfTags = 10;
// vnodeId:0
int64_t start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t));
tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
tsBufFlush(pTSBuf2);
tsBufMerge(pTSBuf1, pTSBuf2, 9);
EXPECT_EQ(pTSBuf1->numOfVnodes, 2);
EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
tsBufDisplay(pTSBuf1);
tsBufDestory(pTSBuf2);
tsBufDestory(pTSBuf1);
}
void mergeIdenticalVnodeBufferTest() {
STSBuf* pTSBuf1 = tsBufCreate(true);
STSBuf* pTSBuf2 = tsBufCreate(true);
int32_t step = 30;
int32_t num = 1000;
int32_t numOfTags = 10;
// vnodeId:0
int64_t start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
for (int32_t i = numOfTags; i < numOfTags * 2; ++i) {
int64_t* list = createTsList(num, start, step);
tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
tsBufFlush(pTSBuf2);
tsBufMerge(pTSBuf1, pTSBuf2, 12);
EXPECT_EQ(pTSBuf1->numOfVnodes, 1);
EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
tsBufResetPos(pTSBuf1);
while (tsBufNextPos(pTSBuf1)) {
STSElem elem = tsBufGetElem(pTSBuf1);
EXPECT_EQ(elem.vnode, 12);
printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
}
tsBufDestory(pTSBuf1);
tsBufDestory(pTSBuf2);
}
} // namespace
TEST(testCase, tsBufTest) {
simpleTest();
largeTSTest();
multiTagsTest();
multiVnodeTagsTest();
loadDataTest();
invalidFileTest();
// randomIncTsTest();
TSTraverse();
mergeDiffVnodeBufferTest();
mergeIdenticalVnodeBufferTest();
}

769
src/query/tests/unitTest.cpp Normal file
View File

@ -0,0 +1,769 @@
#include <gtest/gtest.h>
#include <cassert>
#include <iostream>
#include "taos.h"
#include "tsdb.h"
#include "../../client/inc/tscUtil.h"
#include "ttime.h"
#include "tutil.h"
#include "tvariant.h"
#include "ttokendef.h"
namespace {
int32_t testValidateName(char* name) {
SSQLToken token = {0};
token.z = name;
token.n = strlen(name);
token.type = 0;
tSQLGetToken(name, &token.type);
return tscValidateName(&token);
}
}
static void _init_tvariant_bool(tVariant* t) {
t->i64Key = TSDB_FALSE;
t->nType = TSDB_DATA_TYPE_BOOL;
}
static void _init_tvariant_tinyint(tVariant* t) {
t->i64Key = -27;
t->nType = TSDB_DATA_TYPE_TINYINT;
}
static void _init_tvariant_int(tVariant* t) {
t->i64Key = -23997659;
t->nType = TSDB_DATA_TYPE_INT;
}
static void _init_tvariant_bigint(tVariant* t) {
t->i64Key = -3333333333333;
t->nType = TSDB_DATA_TYPE_BIGINT;
}
static void _init_tvariant_float(tVariant* t) {
t->dKey = -8991212199.8987878776;
t->nType = TSDB_DATA_TYPE_FLOAT;
}
static void _init_tvariant_binary(tVariant* t) {
tVariantDestroy(t);
t->pz = (char*)calloc(1, 20); //"2e3");
t->nType = TSDB_DATA_TYPE_BINARY;
strcpy(t->pz, "2e5");
t->nLen = strlen(t->pz);
}
static void _init_tvariant_nchar(tVariant* t) {
tVariantDestroy(t);
t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE);
t->nType = TSDB_DATA_TYPE_NCHAR;
wcscpy(t->wpz, L"-2000000.8765");
t->nLen = wcslen(t->wpz);
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
/* test validate the names for table/database */
TEST(testCase, db_table_name) {
char t01[] = "abc";
EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS);
char t02[] = "'abc'";
EXPECT_EQ(testValidateName(t02), TSDB_CODE_SUCCESS);
char t1[] = "abc.def";
EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS);
printf("%s\n", t1);
char t2[] = "'abc.def'";
EXPECT_EQ(testValidateName(t2), TSDB_CODE_SUCCESS);
printf("%s\n", t2);
char t3[] = "'abc'.def";
EXPECT_EQ(testValidateName(t3), TSDB_CODE_SUCCESS);
printf("%s\n", t3);
char t4[] = "'abc'.'def'";
EXPECT_EQ(testValidateName(t4), TSDB_CODE_SUCCESS);
char t5[] = "table.'def'";
EXPECT_EQ(testValidateName(t5), TSDB_CODE_INVALID_SQL);
char t6[] = "'table'.'def'";
EXPECT_EQ(testValidateName(t6), TSDB_CODE_INVALID_SQL);
char t7[] = "'_ab1234'.'def'";
EXPECT_EQ(testValidateName(t7), TSDB_CODE_SUCCESS);
printf("%s\n", t7);
char t8[] = "'_ab&^%1234'.'def'";
EXPECT_EQ(testValidateName(t8), TSDB_CODE_INVALID_SQL);
char t9[] = "'_123'.'gtest中文'";
EXPECT_EQ(testValidateName(t9), TSDB_CODE_INVALID_SQL);
char t10[] = "abc.'gtest中文'";
EXPECT_EQ(testValidateName(t10), TSDB_CODE_INVALID_SQL);
char t10_1[] = "abc.'中文gtest'";
EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_INVALID_SQL);
char t11[] = "'192.168.0.1'.abc";
EXPECT_EQ(testValidateName(t11), TSDB_CODE_INVALID_SQL);
char t12[] = "192.168.0.1.abc";
EXPECT_EQ(testValidateName(t12), TSDB_CODE_INVALID_SQL);
char t13[] = "abc.";
EXPECT_EQ(testValidateName(t13), TSDB_CODE_INVALID_SQL);
char t14[] = ".abc";
EXPECT_EQ(testValidateName(t14), TSDB_CODE_INVALID_SQL);
char t15[] = ".'abc'";
EXPECT_EQ(testValidateName(t15), TSDB_CODE_INVALID_SQL);
char t16[] = ".abc'";
EXPECT_EQ(testValidateName(t16), TSDB_CODE_INVALID_SQL);
char t17[] = "123a.\"abc\"";
EXPECT_EQ(testValidateName(t17), TSDB_CODE_INVALID_SQL);
printf("%s\n", t17);
char t18[] = "a.\"abc\"";
EXPECT_EQ(testValidateName(t18), TSDB_CODE_SUCCESS);
printf("%s\n", t18);
char t19[] = "'_ab1234'.'def'.'ab123'";
EXPECT_EQ(testValidateName(t19), TSDB_CODE_INVALID_SQL);
char t20[] = "'_ab1234*&^'";
EXPECT_EQ(testValidateName(t20), TSDB_CODE_INVALID_SQL);
char t21[] = "'1234_abc'";
EXPECT_EQ(testValidateName(t21), TSDB_CODE_INVALID_SQL);
// =======Containing capital letters=================
char t30[] = "ABC";
EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS);
char t31[] = "'ABC'";
EXPECT_EQ(testValidateName(t31), TSDB_CODE_SUCCESS);
char t32[] = "ABC.def";
EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS);
char t33[] = "'ABC.def";
EXPECT_EQ(testValidateName(t33), TSDB_CODE_INVALID_SQL);
char t33_0[] = "abc.DEF'";
EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_INVALID_SQL);
char t34[] = "'ABC.def'";
//int32_t tmp0 = testValidateName(t34);
EXPECT_EQ(testValidateName(t34), TSDB_CODE_SUCCESS);
char t35[] = "'ABC'.def";
EXPECT_EQ(testValidateName(t35), TSDB_CODE_SUCCESS);
char t36[] = "'ABC'.'DEF'";
EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS);
char t37[] = "abc.'DEF'";
EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS);
char t37_1[] = "abc.'_123DEF'";
EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS);
char t38[] = "'abc'.'DEF'";
EXPECT_EQ(testValidateName(t38), TSDB_CODE_SUCCESS);
// do not use key words
char t39[] = "table.'DEF'";
EXPECT_EQ(testValidateName(t39), TSDB_CODE_INVALID_SQL);
char t40[] = "'table'.'DEF'";
EXPECT_EQ(testValidateName(t40), TSDB_CODE_INVALID_SQL);
char t41[] = "'_abXYZ1234'.'deFF'";
EXPECT_EQ(testValidateName(t41), TSDB_CODE_SUCCESS);
char t42[] = "'_abDEF&^%1234'.'DIef'";
EXPECT_EQ(testValidateName(t42), TSDB_CODE_INVALID_SQL);
char t43[] = "'_123'.'Gtest中文'";
EXPECT_EQ(testValidateName(t43), TSDB_CODE_INVALID_SQL);
char t44[] = "'aABC'.'Gtest中文'";
EXPECT_EQ(testValidateName(t44), TSDB_CODE_INVALID_SQL);
char t45[] = "'ABC'.";
EXPECT_EQ(testValidateName(t45), TSDB_CODE_INVALID_SQL);
char t46[] = ".'ABC'";
EXPECT_EQ(testValidateName(t46), TSDB_CODE_INVALID_SQL);
char t47[] = "a.\"aTWc\"";
EXPECT_EQ(testValidateName(t47), TSDB_CODE_SUCCESS);
// ================has space =================
char t60[] = " ABC ";
EXPECT_EQ(testValidateName(t60), TSDB_CODE_INVALID_SQL);
char t60_1[] = " ABC ";
EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_INVALID_SQL);
char t61[] = "' ABC '";
EXPECT_EQ(testValidateName(t61), TSDB_CODE_SUCCESS);
char t61_1[] = "' ABC '";
EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_SUCCESS);
char t62[] = " ABC . def ";
EXPECT_EQ(testValidateName(t62), TSDB_CODE_INVALID_SQL);
char t63[] = "' ABC . def ";
EXPECT_EQ(testValidateName(t63), TSDB_CODE_INVALID_SQL);
char t63_0[] = " abc . DEF ' ";
EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_INVALID_SQL);
char t64[] = " ' ABC . def ' ";
//int32_t tmp1 = testValidateName(t64);
EXPECT_EQ(testValidateName(t64), TSDB_CODE_INVALID_SQL);
char t65[] = " ' ABC '. def ";
EXPECT_EQ(testValidateName(t65), TSDB_CODE_INVALID_SQL);
char t66[] = "' ABC '.' DEF '";
EXPECT_EQ(testValidateName(t66), TSDB_CODE_SUCCESS);
char t67[] = "abc . ' DEF '";
EXPECT_EQ(testValidateName(t67), TSDB_CODE_INVALID_SQL);
char t68[] = "' abc '.' DEF '";
EXPECT_EQ(testValidateName(t68), TSDB_CODE_SUCCESS);
// do not use key words
char t69[] = "table.'DEF'";
EXPECT_EQ(testValidateName(t69), TSDB_CODE_INVALID_SQL);
char t70[] = "'table'.'DEF'";
EXPECT_EQ(testValidateName(t70), TSDB_CODE_INVALID_SQL);
char t71[] = "'_abXYZ1234 '.' deFF '";
EXPECT_EQ(testValidateName(t71), TSDB_CODE_SUCCESS);
char t72[] = "'_abDEF&^%1234'.' DIef'";
EXPECT_EQ(testValidateName(t72), TSDB_CODE_INVALID_SQL);
char t73[] = "'_123'.' Gtest中文'";
EXPECT_EQ(testValidateName(t73), TSDB_CODE_INVALID_SQL);
char t74[] = "' aABC'.'Gtest中文'";
EXPECT_EQ(testValidateName(t74), TSDB_CODE_INVALID_SQL);
char t75[] = "' ABC '.";
EXPECT_EQ(testValidateName(t75), TSDB_CODE_INVALID_SQL);
char t76[] = ".' ABC'";
EXPECT_EQ(testValidateName(t76), TSDB_CODE_INVALID_SQL);
char t77[] = " a . \"aTWc\" ";
EXPECT_EQ(testValidateName(t77), TSDB_CODE_INVALID_SQL);
char t78[] = " a.\"aTWc \"";
EXPECT_EQ(testValidateName(t78), TSDB_CODE_INVALID_SQL);
// ===============muti string by space ===================
// There's no such case.
//char t160[] = "A BC";
//EXPECT_EQ(testValidateName(t160), TSDB_CODE_INVALID_SQL);
//printf("end:%s\n", t160);
// There's no such case.
//char t161[] = "' A BC '";
//EXPECT_EQ(testValidateName(t161), TSDB_CODE_INVALID_SQL);
char t162[] = " AB C . de f ";
EXPECT_EQ(testValidateName(t162), TSDB_CODE_INVALID_SQL);
char t163[] = "' AB C . de f ";
EXPECT_EQ(testValidateName(t163), TSDB_CODE_INVALID_SQL);
char t163_0[] = " ab c . DE F ' ";
EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_INVALID_SQL);
char t164[] = " ' AB C . de f ' ";
//int32_t tmp2 = testValidateName(t164);
EXPECT_EQ(testValidateName(t164), TSDB_CODE_INVALID_SQL);
char t165[] = " ' A BC '. de f ";
EXPECT_EQ(testValidateName(t165), TSDB_CODE_INVALID_SQL);
char t166[] = "' AB C '.' DE F '";
EXPECT_EQ(testValidateName(t166), TSDB_CODE_INVALID_SQL);
char t167[] = "ab c . ' D EF '";
EXPECT_EQ(testValidateName(t167), TSDB_CODE_INVALID_SQL);
char t168[] = "' a bc '.' DE F '";
EXPECT_EQ(testValidateName(t168), TSDB_CODE_INVALID_SQL);
}
/* test parse time function */
TEST(testCase, parse_time) {
taos_options(TSDB_OPTION_TIMEZONE, "GMT-8");
char t1[] = "2018-1-1 1:1:1.952798";
char t13[] = "1970-1-1 0:0:0";
int64_t time = 0, time1 = 0;
taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952);
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND);
char t2[] = "2018-1-1T1:1:1.952Z";
taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952 + 28800000);
char t3[] = "2018-1-1 1:01:01.952";
taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 1514739661952);
char t4[] = "2018-1-1 1:01:01.9";
char t5[] = "2018-1-1 1:01:1.900";
char t6[] = "2018-01-01 1:1:1.90";
char t7[] = "2018-01-01 01:01:01.9";
char t8[] = "2018-01-01 01:01:01.9007865";
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t9[] = "2017-4-3 1:1:2.980";
char t10[] = "2017-4-3T2:1:2.98+9:00";
taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t11[] = "2017-4-3T2:1:2.98+09:00";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
char t12[] = "2017-4-3T2:1:2.98+0900";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, time1);
taos_options(TSDB_OPTION_TIMEZONE, "UTC");
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 0);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
char t14[] = "1970-1-1T0:0:0Z";
taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 0);
char t40[] = "1970-1-1 0:0:0.999999999";
taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND);
char t41[] = "1997-1-1 0:0:0.999999999";
taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 852048000999);
int64_t k = timezone;
char t42[] = "1997-1-1T0:0:0.999999999Z";
taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND);
////////////////////////////////////////////////////////////////////
// illegal timestamp format
char t15[] = "2017-12-33 0:0:0";
EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1);
char t16[] = "2017-12-31 99:0:0";
EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1);
char t17[] = "2017-12-31T9:0:0";
EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1);
char t18[] = "2017-12-31T9:0:0.Z";
EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1);
char t19[] = "2017-12-31 9:0:0.-1";
EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1);
char t20[] = "2017-12-31 9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0);
EXPECT_EQ(time, 1514682000100);
char t21[] = "2017-12-31T9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1);
char t22[] = "2017-12-31 9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0);
char t23[] = "2017-12-31T9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0);
}
TEST(testCase, tvariant_convert) {
// 1. bool data to all other data types
tVariant t = {0};
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
EXPECT_EQ(t.i64Key, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
EXPECT_EQ(t.i64Key, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "FALSE");
tVariantDestroy(&t);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"FALSE");
tVariantDestroy(&t);
// 2. tinyint to other data types
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 1);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
EXPECT_EQ(t.i64Key, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
EXPECT_EQ(t.i64Key, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
EXPECT_EQ(t.i64Key, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-27");
tVariantDestroy(&t);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-27");
tVariantDestroy(&t);
// 3. int to other data
// types//////////////////////////////////////////////////////////////////
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 1);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
EXPECT_EQ(t.i64Key, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-23997659");
tVariantDestroy(&t);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-23997659");
tVariantDestroy(&t);
// 4. bigint to other data
// type//////////////////////////////////////////////////////////////////////////////
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 1);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-3333333333333");
tVariantDestroy(&t);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-3333333333333");
tVariantDestroy(&t);
// 5. float to other data
// types////////////////////////////////////////////////////////////////////////
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 1);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, -8991212199);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-8991212199.898788");
tVariantDestroy(&t);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-8991212199.898788");
tVariantDestroy(&t);
// 6. binary to other data types
// //////////////////////////////////////////////////////////////////
t.pz = "true";
t.nLen = strlen(t.pz);
t.nType = TSDB_DATA_TYPE_BINARY;
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 1);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.dKey, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.dKey, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "2e5");
tVariantDestroy(&t);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"2e5");
tVariantDestroy(&t);
// 7. nchar to other data types
// //////////////////////////////////////////////////////////////////
t.wpz = L"FALSE";
t.nLen = wcslen(t.wpz);
t.nType = TSDB_DATA_TYPE_NCHAR;
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64Key, 0);
_init_tvariant_nchar(&t);
EXPECT_LE(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64Key, -2000000);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-2000000.8765");
tVariantDestroy(&t);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-2000000.8765");
tVariantDestroy(&t);
}
TEST(testCase, tGetToken_Test) {
char* s = ".123 ";
uint32_t type = 0;
int32_t len = tSQLGetToken(s, &type);
EXPECT_EQ(type, TK_FLOAT);
EXPECT_EQ(len, strlen(s) - 1);
char s1[] = "1.123e10 ";
len = tSQLGetToken(s1, &type);
EXPECT_EQ(type, TK_FLOAT);
EXPECT_EQ(len, strlen(s1) - 1);
char s4[] = "0xff ";
len = tSQLGetToken(s4, &type);
EXPECT_EQ(type, TK_HEX);
EXPECT_EQ(len, strlen(s4) - 1);
// invalid data type
char s2[] = "e10 ";
len = tSQLGetToken(s2, &type);
EXPECT_FALSE(type == TK_FLOAT);
char s3[] = "1.1.1.1";
len = tSQLGetToken(s3, &type);
EXPECT_EQ(type, TK_IPTOKEN);
EXPECT_EQ(len, strlen(s3));
char s5[] = "0x ";
len = tSQLGetToken(s5, &type);
EXPECT_FALSE(type == TK_HEX);
}
static SSQLToken createStrToken(char* s) {
SSQLToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)};
t.type = TK_STRING;
t.z = s;
t.n = strlen(s);
return t;
}
TEST(testCase, isValidNumber_test) {
SSQLToken t1 = createStrToken("123abc");
EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL);
t1 = createStrToken("0xabc");
EXPECT_EQ(isValidNumber(&t1), TK_HEX);
t1 = createStrToken("0b11101");
EXPECT_EQ(isValidNumber(&t1), TK_BIN);
t1 = createStrToken(".134abc");
EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL);
t1 = createStrToken("1e1 ");
EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL);
t1 = createStrToken("1+2");
EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL);
t1 = createStrToken("-0x123");
EXPECT_EQ(isValidNumber(&t1), TK_HEX);
t1 = createStrToken("-1");
EXPECT_EQ(isValidNumber(&t1), TK_INTEGER);
t1 = createStrToken("-0b1110");
EXPECT_EQ(isValidNumber(&t1), TK_BIN);
t1 = createStrToken("-.234");
EXPECT_EQ(isValidNumber(&t1), TK_FLOAT);
}
TEST(testCase, getTempFilePath_test) {
char path[4096] = {0};
memset(path, 1, 4096);
getTmpfilePath("new_tmp", path);
printf("%s\n", path);
}

3
src/util/CMakeLists.txt
View File

@ -29,6 +29,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
MESSAGE(STATUS "Failed to find iconv, use default encoding method")
ENDIF ()
ENDIF ()
ADD_SUBDIRECTORY(tests)
ELSEIF (TD_WINDOWS_64)
ADD_DEFINITIONS(-DUSE_LIBICONV)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
@ -113,4 +115,3 @@ ENDIF()
#ENDIF ()

15
src/util/tests/CMakeLists.txt Normal file
View File

@ -0,0 +1,15 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest)
FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib)
IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
MESSAGE(STATUS "gTest library found, build unit test")
INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(utilTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(utilTest tutil gtest pthread)
ENDIF()

142
src/util/tests/cacheTest.cpp Normal file
View File

@ -0,0 +1,142 @@
#include <iostream>
#include <gtest/gtest.h>
#include <sys/time.h>
#include "taos.h"
//#include "tsdb.h"
//#include "testCommon.h"
#include "tstoken.h"
#include "tutil.h"
#include "tcache.h"
#include "ttimer.h"
#include "ttime.h"
namespace {
int32_t tsMaxMgmtConnections = 10000;
int32_t tsMaxMeterConnections = 200;
}
// test cache
TEST(testCase, client_cache_test) {
const int32_t REFRESH_TIME_IN_SEC = 2;
void* tscTmr = taosTmrInit (tsMaxMgmtConnections*2, 200, 6000, "TSC");
SCacheObj* tscCacheHandle = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
char* key1 = "test1";
char* data1 = "test11";
char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1), 1);
sleep(REFRESH_TIME_IN_SEC+1);
printf("obj is still valid: %s\n", cachedObj);
char* data2 = "test22";
taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
/* the object is cleared by cache clean operation */
cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2), 20);
printf("after updated: %s\n", cachedObj);
printf("start to remove data from cache\n");
taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
printf("end of removing data from cache\n");
getchar();
char* key3 = "test2";
char* data3 = "kkkkkkk";
char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3), 1);
printf("%s\n", cachedObj2);
taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
sleep(3);
char* d = (char*) taosCacheAcquireByName(tscCacheHandle, key3);
// assert(d == NULL);
char* key5 = "test5";
char* data5 = "data5kkkkk";
cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5), 20);
char* data6= "new Data after updated";
taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6), 20);
printf("%s\n", cachedObj2);
taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true);
char* data7 = "add call update procedure";
cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7), 20);
printf("%s\n=======================================\n\n", cachedObj2);
char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5);
taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true);
taosCacheRelease(tscCacheHandle, (void**) &cc, false);
char* data8 = "ttft";
char* key6 = "key6";
char* ft = (char*) taosCachePut(tscCacheHandle, key6, data8, strlen(data8), 20);
taosCacheRelease(tscCacheHandle, (void**) &ft, false);
/**
* 140ns
*/
uint64_t startTime = taosGetTimestampUs();
printf("Cache Performance Test\nstart time:%lld\n", startTime);
for(int32_t i=0; i<1000; ++i) {
char* dd = (char*) taosCacheAcquireByName(tscCacheHandle, key6);
if (dd != NULL) {
// printf("get the data\n");
} else {
printf("data has been released\n");
}
taosCacheRelease(tscCacheHandle, (void**) &dd, false);
}
uint64_t endTime = taosGetTimestampUs();
int64_t el = endTime - startTime;
printf("End of Test, %lld\nTotal Elapsed Time:%lld us.avg:%f us\n", endTime, el, el/1000.0);
taosCacheCleanup(tscCacheHandle);
}
TEST(testCase, cache_resize_test) {
const int32_t REFRESH_TIME_IN_SEC = 2;
void* tscTmr = taosTmrInit (1000*2, 200, 6000, "TSC");
auto* pCache = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
char key[256] = {0};
char data[1024] = "abcdefghijk";
int32_t len = strlen(data);
uint64_t startTime = taosGetTimestampUs();
int32_t num = 10000;
for(int32_t i = 0; i < num; ++i) {
int32_t len = sprintf(key, "abc_%7d", i);
taosCachePut(pCache, key, data, len, 3600);
}
uint64_t endTime = taosGetTimestampUs();
printf("add 10,000,000 object cost:%lld us, avg:%f us\n", endTime - startTime, (endTime-startTime)/(double)num);
startTime = taosGetTimestampUs();
for(int32_t i = 0; i < num; ++i) {
int32_t len = sprintf(key, "abc_%7d", i);
void* k = taosCacheAcquireByName(pCache, key);
assert(k != 0);
}
endTime = taosGetTimestampUs();
printf("retrieve 10,000,000 object cost:%lld us,avg:%f\n", endTime - startTime, (endTime - startTime)/(double)num);
taosCacheCleanup(pCache);
taosMsleep(20000);
getchar();
}

156
src/util/tests/hashTest.cpp Normal file
View File

@ -0,0 +1,156 @@
#include <gtest/gtest.h>
#include <limits.h>
#include <taosdef.h>
#include <iostream>
#include "hash.h"
#include "taos.h"
#include "ttime.h"
namespace {
// the simple test code for basic operations
void simpleTest() {
auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
ASSERT_EQ(taosHashGetSize(hashTable), 0);
// put 400 elements in the hash table
for(int32_t i = -200; i < 200; ++i) {
taosHashPut(hashTable, (const char*) &i, sizeof(int32_t), (char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), 400);
for(int32_t i = 0; i < 200; ++i) {
char* p = (char*) taosHashGet(hashTable, (const char*) &i, sizeof(int32_t));
ASSERT_TRUE(p != nullptr);
ASSERT_EQ(*reinterpret_cast<int32_t*>(p), i);
}
for(int32_t i = 1000; i < 2000; ++i) {
taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), 400);
for(int32_t i = 0; i < 100; ++i) {
taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), 300);
for(int32_t i = 100; i < 150; ++i) {
taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), 250);
taosHashCleanup(hashTable);
}
void stringKeyTest() {
auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false);
ASSERT_EQ(taosHashGetSize(hashTable), 0);
char key[128] = {0};
// put 200 elements in the hash table
for(int32_t i = 0; i < 1000; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), 1000);
for(int32_t i = 0; i < 1000; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
char* p = (char*) taosHashGet(hashTable, key, len);
ASSERT_TRUE(p != nullptr);
ASSERT_EQ(*reinterpret_cast<int32_t*>(p), i);
}
for(int32_t i = 500; i < 1000; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
taosHashRemove(hashTable, key, len);
}
ASSERT_EQ(taosHashGetSize(hashTable), 500);
for(int32_t i = 0; i < 499; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
taosHashRemove(hashTable, key, len);
}
ASSERT_EQ(taosHashGetSize(hashTable), 1);
taosHashCleanup(hashTable);
}
void functionTest() {
}
/**
* evaluate the performance issue, by add 10million elements in to hash table in
* a single threads situation
*/
void noLockPerformanceTest() {
auto* hashTable = (SHashObj*) taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false);
ASSERT_EQ(taosHashGetSize(hashTable), 0);
char key[128] = {0};
int32_t num = 5000000;
int64_t st = taosGetTimestampUs();
// put 10M elements in the hash table
for(int32_t i = 0; i < num; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t));
}
ASSERT_EQ(taosHashGetSize(hashTable), num);
int64_t et = taosGetTimestampUs();
printf("Elpased time:%" PRId64 " us to add %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num);
st = taosGetTimestampUs();
for(int32_t i = 0; i < num; ++i) {
int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
char* p = (char*) taosHashGet(hashTable, key, len);
ASSERT_TRUE(p != nullptr);
ASSERT_EQ(*reinterpret_cast<int32_t*>(p), i);
}
et = taosGetTimestampUs();
printf("Elpased time:%" PRId64 " us to fetch all %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num);
printf("The maximum length of overflow linklist in hash table is:%d\n", taosHashGetMaxOverflowLinkLength(hashTable));
taosHashCleanup(hashTable);
}
void multithreadsTest() {
//todo
}
// check the function robustness
void invalidOperationTest() {
}
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
TEST(testCase, hashTest) {
simpleTest();
stringKeyTest();
noLockPerformanceTest();
multithreadsTest();
}

178
src/util/tests/stringTest.cpp Normal file
View File

@ -0,0 +1,178 @@
#include <gtest/gtest.h>
#include <limits.h>
#include <taosdef.h>
#include <iostream>
#include "taos.h"
#include "tutil.h"
TEST(testCase, string_dequote_test) {
char t1[] = "'abc'";
int32_t len = strdequote(t1);
EXPECT_EQ(3, len);
EXPECT_STRCASEEQ(t1, "abc");
char t2[] = "\"abc\"";
len = strdequote(t2);
EXPECT_EQ(3, len);
EXPECT_STRCASEEQ(t1, "abc");
char t21[] = " abc ";
strtrim(t21);
EXPECT_STREQ("abc", t21);
EXPECT_EQ(3, strlen(t21));
}
TEST(testCase, string_replace_test) {
char t3[] = "abc01abc02abc";
char* ret = strreplace(t3, "abc", "7");
EXPECT_EQ(strlen(ret), 7);
EXPECT_STREQ("7017027", ret);
free(ret);
char t4[] = "a01a02b03c04d05";
ret = strreplace(t4, "0", "9999999999");
EXPECT_EQ(strlen(ret), 5 * 10 + 10);
EXPECT_STREQ("a99999999991a99999999992b99999999993c99999999994d99999999995", ret);
free(ret);
char t5[] = "abc";
ret = strreplace(t5, "abc", "12345678901234567890");
EXPECT_EQ(strlen(ret), 20);
EXPECT_STREQ("12345678901234567890", ret);
free(ret);
char t6[] = "abc";
ret = strreplace(t6, "def", "abc");
EXPECT_EQ(strlen(ret), 3);
EXPECT_STREQ("abc", ret);
free(ret);
char t7[] = "abcde000000000000001234";
ret = strreplace(t7, "ab", "0000000");
EXPECT_EQ(strlen(ret), 28);
EXPECT_STREQ("0000000cde000000000000001234", ret);
free(ret);
char t8[] = "abc\ndef";
char t[] = {10, 0};
char f1[] = "\\n";
int32_t fx = strlen(f1);
ret = strreplace(t8, "\n", "\\n");
EXPECT_EQ(strlen(ret), 8);
EXPECT_STREQ("abc\\ndef", ret);
free(ret);
char t9[] = "abc\\ndef";
ret = strreplace(t9, "\\n", "\n");
EXPECT_EQ(strlen(ret), 7);
EXPECT_STREQ("abc\ndef", ret);
free(ret);
char t10[] = "abcdef";
ret = strreplace(t10, "", "0");
EXPECT_EQ(strlen(ret), 6);
EXPECT_STREQ("abcdef", ret);
free(ret);
}
TEST(testCase, string_tolower_test) {
char t[1024] = {1};
memset(t, 1, tListLen(t));
const char* a1 = "ABC";
strtolower(t, a1);
EXPECT_STREQ(t, "abc");
memset(t, 1, tListLen(t));
const char* a2 = "ABC\'ABC\'D";
strtolower(t, a2);
EXPECT_STREQ(t, "abc\'ABC\'d");
memset(t, 1, tListLen(t));
const char* a3 = "";
strtolower(t, a3);
EXPECT_STREQ(t, "");
memset(t, 1, tListLen(t));
const char* a4 = "\"AbcDEF\"";
strtolower(t, a4);
EXPECT_STREQ(t, a4);
memset(t, 1, tListLen(t));
const char* a5 = "1234\"AbcDEF\"456";
strtolower(t, a5);
EXPECT_STREQ(t, a5);
memset(t, 1, tListLen(t));
const char* a6 = "1234";
strtolower(t, a6);
EXPECT_STREQ(t, a6);
}
TEST(testCase, string_strnchr_test) {
char t[1024] = {0};
memset(t, 1, tListLen(t));
char a1[] = "AB.C";
EXPECT_TRUE(strnchr(a1, '.', strlen(a1), true) != NULL);
char a2[] = "abc.";
EXPECT_TRUE(strnchr(a2, '.', strlen(a2), true) != NULL);
char a8[] = "abc.";
EXPECT_TRUE(strnchr(a8, '.', 1, true) == NULL);
char a3[] = ".abc";
EXPECT_TRUE(strnchr(a3, '.', strlen(a3), true) != NULL);
char a4[] = "'.abc'";
EXPECT_TRUE(strnchr(a4, '.', strlen(a4), true) == NULL);
char a5[] = "'.abc.'abc";
EXPECT_TRUE(strnchr(a5, '.', strlen(a5), true) == NULL);
char a6[] = "0123456789.";
EXPECT_TRUE(strnchr(a6, '.', strlen(a6), true) != NULL);
char a7[] = "0123456789.";
EXPECT_TRUE(strnchr(a7, '.', 3, true) == NULL);
char a9[] = "0123456789.";
EXPECT_TRUE(strnchr(a9, '.', 0, true) == NULL);
char a10[] = "0123456789'.'";
EXPECT_TRUE(strnchr(a10, '.', strlen(a10), true) == NULL);
}
TEST(testCase, cache_resize_test) {
char a11[] = "abc'.'";
EXPECT_TRUE(strnchr(a11, '.', strlen(a11), false) != NULL);
char a12[] = "abc'-'";
EXPECT_TRUE(strnchr(a12, '-', strlen(a12), false) != NULL);
char a15[] = "abc'-'";
EXPECT_TRUE(strnchr(a15, '-', strlen(a15), true) == NULL);
char a13[] = "'-'";
EXPECT_TRUE(strnchr(a13, '-', strlen(a13), false) != NULL);
char a14[] = "'-'";
EXPECT_TRUE(strnchr(a14, '-', strlen(a14), true) == NULL);
char a16[] = "'-'.";
EXPECT_TRUE(strnchr(a16, '.', strlen(a16), true) != NULL);
}

4
src/vnode/tsdb/src/tsdbMeta.c
View File

@ -349,10 +349,10 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) {
} else {
// add non-super table to the array
pMeta->tables[pTable->tableId.tid] = pTable;
if (pTable->type == TSDB_CHILD_TABLE) {
// add STABLE to the index
if (pTable->type == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index
tsdbAddTableIntoIndex(pMeta, pTable);
}
pMeta->nTables++;
}

81
src/vnode/tsdb/src/tsdbRead.c
View File

@ -73,7 +73,7 @@ typedef struct SQueryFilesInfo {
char dbFilePathPrefix[PATH_MAX];
} SQueryFilesInfo;
typedef struct STableQueryInfo {
typedef struct STableCheckInfo {
STableId tableId;
TSKEY lastKey;
STable * pTableObj;
@ -81,7 +81,8 @@ typedef struct STableQueryInfo {
int32_t numOfBlocks;
int32_t start;
SCompBlock *pBlock;
} STableQueryInfo;
SSkipListIterator* iter;
} STableCheckInfo;
typedef struct {
SCompBlock *compBlock;
@ -90,14 +91,19 @@ typedef struct {
typedef struct STableDataBlockInfoEx {
SCompBlockFields pBlock;
STableQueryInfo* pMeterDataInfo;
STableCheckInfo* pMeterDataInfo;
int32_t blockIndex;
int32_t groupIdx; /* number of group is less than the total number of meters */
} STableDataBlockInfoEx;
enum {
SINGLE_TABLE_MODEL = 1,
MULTI_TABLE_MODEL = 2,
};
typedef struct STsdbQueryHandle {
struct STsdbRepo* pTsdb;
int8_t model; // access model, single table model or multi-table model
SQueryHandlePos cur; // current position
SQueryHandlePos start; // the start position, used for secondary/third iteration
int32_t unzipBufSize;
@ -120,14 +126,13 @@ typedef struct STsdbQueryHandle {
bool locateStart;
int32_t realNumOfRows;
bool loadDataAfterSeek; // load data after seek.
SArray* pTableQueryInfo;
SArray* pTableCheckInfo;
int32_t activeIndex;
int32_t tableIndex;
bool isFirstSlot;
void * qinfo; // query info handle, for debug purpose
SSkipListIterator* memIter;
STableDataBlockInfoEx *pDataBlockInfoEx;
} STsdbQueryHandle;
@ -271,19 +276,21 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond
size_t size = taosArrayGetSize(idList);
assert(size >= 1);
pQueryHandle->pTableQueryInfo = taosArrayInit(size, sizeof(STableQueryInfo));
pQueryHandle->pTableCheckInfo = taosArrayInit(size, sizeof(STableCheckInfo));
for(int32_t i = 0; i < size; ++i) {
STableId id = *(STableId*) taosArrayGet(idList, i);
STableQueryInfo info = {
STableCheckInfo info = {
.lastKey = pQueryHandle->window.skey,
.tableId = id,
.pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), //todo this may be failed
};
taosArrayPush(pQueryHandle->pTableQueryInfo, &info);
taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
}
pQueryHandle->model = (size > 1)? MULTI_TABLE_MODEL:SINGLE_TABLE_MODEL;
pQueryHandle->activeIndex = 0;
// malloc buffer in order to load data from file
@ -314,11 +321,13 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond
return (tsdb_query_handle_t)pQueryHandle;
}
bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex);
static bool hasMoreDataInCacheForSingleModel(STsdbQueryHandle* pHandle) {
assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1);
STable *pTable = pTableQInfo->pTableObj;
STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
STable *pTable = pTableCheckInfo->pTableObj;
assert(pTable != NULL);
// no data in cache, abort
if (pTable->mem == NULL && pTable->imem == NULL) {
@ -326,13 +335,49 @@ bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) {
}
// all data in mem are checked already.
if (pTableQInfo->lastKey > pTable->mem->keyLast) {
if (pTableCheckInfo->lastKey > pTable->mem->keyLast) {
return false;
}
return true;
}
static bool hasMoreDataInCacheForMultiModel(STsdbQueryHandle* pHandle) {
size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo);
assert(numOfTables > 0);
while(pHandle->activeIndex < numOfTables) {
STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
STable *pTable = pTableCheckInfo->pTableObj;
if (pTable->mem == NULL && pTable->imem == NULL) {
pHandle->activeIndex += 1; // try next table if exits
continue;
}
// all data in mem are checked already.
if (pTableCheckInfo->lastKey > pTable->mem->keyLast) {
pHandle->activeIndex += 1; // try next table if exits
continue;
}
return true;
}
// all tables has checked already
return false;
}
// handle data in cache situation
bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
if (pHandle->model == SINGLE_TABLE_MODEL) {
return hasMoreDataInCacheForSingleModel(pHandle);
} else {
return hasMoreDataInCacheForMultiModel(pHandle);
}
}
static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead,
TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pHandle) {
int numOfRows = 0;
@ -370,7 +415,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max
SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex);
STableCheckInfo* pTableQInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
STable *pTable = pTableQInfo->pTableObj;
TSKEY skey = 0, ekey = 0;
@ -379,11 +424,11 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) {
if (pTable->mem != NULL) {
// create mem table iterator if it is not created yet
if (pHandle->memIter == NULL) {
pHandle->memIter = tSkipListCreateIter(pTable->mem->pData);
if (pTableQInfo->iter == NULL) {
pTableQInfo->iter = tSkipListCreateIter(pTable->mem->pData);
}
rows = tsdbReadRowsFromCache(pHandle->memIter, INT64_MAX, 2, &skey, &ekey, pHandle);
rows = tsdbReadRowsFromCache(pTableQInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle);
}
SDataBlockInfo blockInfo = {

7
src/vnode/tsdb/tests/CMakeLists.txt
View File

@ -3,9 +3,4 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
add_executable(tsdbTests ${SOURCE_LIST})
target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb)
add_test(
NAME
unit
COMMAND
${CMAKE_CURRENT_BINARY_DIR}/tsdbTests
)
add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests)