xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'feat/TS-4243-3.0' of https://github.com/taosdata/TDengine into feat/TS-4243-tmq

This commit is contained in:
wangmm0220 2024-04-01 09:11:39 +08:00
parent 463cd1bf13 719b8439f0
commit fe7a42b118
45 changed files with 2026 additions and 399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/common/tdataformat.h
View File

@ -126,6 +126,7 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag);
void tRowGetKey(SRow *pRow, SRowKey *key);
int32_t tRowKeyCompare(const void *p1, const void *p2);
int32_t tRowKeyAssign(SRowKey* pDst, SRowKey* pSrc);
// SRowIter ================================
int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter);

2
include/libs/executor/storageapi.h
View File

@ -192,7 +192,7 @@ typedef struct TsdReader {
typedef struct SStoreCacheReader {
int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr,
SArray *pFuncTypeList);
SArray *pFuncTypeList, SColumnInfo* pPkCol, int32_t numOfPks);
void *(*closeReader)(void *pReader);
int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUidList);

12
include/libs/function/function.h
View File

@ -185,6 +185,7 @@ typedef struct SFuncInputRow {
TSKEY ts;
bool isDataNull;
char* pData;
char* pPk;
SSDataBlock* block; // prev row block or src block
int32_t rowIndex; // prev row block ? 0 : rowIndex in srcBlock
@ -198,7 +199,8 @@ typedef struct SFuncInputRowIter {
bool hasPrev;
SInputColumnInfoData* pInput;
SColumnInfoData* pData;
SColumnInfoData* pDataCol;
SColumnInfoData* pPkCol;
TSKEY* tsList;
int32_t rowIndex;
int32_t inputEndIndex;
@ -207,13 +209,11 @@ typedef struct SFuncInputRowIter {
TSKEY prevBlockTsEnd;
bool prevIsDataNull;
char* pPrevData;
char* pPrevPk;
SSDataBlock* pPrevRowBlock; // pre one row block
//TODO:
// int32_t prevStartOffset; // for diff, derivative.
// SPoint1 prevStartPoint; // for twa.
// int32_t startOffset; // for diff, derivative.
// SPoint1 startPoint; // for twa.
uint64_t groupId;
bool hasGroupId;
bool finalRow;
} SFuncInputRowIter;

2
include/libs/function/functionMgt.h
View File

@ -256,7 +256,7 @@ typedef enum EFuncDataRequired {
} EFuncDataRequired;
EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, STimeWindow* pTimeWindow);
EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo);
int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);

2
include/libs/qcom/query.h
View File

@ -197,7 +197,7 @@ typedef struct STableDataCxt {
SBoundColInfo boundColsInfo;
SArray* pValues;
SSubmitTbData* pData;
TSKEY lastTs;
SRowKey lastKey;
bool ordered;
bool duplicateTs;
} STableDataCxt;

1
include/util/taoserror.h
View File

@ -765,6 +765,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_TAG_IS_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x2671)
#define TSDB_CODE_PAR_SECOND_COL_PK TAOS_DEF_ERROR_CODE(0, 0x2672)
#define TSDB_CODE_PAR_COL_PK_TYPE TAOS_DEF_ERROR_CODE(0, 0x2673)
#define TSDB_CODE_PAR_INVALID_PK_OP TAOS_DEF_ERROR_CODE(0, 0x2674)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner

4
source/common/src/tdatablock.c
View File

@ -546,8 +546,8 @@ int32_t blockDataUpdatePkRange(SSDataBlock* pDataBlock, int32_t pkColumnIndex, b
}
} else {
if (IS_NUMERIC_TYPE(pColInfoData->info.type)) {
pDataBlock->info.pks[0].val = *(int64_t*) ekey;
pDataBlock->info.pks[1].val = *(int64_t*) skey;
pDataBlock->info.pks[0].val = *(int32_t*) ekey;
pDataBlock->info.pks[1].val = *(int32_t*) skey;
} else { // todo refactor
memcpy(pDataBlock->info.pks[0].pData, varDataVal(ekey), varDataLen(ekey));
pDataBlock->info.pks[0].nData = varDataLen(ekey);

43
source/common/src/tdataformat.c
View File

@ -557,13 +557,10 @@ void tRowDestroy(SRow *pRow) {
}
static int32_t tRowPCmprFn(const void *p1, const void *p2) {
if ((*(SRow **)p1)->ts < (*(SRow **)p2)->ts) {
return -1;
} else if ((*(SRow **)p1)->ts > (*(SRow **)p2)->ts) {
return 1;
}
return 0;
SRowKey key1, key2;
tRowGetKey(*(SRow **)p1, &key1);
tRowGetKey(*(SRow **)p2, &key2);
return tRowKeyCompare(&key1, &key2);
}
static void tRowPDestroy(SRow **ppRow) { tRowDestroy(*ppRow); }
static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart, int32_t iEnd, int8_t flag) {
@ -645,13 +642,18 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag) {
int32_t iStart = 0;
while (iStart < aRowP->size) {
SRow *pRow = (SRow *)taosArrayGetP(aRowP, iStart);
SRowKey key1;
SRow *row1 = (SRow *)taosArrayGetP(aRowP, iStart);
tRowGetKey(row1, &key1);
int32_t iEnd = iStart + 1;
while (iEnd < aRowP->size) {
SRow *pRowT = (SRow *)taosArrayGetP(aRowP, iEnd);
SRowKey key2;
SRow *row2 = (SRow *)taosArrayGetP(aRowP, iEnd);
tRowGetKey(row2, &key2);
if (pRow->ts != pRowT->ts) break;
if (tRowKeyCompare(&key1, &key2) != 0) break;
iEnd++;
}
@ -1284,6 +1286,27 @@ int32_t tRowKeyCompare(const void *p1, const void *p2) {
return 0;
}
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey* pSrc) {
pDst->ts = pSrc->ts;
pDst->numOfPKs = pSrc->numOfPKs;
if (pSrc->numOfPKs > 0) {
for (int32_t i = 0; i < pSrc->numOfPKs; ++i) {
SValue *pVal = &pDst->pks[i];
pVal->type = pSrc->pks[i].type;
if (IS_NUMERIC_TYPE(pVal->type)) {
pVal->val = pSrc->pks[i].val;
} else {
memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData);
pVal->nData = pSrc->pks[i].nData;
}
}
}
return TSDB_CODE_SUCCESS;
}
// STag ========================================
static int tTagValCmprFn(const void *p1, const void *p2) {
if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) {

2
source/dnode/vnode/inc/vnode.h
View File

@ -181,7 +181,7 @@ void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn not
int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr,
SArray* pFuncTypeList);
SArray* pFuncTypeList, SColumnInfo* pkCol, int32_t numOfPks);
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUids);
void *tsdbCacherowsReaderClose(void *pReader);

6
source/dnode/vnode/src/inc/tsdb.h
View File

@ -836,6 +836,8 @@ struct SLDataIter {
STimeWindow timeWindow;
SVersionRange verRange;
SSttBlockLoadInfo *pBlockLoadInfo;
SRowKey startRowKey; // current row key
__compar_fn_t comparFn;
bool ignoreEarlierTs;
struct SSttFileReader *pReader;
};
@ -846,7 +848,7 @@ struct SSttFileReader;
typedef int32_t (*_load_tomb_fn)(STsdbReader *pReader, struct SSttFileReader *pSttFileReader,
SSttBlockLoadInfo *pLoadInfo);
typedef struct {
typedef struct SMergeTreeConf {
int8_t backward;
STsdb *pTsdb;
uint64_t suid;
@ -859,7 +861,9 @@ typedef struct {
STSchema *pSchema;
int16_t *pCols;
int32_t numOfCols;
SRowKey *pCurRowKey;
_load_tomb_fn loadTombFn;
__compar_fn_t comparFn;
void *pReader;
void *idstr;
bool rspRows; // response the rows in stt-file, if possible

1
source/dnode/vnode/src/tq/tq.c
View File

@ -1195,7 +1195,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
return TSDB_CODE_SUCCESS;
}
} else {
// ASSERT(status == TASK_STATUS__HALT);
if (status != TASK_STATUS__HALT) {
tqError("s-task:%s should in halt status, let's halt it directly", pTask->id.idStr);
// streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT);

2
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -2288,6 +2288,8 @@ static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb
.loadTombFn = loadSttTomb,
.pReader = pr,
.idstr = pr->idstr,
.comparFn = pr->pkComparFn,
.pCurRowKey = &pr->rowKey,
};
code = tMergeTreeOpen2(&iter->mergeTree, &conf, NULL);

16
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -209,7 +209,7 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf
int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray* pCidList, int32_t* pSlotIds, uint64_t suid, void** pReader, const char* idstr,
SArray* pFuncTypeList) {
SArray* pFuncTypeList, SColumnInfo* pPkCol, int32_t numOfPks) {
*pReader = NULL;
SCacheRowsReader* p = taosMemoryCalloc(1, sizeof(SCacheRowsReader));
if (p == NULL) {
@ -226,6 +226,15 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
p->pSlotIds = pSlotIds;
p->pFuncTypeList = pFuncTypeList;
p->rowKey.numOfPKs = numOfPks;
if (numOfPks > 0) {
p->pkComparFn = getComparFunc(pPkCol->type, 0);
p->rowKey.pks[0].type = pPkCol->type;
if (IS_VAR_DATA_TYPE(pPkCol->type)) {
p->rowKey.pks[0].pData = taosMemoryCalloc(1, pPkCol->bytes);
}
}
if (numOfTables == 0) {
*pReader = p;
return TSDB_CODE_SUCCESS;
@ -359,10 +368,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
for (int32_t j = 0; j < pr->numOfCols; ++j) {
int32_t bytes;
if (slotIds[j] == -1)
if (slotIds[j] == -1) {
bytes = 1;
else
} else {
bytes = pr->pSchema->columns[slotIds[j]].bytes;
}
pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + bytes + VARSTR_HEADER_SIZE);
SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]);

33
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -365,7 +365,7 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl
}
if (pStatisBlkArray->data[k].maxTbid.suid == suid) {
taosArrayAddBatch(pBlockLoadInfo->info.pUid, tBufferGetDataAt(&block.suids, i * sizeof(int64_t)), rows - i);
taosArrayAddBatch(pBlockLoadInfo->info.pUid, tBufferGetDataAt(&block.uids, i * sizeof(int64_t)), rows - i);
taosArrayAddBatch(pBlockLoadInfo->info.pFirstKey,
tBufferGetDataAt(&block.firstKeyTimestamps, i * sizeof(int64_t)), rows - i);
taosArrayAddBatch(pBlockLoadInfo->info.pLastKey,
@ -479,6 +479,9 @@ int32_t tLDataIterOpen2(SLDataIter *pIter, SSttFileReader *pSttFileReader, int32
pIter->verRange.maxVer = pConf->verRange.maxVer;
pIter->timeWindow.skey = pConf->timewindow.skey;
pIter->timeWindow.ekey = pConf->timewindow.ekey;
pIter->comparFn = pConf->comparFn;
tRowKeyAssign(&pIter->startRowKey, pConf->pCurRowKey);
pIter->pReader = pSttFileReader;
pIter->pBlockLoadInfo = pBlockLoadInfo;
@ -621,15 +624,37 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) {
if (!pIter->backward) { // asc
if (ts > pIter->timeWindow.ekey) { // no more data
break;
} else if (ts < pIter->timeWindow.skey) {
} else {
if (ts < pIter->timeWindow.skey) {
continue;
}
if (ts == pIter->timeWindow.skey && pIter->startRowKey.numOfPKs > 0) {
SRowKey key;
tColRowGetKey(pData, i, &key);
int32_t ret = pkCompEx(pIter->comparFn, &key, &pIter->startRowKey);
if (ret < 0) {
continue;
}
}
}
} else {
if (ts < pIter->timeWindow.skey) {
break;
} else if (ts > pIter->timeWindow.ekey) {
} else {
if (ts > pIter->timeWindow.ekey) {
continue;
}
if (ts == pIter->timeWindow.ekey && pIter->startRowKey.numOfPKs > 0) {
SRowKey key;
tColRowGetKey(pData, i, &key);
int32_t ret = pkCompEx(pIter->comparFn, &key, &pIter->startRowKey);
if (ret > 0) {
continue;
}
}
}
}
int64_t ver = pData->aVersion[i];
@ -802,8 +827,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF
STimeWindow w = {0};
int64_t numOfRows = 0;
int64_t cid = pSttLevel->fobjArr->data[i]->f->cid;
code = tLDataIterOpen2(pIter, pSttFileReader, cid, pMTree->backward, pConf, pLoadInfo, &w, &numOfRows,
pMTree->idStr);
if (code != TSDB_CODE_SUCCESS) {

192
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -89,7 +89,7 @@ static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWi
static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus);
static int32_t pkCompEx(__compar_fn_t comparFn, SRowKey* p1, SRowKey* p2) {
int32_t pkCompEx(__compar_fn_t comparFn, SRowKey* p1, SRowKey* p2) {
if (p2 == NULL) {
return 1;
}
@ -118,12 +118,13 @@ static void tColRowGetKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slot
return;
}
pKey->numOfPKs = 1;
SColData* pColData = &pBlock->aColData[slotId];
SColVal cv;
tColDataGetValue(pColData, irow, &cv);
pKey->numOfPKs = 1;
pKey->pks[0].type = cv.value.type;
if (IS_NUMERIC_TYPE(cv.value.type)) {
pKey->pks[0].val = cv.value.val;
} else {
@ -141,13 +142,14 @@ static int32_t tGetPrimaryKeyIndex(uint8_t *p, SPrimaryKeyIndex *index) {
}
static void tRowGetKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
pKey->ts = pKey->ts;
pKey->ts = pRow->ts;
pKey->numOfPKs = pRow->numOfPKs;
if (pKey->numOfPKs == 0) {
return;
}
SPrimaryKeyIndex indices[TD_MAX_PK_COLS];
ASSERT(pKey->numOfPKs <= TD_MAX_PK_COLS);
uint8_t *data = pRow->data;
for (int32_t i = 0; i < pRow->numOfPKs; i++) {
@ -163,7 +165,7 @@ static void tRowGetKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
pKey->pks[i].pData = memcpy(pKey->pks[i].pData, data + indices[i].offset, pKey->pks[i].nData);
pKey->pks[i].pData += pKey->pks[i].nData;
} else {
pKey->pks[i].val = *(int64_t*) data + indices[i].offset;
pKey->pks[i].val = *(int64_t*) (data + indices[i].offset);
}
}
}
@ -648,7 +650,6 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
int32_t k = 0;
int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
int32_t step = asc ? 1 : -1;
STimeWindow w = pReader->info.window;
SBrinRecord* pRecord = NULL;
@ -686,13 +687,14 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr);
// todo: here we should find the first timestamp that is greater than the lastProcKey
// the window is an open interval NOW.
if (asc) {
w.skey = pScanInfo->lastProcKey.ts + step;
w.skey = pScanInfo->lastProcKey.ts;
} else {
w.ekey = pScanInfo->lastProcKey.ts + step;
w.ekey = pScanInfo->lastProcKey.ts;
}
if (isEmptyQueryTimeWindow(&w)) {
if (/*isEmptyQueryTimeWindow(&w)*/ w.ekey - w.skey < 1) { // NOTE: specialized for open interval
k += 1;
if (k >= numOfTables) {
@ -707,7 +709,17 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
continue;
}
// 2. version range check
if (asc) {
if (pkCompEx(pReader->pkComparFn, &pRecord->lastKey.key, &pScanInfo->lastProcKey) <= 0) {
continue;
}
} else {
if (pkCompEx(pReader->pkComparFn, &pRecord->firstKey.key, &pScanInfo->lastProcKey) >= 0) {
continue;
}
}
// 2. version range check, version range is an CLOSED interval
if (pRecord->minVer > pReader->info.verRange.maxVer || pRecord->maxVer < pReader->info.verRange.minVer) {
continue;
}
@ -724,6 +736,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
if (pScanInfo->filesetWindow.skey > pRecord->firstKey.key.ts) {
pScanInfo->filesetWindow.skey = pRecord->firstKey.key.ts;
}
if (pScanInfo->filesetWindow.ekey < pRecord->lastKey.key.ts) {
pScanInfo->filesetWindow.ekey = pRecord->lastKey.key.ts;
}
@ -1387,7 +1400,7 @@ static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, STableBlockScanI
int64_t key = 0;
if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) {
int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey;
int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;
key = ascScan ? TMIN(pBlock->firstKey, keyInStt) : TMAX(pBlock->lastKey, keyInStt);
} else {
key = ascScan ? pBlock->firstKey : pBlock->lastKey;
@ -1430,9 +1443,10 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo*
pInfo->hasDupTs = (pBlockInfo->numRow > pBlockInfo->count) || (pBlockInfo->count <= 0);
pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pRecord, pReader->info.order);
// todo handle the primary key overlap case
ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT);
if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) {
int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey;
int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;
pInfo->overlapWithSttBlock = !(pBlockInfo->lastKey < nextProcKeyInStt || pBlockInfo->firstKey > nextProcKeyInStt);
}
@ -1531,6 +1545,7 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc
SVersionRange* pVerRange) {
int32_t order = pSttBlockReader->order;
int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1;
SRowKey* pNextProc = &pScanInfo->sttKeyInfo.nextProcKey;
while (1) {
bool hasVal = tMergeTreeNext(&pSttBlockReader->mergeTree);
@ -1538,7 +1553,14 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc
pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;
// next file, the timestamps in the next file must be greater than those in current
pScanInfo->sttKeyInfo.nextProcKey += step;
pNextProc->ts += step;
if (pSttBlockReader->numOfPks > 0) {
if (IS_NUMERIC_TYPE(pNextProc->pks[0].type)) {
pNextProc->pks[0].val = INT64_MIN;
} else {
memset(pNextProc->pks[0].pData, 0, pNextProc->pks[0].nData);
}
}
return false;
}
@ -1552,7 +1574,8 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc
tColRowGetKeyDeepCopy(pRow->pBlockData, pRow->iRow, pkSrcSlot, &pSttBlockReader->currentKey);
}
pScanInfo->sttKeyInfo.nextProcKey = key;
tColRowGetKeyDeepCopy(pRow->pBlockData, pRow->iRow, pkSrcSlot, pNextProc);
if (pScanInfo->delSkyline != NULL && TARRAY_SIZE(pScanInfo->delSkyline) > 0) {
if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange)) {
pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA;
@ -1637,9 +1660,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
__compar_fn_t compFn = pReader->pkComparFn;
int32_t pkSrcSlot = pReader->suppInfo.pkSrcSlot;
SRowKey* pSttKey = NULL;
SRowKey* pSttKey = &(SRowKey){0};
if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) {
pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);
tRowKeyAssign(pSttKey, getCurrentKeyInSttBlock(pSttBlockReader));
} else {
pSttKey = NULL;
}
SRowKey k;
@ -1675,21 +1700,21 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
if (pReader->info.order == TSDB_ORDER_ASC) {
minKey = k; // chosen the minimum value
if (pkCompEx(compFn, pfKey, &minKey) < 0) {
if (pfKey != NULL && pkCompEx(compFn, pfKey, &minKey) < 0) {
minKey = *pfKey;
}
if (pkCompEx(compFn, pSttKey, &minKey) < 0) {
if (pSttKey != NULL && pkCompEx(compFn, pSttKey, &minKey) < 0) {
minKey = *pSttKey;
}
} else {
minKey = k;
if (pkCompEx(compFn, pfKey, &minKey) > 0) {
if (pfKey != NULL && pkCompEx(compFn, pfKey, &minKey) > 0) {
minKey = *pfKey;
}
if (pkCompEx(compFn, pSttKey, &minKey) > 0) {
if (pSttKey != NULL && pkCompEx(compFn, pSttKey, &minKey) > 0) {
minKey = *pSttKey;
}
}
@ -1792,6 +1817,7 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader*
}
}
// pKey == pSttKey
tRowKeyAssign(&pBlockScanInfo->lastProcKey, pKey);
// the following for key == sttKey->key.ts
@ -1810,7 +1836,8 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader*
return code;
}
doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pSttKey, pMerger, pkSrcSlot, &pReader->info.verRange,
// pSttKey will be changed when sttBlockReader iterates to the next row, so use pKey instead.
doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pKey, pMerger, pkSrcSlot, &pReader->info.verRange,
pReader->idStr);
code = tsdbRowMergerGetRow(pMerger, &pTSRow);
@ -1839,9 +1866,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader);
TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader);
SRowKey* pSttKey = NULL;
SRowKey* pSttKey = &(SRowKey){0};
if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) {
pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);
tRowKeyAssign(pSttKey, getCurrentKeyInSttBlock(pSttBlockReader));
} else {
pSttKey = NULL;
}
SRowKey* pfKey = &(SRowKey){0};
@ -1880,7 +1909,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
}
}
SRowKey minKey;
SRowKey minKey = {0};
if (ASCENDING_TRAVERSE(pReader->info.order)) {
minKey = k; // let's find the minimum
@ -1888,11 +1917,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
minKey = ik;
}
if (hasDataInFileBlock(pBlockData, pDumpInfo) && (pkCompEx(compFn, pfKey, &minKey) < 0)) {
if ((pfKey != NULL) && (pkCompEx(compFn, pfKey, &minKey) < 0)) {
minKey = *pfKey;
}
if (pSttKey != NULL && (pkCompEx(compFn, pSttKey, &minKey) < 0)) {
if ((pSttKey != NULL) && (pkCompEx(compFn, pSttKey, &minKey) < 0)) {
minKey = *pSttKey;
}
} else {
@ -1901,11 +1930,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
minKey = ik;
}
if (hasDataInFileBlock(pBlockData, pDumpInfo) && (pkCompEx(compFn, pfKey, &minKey) > 0)) {
if ((pfKey != NULL) && (pkCompEx(compFn, pfKey, &minKey) > 0)) {
minKey = *pfKey;
}
if (pSttKey != NULL && (pkCompEx(compFn, pSttKey, &minKey) > 0)) {
if ((pSttKey != NULL) && (pkCompEx(compFn, pSttKey, &minKey) > 0)) {
minKey = *pSttKey;
}
}
@ -2001,42 +2030,68 @@ int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScan
return code;
}
static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
SRowKey rowKey;
while (1) {
TSDBROW* pRow = getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader);
if (!pIter->hasVal) {
break;
}
tRowGetKeyEx(pRow, &rowKey);
int32_t ret = pkCompEx(pReader->pkComparFn, pKey, &rowKey);
if (ret == 0) {
pIter->hasVal = tsdbTbDataIterNext(pIter->iter);
} else {
break;
}
}
}
// handle the open interval issue. Find the first row key that is greater than the given one.
static int32_t forwardDataIter(SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
doForwardDataIter(pKey, &pBlockScanInfo->iter, pBlockScanInfo, pReader);
doForwardDataIter(pKey, &pBlockScanInfo->iiter, pBlockScanInfo, pReader);
return TSDB_CODE_SUCCESS;
}
static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
STbData* d = NULL;
STbData* di = NULL;
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
bool forward = true;
STsdbReadSnap* pSnap = pReader->pReadSnap;
STimeWindow* pWindow = &pReader->info.window;
if (pBlockScanInfo->iterInit) {
return TSDB_CODE_SUCCESS;
}
STbData* d = NULL;
STsdbRowKey startKey = {0};
if (ASCENDING_TRAVERSE(pReader->info.order)) {
startKey = (STsdbRowKey){.version = pReader->info.verRange.minVer,
.key = {
.ts = pBlockScanInfo->lastProcKey.ts + 1,
.numOfPKs = pReader->suppInfo.numOfPks,
}};
} else {
startKey = (STsdbRowKey){.version = pReader->info.verRange.maxVer,
.key = {
.ts = pBlockScanInfo->lastProcKey.ts - 1,
.numOfPKs = pReader->suppInfo.numOfPks,
}};
STsdbRowKey startKey;
startKey.key = pBlockScanInfo->lastProcKey;
startKey.version = asc ? pReader->info.verRange.minVer : pReader->info.verRange.maxVer;
if ((asc && (startKey.key.ts < pWindow->skey)) || ((!asc) && startKey.key.ts > pWindow->ekey)) {
startKey.key.ts = asc? pWindow->skey:pWindow->ekey;
forward = false;
}
int32_t code =
doInitMemDataIter(pReader, &d, pBlockScanInfo, &startKey, pReader->pReadSnap->pMem, &pBlockScanInfo->iter, "mem");
int32_t code = doInitMemDataIter(pReader, &d, pBlockScanInfo, &startKey, pSnap->pMem, &pBlockScanInfo->iter, "mem");
if (code != TSDB_CODE_SUCCESS) {
return code;
}
STbData* di = NULL;
code = doInitMemDataIter(pReader, &di, pBlockScanInfo, &startKey, pReader->pReadSnap->pIMem, &pBlockScanInfo->iiter,
"imem");
code = doInitMemDataIter(pReader, &di, pBlockScanInfo, &startKey, pSnap->pIMem, &pBlockScanInfo->iiter, "imem");
if (code != TSDB_CODE_SUCCESS) {
return code;
}
loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer);
if (forward) {
forwardDataIter(&startKey.key, pBlockScanInfo, pReader);
}
pBlockScanInfo->iterInit = true;
return TSDB_CODE_SUCCESS;
}
@ -2087,6 +2142,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STable
static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
bool hasData = true;
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
// the stt block reader has been initialized for this table.
if (pSttBlockReader->uid == pScanInfo->uid) {
@ -2105,10 +2161,10 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
}
STimeWindow w = pSttBlockReader->window;
if (ASCENDING_TRAVERSE(pSttBlockReader->order)) {
w.skey = pScanInfo->sttKeyInfo.nextProcKey;
if (asc) {
w.skey = pScanInfo->sttKeyInfo.nextProcKey.ts;
} else {
w.ekey = pScanInfo->sttKeyInfo.nextProcKey;
w.ekey = pScanInfo->sttKeyInfo.nextProcKey.ts;
}
int64_t st = taosGetTimestampUs();
@ -2129,6 +2185,8 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
.pCols = pReader->suppInfo.colId,
.numOfCols = pReader->suppInfo.numOfCols,
.loadTombFn = loadSttTombDataForAll,
.pCurRowKey = &pScanInfo->sttKeyInfo.nextProcKey,
.comparFn = pReader->pkComparFn,
.pReader = pReader,
.idstr = pReader->idStr,
.rspRows = (pReader->info.execMode == READER_EXEC_ROWS),
@ -2165,8 +2223,9 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
}
pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pTimeWindowList) ? STT_FILE_HAS_DATA : STT_FILE_NO_DATA;
pScanInfo->sttKeyInfo.nextProcKey =
ASCENDING_TRAVERSE(pReader->info.order) ? pScanInfo->sttWindow.skey : pScanInfo->sttWindow.ekey;
// todo set the primary key value
pScanInfo->sttKeyInfo.nextProcKey.ts = asc ? pScanInfo->sttWindow.skey : pScanInfo->sttWindow.ekey;
hasData = (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA);
} else { // not clean stt blocks
INIT_TIMEWINDOW(&pScanInfo->sttWindow); //reset the time window
@ -2727,7 +2786,7 @@ static void buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInf
setComposedBlockFlag(pReader, true);
pScanInfo->sttKeyInfo.nextProcKey = asc ? pScanInfo->sttWindow.ekey + 1 : pScanInfo->sttWindow.skey - 1;
pScanInfo->sttKeyInfo.nextProcKey.ts = asc ? pScanInfo->sttWindow.ekey + 1 : pScanInfo->sttWindow.skey - 1;
pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;
pScanInfo->lastProcKey.ts = asc ? pScanInfo->sttWindow.ekey : pScanInfo->sttWindow.skey;
@ -2887,7 +2946,7 @@ static bool notOverlapWithFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanI
if ((!hasDataInSttBlock(pScanInfo)) || (pScanInfo->cleanSttBlocks == true)) {
return true;
} else {
int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey;
int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;
return (asc && pBlockInfo->lastKey < keyInStt) || (!asc && pBlockInfo->firstKey > keyInStt);
}
}
@ -2935,7 +2994,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
code = buildDataBlockFromBuf(pReader, pScanInfo, endKey);
} else {
if (notOverlapWithFiles(pBlockInfo, pScanInfo, asc)) {
int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey;
int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;
if ((!hasDataInSttBlock(pScanInfo)) || (asc && pBlockInfo->lastKey < keyInStt) ||
(!asc && pBlockInfo->firstKey > keyInStt)) {
@ -3101,6 +3160,7 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter)
SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo;
if (pBlockInfo) {
// todo handle
// STableBlockScanInfo* pScanInfo = tSimpleHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
// if (pScanInfo) {
// tsdbRowKeyAssign(&pDumpInfo->lastKey, &pScanInfo->lastProcKey);
@ -3618,10 +3678,9 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI
TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
tsdbRowMergerAdd(pMerger, pRow1, NULL);
} else {
ASSERT(ret < 0);
tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid,
pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline),
pScanInfo->sttKeyInfo.nextProcKey, idStr);
pScanInfo->sttKeyInfo.nextProcKey.ts, idStr);
break;
}
}
@ -3658,7 +3717,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt
SRowKey nextRowKey = {0};
tRowGetKeyEx(pNextRow, &nextRowKey);
if (pkCompEx(pReader->pkComparFn, pKey, &nextRowKey) != 0) {
if (pKey->numOfPKs > 0 && pkCompEx(pReader->pkComparFn, pKey, &nextRowKey) != 0) {
*pResRow = current;
*freeTSRow = false;
return TSDB_CODE_SUCCESS;
@ -3965,6 +4024,9 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
if (row.type == TSDBROW_ROW_FMT) {
code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo);
if (code == TSDB_CODE_SUCCESS) {
tRowGetKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey);
}
if (freeTSRow) {
taosMemoryFree(row.pTSRow);
@ -3973,8 +4035,6 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
if (code) {
return code;
}
tRowGetKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey);
} else {
code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow);
if (code) {
@ -4116,6 +4176,9 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi
blockDataEnsureCapacity(pResBlock, capacity);
}
// for debug purpose
// capacity = 7;
int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
@ -4665,10 +4728,10 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) {
qTrace("tsdb/read: %p, unlock read mutex", pReader);
tsdbReleaseReader(pReader);
}
return code;
}
if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) {
// prepare for the next row scan
int32_t step = -1;
@ -4882,6 +4945,10 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) {
SReaderStatus* pStatus = &pTReader->status;
if (pStatus->composedDataBlock || pReader->info.execMode == READER_EXEC_ROWS) {
// tsdbReaderSuspend2(pReader);
// tsdbReaderResume2(pReader);
return pTReader->resBlockInfo.pResBlock;
}
@ -4890,6 +4957,9 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) {
qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader);
tsdbReleaseReader(pReader);
// tsdbReaderSuspend2(pReader);
// tsdbReaderResume2(pReader);
return ret;
}

72
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -130,25 +130,62 @@ STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, c
return *p;
}
static void initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader) {
SRowKey* pRowKey = &pScanInfo->lastProcKey;
if (ASCENDING_TRAVERSE(pReader->info.order)) {
int64_t skey = pReader->info.window.skey;
pRowKey->ts = (skey > INT64_MIN) ? (skey - 1) : skey;
pScanInfo->sttKeyInfo.nextProcKey = skey;
static int32_t initSRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) {
pKey->numOfPKs = numOfPks;
pKey->ts = ts;
if (numOfPks > 0) {
pKey->pks[0].type = type;
if (IS_NUMERIC_TYPE(pKey->pks[0].type)) {
char* p = (char*)&pKey->pks[0].val;
if (asc) {
switch(pKey->pks[0].type) {
case TSDB_DATA_TYPE_BIGINT:*(int64_t*)p = INT64_MIN;break;
case TSDB_DATA_TYPE_INT:*(int32_t*)p = INT32_MIN;break;
case TSDB_DATA_TYPE_SMALLINT:*(int16_t*)p = INT16_MIN;break;
case TSDB_DATA_TYPE_TINYINT:*(int8_t*)p = INT8_MIN;break;
}
} else {
int64_t ekey = pReader->info.window.ekey;
pRowKey->ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey;
pScanInfo->sttKeyInfo.nextProcKey = ekey;
switch(pKey->pks[0].type) {
case TSDB_DATA_TYPE_BIGINT:*(int64_t*)p = INT64_MAX;break;
case TSDB_DATA_TYPE_INT:*(int32_t*)p = INT32_MAX;break;
case TSDB_DATA_TYPE_SMALLINT:*(int16_t*)p = INT16_MAX;break;
case TSDB_DATA_TYPE_TINYINT:*(int8_t*)p = INT8_MAX;break;
}
}
} else {
pKey->pks[0].pData = taosMemoryCalloc(1, len);
pKey->pks[0].nData = 0;
if (pKey->pks[0].pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
}
// only handle the first primary key.
pRowKey->numOfPKs = pReader->suppInfo.numOfPks;
if (pReader->suppInfo.numOfPks > 0) {
if (IS_VAR_DATA_TYPE(pReader->suppInfo.pk.type)) {
pRowKey->pks[0].pData = taosMemoryCalloc(1, pReader->suppInfo.pk.bytes);
return TSDB_CODE_SUCCESS;
}
pRowKey->pks[0].type = pReader->suppInfo.pk.type;
static void initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader) {
int32_t numOfPks = pReader->suppInfo.numOfPks;
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
SRowKey* pRowKey = &pScanInfo->lastProcKey;
if (asc) {
int64_t skey = pReader->info.window.skey;
int64_t ts = (skey > INT64_MIN) ? (skey - 1) : skey;
initSRowKey(pRowKey, ts, numOfPks, pReader->suppInfo.pk.type, pReader->suppInfo.pk.bytes, asc);
initSRowKey(&pScanInfo->sttKeyInfo.nextProcKey, skey, numOfPks, pReader->suppInfo.pk.type,
pReader->suppInfo.pk.bytes, asc);
} else {
int64_t ekey = pReader->info.window.ekey;
int64_t ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey;
initSRowKey(pRowKey, ts, numOfPks, pReader->suppInfo.pk.type, pReader->suppInfo.pk.bytes, asc);
initSRowKey(&pScanInfo->sttKeyInfo.nextProcKey, ekey, numOfPks, pReader->suppInfo.pk.type,
pReader->suppInfo.pk.bytes, asc);
}
}
@ -229,9 +266,8 @@ void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {
pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline);
pInfo->lastProcKey.ts = ts;
ASSERT(0);
pInfo->sttKeyInfo.nextProcKey = ts + step;
// todo check the nextProcKey info
pInfo->sttKeyInfo.nextProcKey.ts = ts + step;
}
}

7
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -77,7 +77,9 @@ typedef enum ESttKeyStatus {
typedef struct SSttKeyInfo {
ESttKeyStatus status; // this value should be updated when switch to the next fileset
int64_t nextProcKey; // todo remove this attribute, since it is impossible to set correct nextProcKey value
SRowKey nextProcKey;
// int64_t nextProcKey; // todo remove this attribute, since it is impossible to set correct nextProcKey
// value
} SSttKeyInfo;
// clean stt file blocks:
@ -333,6 +335,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
const char* pstr);
bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, int32_t order);
bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order);
int32_t pkCompEx(__compar_fn_t comparFn, SRowKey* p1, SRowKey* p2);
typedef struct {
SArray* pTombData;
@ -366,6 +369,8 @@ typedef struct SCacheRowsReader {
char* idstr;
int64_t lastTs;
SArray* pFuncTypeList;
__compar_fn_t pkComparFn;
SRowKey rowKey;
} SCacheRowsReader;
int32_t tsdbCacheGetBatch(STsdb* pTsdb, tb_uid_t uid, SArray* pLastArray, SCacheRowsReader* pr, int8_t ltype);

21
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -628,27 +628,6 @@ void tColRowGetKey(SBlockData* pBlock, int32_t irow, SRowKey* key) {
}
}
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey* pSrc) {
pDst->ts = pSrc->ts;
pDst->numOfPKs = pSrc->numOfPKs;
if (pSrc->numOfPKs > 0) {
for (int32_t i = 0; i < pDst->numOfPKs; ++i) {
SValue *pVal = &pDst->pks[i];
pVal->type = pSrc->pks[i].type;
if (IS_NUMERIC_TYPE(pVal->type)) {
pVal->val = pSrc->pks[i].val;
} else {
memcpy(pVal->pData, pVal->pData, pVal->nData);
pVal->nData = pSrc->pks[i].nData;
}
}
}
return TSDB_CODE_SUCCESS;
}
int32_t tsdbRowKeyCmpr(const STsdbRowKey *key1, const STsdbRowKey *key2) {
int32_t c = tRowKeyCompare(&key1->key, &key2->key);

26
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -1586,12 +1586,26 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in
} else {
int32_t nRow = TARRAY_SIZE(pSubmitTbData->aRowP);
SRow **aRow = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP);
SRowKey lastRowKey;
for (int32_t iRow = 0; iRow < nRow; ++iRow) {
if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey || (iRow > 0 && aRow[iRow]->ts <= aRow[iRow - 1]->ts)) {
if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey) {
code = TSDB_CODE_INVALID_MSG;
vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), ver);
goto _exit;
}
if (iRow == 0) {
tRowGetKey(aRow[iRow], &lastRowKey);
} else {
SRowKey rowKey;
tRowGetKey(aRow[iRow], &rowKey);
if (tRowKeyCompare(&lastRowKey, &rowKey) >= 0) {
code = TSDB_CODE_INVALID_MSG;
vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), ver);
goto _exit;
}
lastRowKey = rowKey;
}
}
}
}
@ -1736,9 +1750,13 @@ _exit:
atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1);
if (tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0) {
const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId,
pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId,
pOriginalMsg->info.conn.user, "Success"};
const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS,
pVnode->monitor.strClusterId,
pVnode->monitor.strDnodeId,
tsLocalEp,
pVnode->monitor.strVgId,
pOriginalMsg->info.conn.user,
"Success"};
taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels);
}

6
source/libs/command/src/command.c
View File

@ -501,10 +501,14 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
} else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
}
char* pk = (pSchema->flags & COL_IS_KEY) ? "PRIMARY KEY" : "";
if (!(pSchema->flags & COL_IS_KEY)) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type);
} else {
char* pk = "PRIMARY KEY";
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s %s", ((i > 0) ? ", " : ""), pSchema->name, type, pk);
}
}
}
void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfTags; ++i) {

17
source/libs/executor/src/cachescanoperator.c
View File

@ -46,6 +46,8 @@ typedef struct SCacheRowsScanInfo {
int32_t indexOfBufferedRes;
STableListInfo* pTableList;
SArray* pFuncTypeList;
int32_t numOfPks;
SColumnInfo pkCol;
} SCacheRowsScanInfo;
static SSDataBlock* doScanCache(SOperatorInfo* pOperator);
@ -106,6 +108,16 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
goto _error;
}
for(int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) {
SColMatchItem* pItem = taosArrayGet(pInfo->matchInfo.pList, i);
if (pItem->isPk) {
pInfo->numOfPks += 1;
pInfo->pkCol.type = pItem->dataType.type; // only record one primary key
pInfo->pkCol.bytes = pItem->dataType.bytes; // only record one primary key
pInfo->pkCol.pk = 1;
}
}
SArray* pCidList = taosArrayInit(numOfCols, sizeof(int16_t));
pInfo->pFuncTypeList = taosArrayInit(taosArrayGetSize(pScanNode->pFuncTypes), sizeof(int32_t));
taosArrayAddAll(pInfo->pFuncTypeList, pScanNode->pFuncTypes);
@ -140,7 +152,8 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
uint64_t suid = tableListGetSuid(pTableListInfo);
code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables,
taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds,
suid, &pInfo->pLastrowReader, pTaskInfo->id.str, pScanNode->pFuncTypes);
suid, &pInfo->pLastrowReader, pTaskInfo->id.str, pScanNode->pFuncTypes,
&pInfo->pkCol, pInfo->numOfPks);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -282,7 +295,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
if (NULL == pInfo->pLastrowReader) {
code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader,
pTaskInfo->id.str, pInfo->pFuncTypeList);
pTaskInfo->id.str, pInfo->pFuncTypeList, &pInfo->pkCol, pInfo->numOfPks);
if (code != TSDB_CODE_SUCCESS) {
pInfo->currentGroupIndex += 1;
taosArrayClear(pInfo->pUidList);

2
source/libs/executor/src/scanoperator.c
View File

@ -201,7 +201,7 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo*
SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset);
int32_t reqStatus = fmFuncDynDataRequired(functionId, pEntry, &pBlockInfo->window);
int32_t reqStatus = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo);
if (reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) {
notLoadBlock = false;
break;

85
source/libs/executor/src/timesliceoperator.c
View File

@ -39,13 +39,15 @@ typedef struct STimeSliceOperatorInfo {
SColumn tsCol; // primary timestamp column
SExprSupp scalarSup; // scalar calculation
struct SFillColInfo* pFillColInfo; // fill column info
int64_t prevTs;
SRowKey prevKey;
bool prevTsSet;
uint64_t groupId;
SGroupKeys* pPrevGroupKey;
SSDataBlock* pNextGroupRes;
SSDataBlock* pRemainRes; // save block unfinished processing
int32_t remainIndex; // the remaining index in the block to be processed
bool hasPk;
SColumn pkCol;
} STimeSliceOperatorInfo;
static void destroyTimeSliceOperatorInfo(void* param);
@ -176,22 +178,49 @@ static bool isIsfilledPseudoColumn(SExprInfo* pExprInfo) {
return (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type) && strcasecmp(name, "_isfilled") == 0);
}
static void tRowGetKeyFromColData(int64_t ts, SColumnInfoData* pPkCol, int32_t rowIndex, SRowKey* pKey) {
pKey->ts = ts;
pKey->numOfPKs = 1;
int8_t t = pPkCol->info.type;
pKey->pks[0].type = t;
if (IS_NUMERIC_TYPE(t)) {
GET_TYPED_DATA(pKey->pks[0].val, int64_t, t, colDataGetNumData(pPkCol, rowIndex));
} else {
char* p = colDataGetVarData(pPkCol, rowIndex);
pKey->pks[0].pData = (uint8_t*)varDataVal(p);
pKey->pks[0].nData = varDataLen(p);
}
}
static bool checkDuplicateTimestamps(STimeSliceOperatorInfo* pSliceInfo, SColumnInfoData* pTsCol,
int32_t curIndex, int32_t rows) {
SColumnInfoData* pPkCol, int32_t curIndex, int32_t rows) {
int64_t currentTs = *(int64_t*)colDataGetData(pTsCol, curIndex);
if (currentTs > pSliceInfo->win.ekey) {
return false;
}
if ((pSliceInfo->prevTsSet == true) && (currentTs == pSliceInfo->prevTs)) {
SRowKey cur = {.ts = currentTs, .numOfPKs = (pPkCol != NULL)? 1:0};
if (pPkCol != NULL) {
cur.pks[0].type = pPkCol->info.type;
}
if ((pSliceInfo->prevTsSet == true) && (currentTs == pSliceInfo->prevKey.ts)) {
// if (pPkCol == NULL) {
return true;
/* } else {
tRowGetKeyFromColData(currentTs, pPkCol, curIndex, &cur);
if (tRowKeyCompare(&cur, &pSliceInfo->prevKey) == 0) {
return true;
}
}*/
}
pSliceInfo->prevTsSet = true;
pSliceInfo->prevTs = currentTs;
tRowKeyAssign(&pSliceInfo->prevKey, &cur);
// todo handle next
if (currentTs == pSliceInfo->win.ekey && curIndex < rows - 1) {
int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, curIndex + 1);
if (currentTs == nextTs) {
@ -693,14 +722,20 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS
SInterval* pInterval = &pSliceInfo->interval;
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
SColumnInfoData* pPkCol = NULL;
if (pSliceInfo->hasPk) {
pPkCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->pkCol.slotId);
}
int32_t i = (pSliceInfo->pRemainRes == NULL) ? 0 : pSliceInfo->remainIndex;
for (; i < pBlock->info.rows; ++i) {
int64_t ts = *(int64_t*)colDataGetData(pTsCol, i);
// check for duplicate timestamps
if (checkDuplicateTimestamps(pSliceInfo, pTsCol, i, pBlock->info.rows)) {
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_FUNC_DUP_TIMESTAMP);
if (checkDuplicateTimestamps(pSliceInfo, pTsCol, pPkCol, i, pBlock->info.rows)) {
continue;
// T_LONG_JMP(pTaskInfo->env, TSDB_CODE_FUNC_DUP_TIMESTAMP);
}
if (checkNullRow(&pOperator->exprSupp, pBlock, i, ignoreNull)) {
@ -873,11 +908,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
return NULL;
}
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
SSDataBlock* pResBlock = pSliceInfo->pRes;
SOperatorInfo* downstream = pOperator->pDownstream[0];
blockDataCleanup(pResBlock);
while (1) {
@ -953,6 +986,25 @@ _finished:
return pResBlock->info.rows == 0 ? NULL : pResBlock;
}
static int32_t extractPkColumnFromFuncs(SNodeList* pFuncs, bool* pHasPk, SColumn* pPkColumn) {
SNode* pNode;
FOREACH(pNode, pFuncs) {
if ((nodeType(pNode) == QUERY_NODE_TARGET) &&
(nodeType(((STargetNode*)pNode)->pExpr) == QUERY_NODE_FUNCTION)) {
SFunctionNode* pFunc = (SFunctionNode*)((STargetNode*)pNode)->pExpr;
if (fmIsInterpFunc(pFunc->funcId) && pFunc->hasPk) {
SNode* pNode2 = (pFunc->pParameterList->pTail->pNode);
if ((nodeType(pNode2) == QUERY_NODE_COLUMN) && ((SColumnNode*)pNode2)->isPk) {
*pHasPk = true;
*pPkColumn = extractColumnFromColumnNode((SColumnNode*)pNode2);
break;
}
}
}
}
return TSDB_CODE_SUCCESS;
}
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@ -985,6 +1037,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
}
pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries);
extractPkColumnFromFuncs(pInterpPhyNode->pFuncs, &pInfo->hasPk, &pInfo->pkCol);
pInfo->fillType = convertFillType(pInterpPhyNode->fillMode);
initResultSizeInfo(&pOperator->resultInfo, 4096);
@ -995,13 +1048,23 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
pInfo->interval.interval = pInterpPhyNode->interval;
pInfo->current = pInfo->win.skey;
pInfo->prevTsSet = false;
pInfo->prevTs = 0;
pInfo->prevKey.ts = INT64_MIN;
pInfo->groupId = 0;
pInfo->pPrevGroupKey = NULL;
pInfo->pNextGroupRes = NULL;
pInfo->pRemainRes = NULL;
pInfo->remainIndex = 0;
if (pInfo->hasPk) {
pInfo->prevKey.numOfPKs = 1;
pInfo->prevKey.ts = INT64_MIN;
pInfo->prevKey.pks[0].type = pInfo->pkCol.type;
if (IS_VAR_DATA_TYPE(pInfo->pkCol.type)) {
pInfo->prevKey.pks[0].pData = taosMemoryCalloc(1, pInfo->pkCol.bytes);
}
}
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info;
pScanInfo->base.cond.twindows = pInfo->win;

5
source/libs/executor/src/timewindowoperator.c
View File

@ -658,11 +658,12 @@ static bool isCalculatedWin(SIntervalAggOperatorInfo* pInfo, const STimeWindow*
*/
static bool filterWindowWithLimit(SIntervalAggOperatorInfo* pOperatorInfo, STimeWindow* win, uint64_t groupId) {
if (!pOperatorInfo->limited // if no limit info, no filter will be applied
|| pOperatorInfo->binfo.inputTsOrder !=
pOperatorInfo->binfo.outputTsOrder // if input/output ts order mismatch, no filter
|| pOperatorInfo->binfo.inputTsOrder != pOperatorInfo->binfo.outputTsOrder
// if input/output ts order mismatch, no filter
) {
return false;
}
if (pOperatorInfo->limit == 0) return true;
if (pOperatorInfo->pBQ == NULL) {

5
source/libs/executor/src/tsort.c
View File

@ -1306,6 +1306,7 @@ static void appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSource
static void initRowIdSort(SSortHandle* pHandle) {
SBlockOrderInfo* pkOrder = (pHandle->bSortPk) ? taosArrayGet(pHandle->aExtRowsOrders, 1) : NULL;
SColumnInfoData* extPkCol = (pHandle->bSortPk) ? taosArrayGet(pHandle->pDataBlock->pDataBlock, pkOrder->slotId) : NULL;
SColumnInfoData pkCol = {0};
SSDataBlock* pSortInput = createDataBlock();
SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1);
@ -1317,7 +1318,7 @@ static void initRowIdSort(SSortHandle* pHandle) {
SColumnInfoData lengthCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 4);
blockDataAppendColInfo(pSortInput, &lengthCol);
if (pHandle->bSortPk) {
SColumnInfoData pkCol = createColumnInfoData(extPkCol->info.type, extPkCol->info.bytes, 5);
pkCol = createColumnInfoData(extPkCol->info.type, extPkCol->info.bytes, 5);
blockDataAppendColInfo(pSortInput, &pkCol);
}
blockDataDestroy(pHandle->pDataBlock);
@ -1343,7 +1344,7 @@ static void initRowIdSort(SSortHandle* pHandle) {
biPk.order = pkOrder->order;
biPk.slotId = 4;
biPk.nullFirst = (biPk.order == TSDB_ORDER_ASC);
biPk.compFn = getKeyComparFunc(extPkCol->info.type, biPk.order);
biPk.compFn = getKeyComparFunc(pkCol.info.type, biPk.order);
taosArrayPush(aOrder, &biPk);
}
taosArrayDestroy(pHandle->pSortInfo);

2
source/libs/function/inc/builtins.h
View File

@ -25,7 +25,7 @@ extern "C" {
typedef int32_t (*FTranslateFunc)(SFunctionNode* pFunc, char* pErrBuf, int32_t len);
typedef EFuncDataRequired (*FFuncDataRequired)(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
typedef int32_t (*FCreateMergeFuncParameters)(SNodeList* pRawParameters, SNode* pPartialRes, SNodeList** pParameters);
typedef EFuncDataRequired (*FFuncDynDataRequired)(void* pRes, STimeWindow* pTimeWindow);
typedef EFuncDataRequired (*FFuncDynDataRequired)(void* pRes, SDataBlockInfo* pBlocInfo);
typedef EFuncReturnRows (*FEstimateReturnRows)(SFunctionNode* pFunc);
typedef struct SBuiltinFuncDefinition {

6
source/libs/function/inc/builtinsimpl.h
View File

@ -144,7 +144,7 @@ int32_t irateFunction(SqlFunctionCtx* pCtx);
int32_t irateFunctionMerge(SqlFunctionCtx* pCtx);
int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t iratePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t getIrateInfoSize();
int32_t getIrateInfoSize(int32_t pkBytes);
int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx);
@ -158,8 +158,8 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pB
int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t getFirstLastInfoSize(int32_t resBytes, int32_t pkBytes);
EFuncDataRequired firstDynDataReq(void* pRes, STimeWindow* pTimeWindow);
EFuncDataRequired lastDynDataReq(void* pRes, STimeWindow* pTimeWindow);
EFuncDataRequired firstDynDataReq(void* pRes, SDataBlockInfo* pBlockInfo);
EFuncDataRequired lastDynDataReq(void* pRes, SDataBlockInfo* pBlockInfo);
int32_t lastRowFunction(SqlFunctionCtx* pCtx);

5
source/libs/function/src/builtins.c
View File

@ -1632,13 +1632,14 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
static int32_t translateIrateImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
uint8_t colType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (isPartial) {
if (3 != LIST_LENGTH(pFunc->pParameterList)) {
if (3 != LIST_LENGTH(pFunc->pParameterList) && 4 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
if (!IS_NUMERIC_TYPE(colType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getIrateInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
int32_t pkBytes = (pFunc->hasPk) ? pFunc->pkBytes : 0;
pFunc->node.resType = (SDataType){.bytes = getIrateInfoSize(pkBytes) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
} else {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);

205
source/libs/function/src/builtinsimpl.c
View File

@ -269,6 +269,12 @@ typedef struct SRateInfo {
double lastValue;
TSKEY lastKey;
int8_t hasResult; // flag to denote has value
char* firstPk;
char* lastPk;
int8_t pkType;
int32_t pkBytes;
char pkData[];
} SRateInfo;
typedef struct SGroupKeyInfo {
@ -424,10 +430,16 @@ int32_t funcInputUpdate(SqlFunctionCtx* pCtx) {
if (!pCtx->bInputFinished) {
pIter->pInput = &pCtx->input;
pIter->tsList = (TSKEY*)pIter->pInput->pPTS->pData;
pIter->pData = pIter->pInput->pData[0];
pIter->pDataCol = pIter->pInput->pData[0];
pIter->pPkCol = pIter->pInput->pPrimaryKey;
pIter->rowIndex = pIter->pInput->startRowIndex;
pIter->inputEndIndex = pIter->rowIndex + pIter->pInput->numOfRows - 1;
pIter->pSrcBlock = pCtx->pSrcBlock;
if (!pIter->hasGroupId || pIter->groupId != pIter->pSrcBlock->info.id.groupId) {
pIter->hasGroupId = true;
pIter->groupId = pIter->pSrcBlock->info.id.groupId;
pIter->hasPrev = false;
}
} else {
pIter->finalRow = true;
}
@ -454,10 +466,16 @@ bool funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow) {
if (pIter->prevBlockTsEnd == pIter->tsList[pIter->inputEndIndex]) {
blockDataDestroy(pIter->pPrevRowBlock);
pIter->pPrevRowBlock = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1);
pIter->prevIsDataNull = colDataIsNull_f(pIter->pData->nullbitmap, pIter->inputEndIndex);
pIter->pPrevData = taosMemoryMalloc(pIter->pData->info.bytes);
char* srcData = colDataGetData(pIter->pData, pIter->inputEndIndex);
memcpy(pIter->pPrevData, srcData, pIter->pData->info.bytes);
pIter->prevIsDataNull = colDataIsNull_f(pIter->pDataCol->nullbitmap, pIter->inputEndIndex);
pIter->pPrevData = taosMemoryMalloc(pIter->pDataCol->info.bytes);
char* srcData = colDataGetData(pIter->pDataCol, pIter->inputEndIndex);
memcpy(pIter->pPrevData, srcData, pIter->pDataCol->info.bytes);
pIter->pPrevPk = taosMemoryMalloc(pIter->pPkCol->info.bytes);
char* pkData = colDataGetData(pIter->pPkCol, pIter->inputEndIndex);
memcpy(pIter->pPrevPk, pkData, pIter->pPkCol->info.bytes);
pIter->pPrevRowBlock = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1);
pIter->hasPrev = true;
@ -475,8 +493,9 @@ bool funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow) {
pRow->rowIndex = 0;
} else {
pRow->ts = pIter->tsList[idx - 1];
pRow->isDataNull = colDataIsNull_f(pIter->pData->nullbitmap, idx - 1);
pRow->pData = colDataGetData(pIter->pData, idx - 1);
pRow->isDataNull = colDataIsNull_f(pIter->pDataCol->nullbitmap, idx - 1);
pRow->pData = colDataGetData(pIter->pDataCol, idx - 1);
pRow->pPk = colDataGetData(pIter->pPkCol, idx - 1);
pRow->block = pIter->pSrcBlock;
pRow->rowIndex = idx - 1;
}
@ -492,20 +511,22 @@ bool funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow) {
++idx;
}
pRow->ts = pIter->tsList[idx];
pRow->isDataNull = colDataIsNull_f(pIter->pData->nullbitmap, idx);
pRow->pData = colDataGetData(pIter->pData, pIter->rowIndex);
pRow->isDataNull = colDataIsNull_f(pIter->pDataCol->nullbitmap, idx);
pRow->pData = colDataGetData(pIter->pDataCol, idx);
pRow->pPk = colDataGetData(pIter->pPkCol, idx);
pRow->block = pIter->pSrcBlock;
pRow->rowIndex = idx;
pIter->rowIndex = idx + 1;
return true;
} else {
pIter->hasPrev = true;
pIter->prevBlockTsEnd = tsEnd;
// TODO
pIter->prevIsDataNull = colDataIsNull_f(pIter->pData->nullbitmap, pIter->inputEndIndex);
pIter->pPrevData = taosMemoryMalloc(pIter->pData->info.bytes);
memcpy(pIter->pPrevData, colDataGetData(pIter->pData, pIter->inputEndIndex), pIter->pData->info.bytes);
pIter->prevIsDataNull = colDataIsNull_f(pIter->pDataCol->nullbitmap, pIter->inputEndIndex);
pIter->pPrevData = taosMemoryMalloc(pIter->pDataCol->info.bytes);
memcpy(pIter->pPrevData, colDataGetData(pIter->pDataCol, pIter->inputEndIndex), pIter->pDataCol->info.bytes);
pIter->pPrevPk = taosMemoryMalloc(pIter->pPkCol->info.bytes);
memcpy(pIter->pPrevPk, colDataGetData(pIter->pPkCol, pIter->inputEndIndex), pIter->pPkCol->info.bytes);
pIter->pPrevRowBlock = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1);
return false;
}
@ -523,8 +544,9 @@ bool funcInputGetNextRowAscPk(SFuncInputRowIter *pIter, SFuncInputRow* pRow) {
++idx;
}
pRow->ts = pIter->tsList[idx];
pRow->isDataNull = colDataIsNull_f(pIter->pData->nullbitmap, idx);
pRow->pData = colDataGetData(pIter->pData, idx);
pRow->isDataNull = colDataIsNull_f(pIter->pDataCol->nullbitmap, idx);
pRow->pData = colDataGetData(pIter->pDataCol, idx);
pRow->pPk = colDataGetData(pIter->pPkCol, idx);
pRow->block = pIter->pSrcBlock;
pRow->rowIndex = idx;
@ -535,8 +557,9 @@ bool funcInputGetNextRowAscPk(SFuncInputRowIter *pIter, SFuncInputRow* pRow) {
} else {
if (pIter->rowIndex <= pIter->inputEndIndex) {
pRow->ts = pIter->tsList[pIter->rowIndex];
pRow->isDataNull = colDataIsNull_f(pIter->pData->nullbitmap, pIter->rowIndex);
pRow->pData = colDataGetData(pIter->pData, pIter->rowIndex);
pRow->isDataNull = colDataIsNull_f(pIter->pDataCol->nullbitmap, pIter->rowIndex);
pRow->pData = colDataGetData(pIter->pDataCol, pIter->rowIndex);
pRow->pPk = colDataGetData(pIter->pPkCol, pIter->rowIndex);
pRow->block = pIter->pSrcBlock;
pRow->rowIndex = pIter->rowIndex;
@ -563,8 +586,9 @@ bool funcInputGetNextRowAscPk(SFuncInputRowIter *pIter, SFuncInputRow* pRow) {
bool funcInputGetNextRowNoPk(SFuncInputRowIter *pIter, SFuncInputRow* pRow) {
if (pIter->rowIndex <= pIter->inputEndIndex) {
pRow->ts = pIter->tsList[pIter->rowIndex];
pRow->isDataNull = colDataIsNull_f(pIter->pData->nullbitmap, pIter->rowIndex);
pRow->pData = colDataGetData(pIter->pData, pIter->rowIndex);
pRow->isDataNull = colDataIsNull_f(pIter->pDataCol->nullbitmap, pIter->rowIndex);
pRow->pData = colDataGetData(pIter->pDataCol, pIter->rowIndex);
pRow->pPk = NULL;
pRow->block = pIter->pSrcBlock;
pRow->rowIndex = pIter->rowIndex;
@ -2290,7 +2314,31 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx)
return TSDB_CODE_SUCCESS;
}
EFuncDataRequired firstDynDataReq(void* pRes, STimeWindow* pTimeWindow) {
// TODO: change this function when block data info pks changed
static int32_t comparePkDataWithSValue(int8_t pkType, char* pkData, SValue* pVal, int32_t order) {
char numVal[8] = {0};
switch (pkType) {
case TSDB_DATA_TYPE_INT:
*(int32_t*)numVal = (int32_t)pVal->val;
break;
case TSDB_DATA_TYPE_UINT:
*(uint32_t*)numVal = (uint32_t)pVal->val;
break;
case TSDB_DATA_TYPE_BIGINT:
*(int64_t*)numVal = (int64_t)pVal->val;
break;
case TSDB_DATA_TYPE_UBIGINT:
*(uint64_t*)numVal = (uint64_t)pVal->val;
break;
default:
break;
}
char* blockData = (IS_NUMERIC_TYPE(pkType)) ? (char*) numVal : (char*)pVal->pData;
__compar_fn_t fn = getKeyComparFunc(pkType, order);
return fn(pkData, blockData);
}
EFuncDataRequired firstDynDataReq(void* pRes, SDataBlockInfo* pBlockInfo) {
SResultRowEntryInfo* pEntry = (SResultRowEntryInfo*)pRes;
// not initialized yet, data is required
@ -2299,14 +2347,26 @@ EFuncDataRequired firstDynDataReq(void* pRes, STimeWindow* pTimeWindow) {
}
SFirstLastRes* pResult = GET_ROWCELL_INTERBUF(pEntry);
if (pResult->hasResult && pResult->ts <= pTimeWindow->skey) {
if (pResult->hasResult) {
if (pResult->pkBytes > 0) {
pResult->pkData = pResult->buf + pResult->bytes;
} else {
pResult->pkData = NULL;
}
if (pResult->ts < pBlockInfo->window.skey) {
return FUNC_DATA_REQUIRED_NOT_LOAD;
} else if (pResult->ts == pBlockInfo->window.skey && pResult->pkData) {
if (comparePkDataWithSValue(pResult->pkType, pResult->pkData, pBlockInfo->pks + 0, TSDB_ORDER_ASC) < 0) {
return FUNC_DATA_REQUIRED_NOT_LOAD;
}
}
return FUNC_DATA_REQUIRED_DATA_LOAD;
} else {
return FUNC_DATA_REQUIRED_DATA_LOAD;
}
}
EFuncDataRequired lastDynDataReq(void* pRes, STimeWindow* pTimeWindow) {
EFuncDataRequired lastDynDataReq(void* pRes, SDataBlockInfo* pBlockInfo) {
SResultRowEntryInfo* pEntry = (SResultRowEntryInfo*)pRes;
// not initialized yet, data is required
@ -2315,8 +2375,20 @@ EFuncDataRequired lastDynDataReq(void* pRes, STimeWindow* pTimeWindow) {
}
SFirstLastRes* pResult = GET_ROWCELL_INTERBUF(pEntry);
if (pResult->hasResult && pResult->ts >= pTimeWindow->ekey) {
if (pResult->hasResult) {
if (pResult->pkBytes > 0) {
pResult->pkData = pResult->buf + pResult->bytes;
} else {
pResult->pkData = NULL;
}
if (pResult->ts > pBlockInfo->window.ekey) {
return FUNC_DATA_REQUIRED_NOT_LOAD;
} else if (pResult->ts == pBlockInfo->window.ekey && pResult->pkData) {
if (comparePkDataWithSValue(pResult->pkType, pResult->pkData, pBlockInfo->pks + 1, TSDB_ORDER_DESC) < 0) {
return FUNC_DATA_REQUIRED_NOT_LOAD;
}
}
return FUNC_DATA_REQUIRED_DATA_LOAD;
} else {
return FUNC_DATA_REQUIRED_DATA_LOAD;
}
@ -6114,10 +6186,11 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
int32_t getIrateInfoSize() { return (int32_t)sizeof(SRateInfo); }
int32_t getIrateInfoSize(int32_t pkBytes) { return (int32_t)sizeof(SRateInfo) + 2 * pkBytes; }
bool getIrateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SRateInfo);
int32_t pkBytes = (pFunc->hasPk) ? pFunc->pkBytes : 0;
pEnv->calcMemSize = getIrateInfoSize(pkBytes);
return true;
}
@ -6137,6 +6210,41 @@ bool irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) {
return true;
}
static void doSaveRateInfo(SRateInfo* pRateInfo, bool isFirst, int64_t ts, char* pk, double v) {
if (isFirst) {
pRateInfo->firstValue = v;
pRateInfo->firstKey = ts;
if (pRateInfo->firstPk) {
int32_t pkBytes = IS_VAR_DATA_TYPE(pRateInfo->pkType) ? varDataTLen(pk) : pRateInfo->pkBytes;
memcpy(pRateInfo->firstPk, pk, pkBytes);
}
} else {
pRateInfo->lastValue = v;
pRateInfo->lastKey = ts;
if (pRateInfo->lastPk) {
int32_t pkBytes = IS_VAR_DATA_TYPE(pRateInfo->pkType) ? varDataTLen(pk) : pRateInfo->pkBytes;
memcpy(pRateInfo->lastPk, pk, pkBytes);
}
}
}
static void initializeRateInfo(SqlFunctionCtx* pCtx, SRateInfo* pRateInfo, bool isMerge) {
if (pCtx->hasPrimaryKey) {
if (!isMerge) {
pRateInfo->pkType = pCtx->input.pPrimaryKey->info.type;
pRateInfo->pkBytes = pCtx->input.pPrimaryKey->info.bytes;
pRateInfo->firstPk = pRateInfo->pkData;
pRateInfo->lastPk = pRateInfo->pkData + pRateInfo->pkBytes;
} else {
pRateInfo->firstPk = pRateInfo->pkData;
pRateInfo->lastPk = pRateInfo->pkData + pRateInfo->pkBytes;
}
} else {
pRateInfo->firstPk = NULL;
pRateInfo->lastPk = NULL;
}
}
int32_t irateFunction(SqlFunctionCtx* pCtx) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SRateInfo* pRateInfo = GET_ROWCELL_INTERBUF(pResInfo);
@ -6148,6 +6256,8 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) {
funcInputUpdate(pCtx);
initializeRateInfo(pCtx, pRateInfo, false);
int32_t numOfElems = 0;
int32_t type = pInputCol->info.type;
SFuncInputRow row = {0};
@ -6161,21 +6271,16 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) {
GET_TYPED_DATA(v, double, type, data);
if (INT64_MIN == pRateInfo->lastKey) {
pRateInfo->lastValue = v;
pRateInfo->lastKey = row.ts;
doSaveRateInfo(pRateInfo, false, row.ts, row.pPk, v);
pRateInfo->hasResult = 1;
continue;
}
if (row.ts > pRateInfo->lastKey) {
if ((INT64_MIN == pRateInfo->firstKey) || pRateInfo->lastKey > pRateInfo->firstKey) {
pRateInfo->firstValue = pRateInfo->lastValue;
pRateInfo->firstKey = pRateInfo->lastKey;
doSaveRateInfo(pRateInfo, true, pRateInfo->lastKey, pRateInfo->lastPk, pRateInfo->lastValue);
}
pRateInfo->lastValue = v;
pRateInfo->lastKey = row.ts;
doSaveRateInfo(pRateInfo, false, row.ts, row.pPk, v);
continue;
} else if (row.ts == pRateInfo->lastKey) {
return TSDB_CODE_FUNC_DUP_TIMESTAMP;
@ -6183,8 +6288,7 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) {
if ((INT64_MIN == pRateInfo->firstKey) || row.ts > pRateInfo->firstKey) {
pRateInfo->firstValue = v;
pRateInfo->firstKey = row.ts;
doSaveRateInfo(pRateInfo, true, row.ts, row.pPk, v);
} else if (row.ts == pRateInfo->firstKey) {
return TSDB_CODE_FUNC_DUP_TIMESTAMP;
}
@ -6220,25 +6324,26 @@ static double doCalcRate(const SRateInfo* pRateInfo, double tickPerSec) {
static void irateTransferInfoImpl(TSKEY inputKey, SRateInfo* pInput, SRateInfo* pOutput, bool isFirstKey) {
if (inputKey > pOutput->lastKey) {
pOutput->firstKey = pOutput->lastKey;
pOutput->firstValue = pOutput->lastValue;
pOutput->lastKey = isFirstKey ? pInput->firstKey : pInput->lastKey;
pOutput->lastValue = isFirstKey ? pInput->firstValue : pInput->lastValue;
doSaveRateInfo(pOutput, true, pOutput->lastKey, pOutput->lastPk, pOutput->lastValue);
if (isFirstKey) {
doSaveRateInfo(pOutput, false, pInput->firstKey, pInput->firstPk, pInput->firstValue);
} else {
doSaveRateInfo(pOutput, false, pInput->lastKey, pInput->lastPk, pInput->lastValue);
}
} else if ((inputKey < pOutput->lastKey) && (inputKey > pOutput->firstKey)) {
pOutput->firstKey = isFirstKey ? pInput->firstKey : pInput->lastKey;
pOutput->firstValue = isFirstKey ? pInput->firstValue : pInput->lastValue;
if (isFirstKey) {
doSaveRateInfo(pOutput, true, pInput->firstKey, pInput->firstPk, pInput->firstValue);
} else {
doSaveRateInfo(pOutput, true, pInput->lastKey, pInput->lastPk, pInput->lastValue);
}
} else {
// inputKey < pOutput->firstKey
}
}
static void irateCopyInfo(SRateInfo* pInput, SRateInfo* pOutput) {
pOutput->firstKey = pInput->firstKey;
pOutput->lastKey = pInput->lastKey;
pOutput->firstValue = pInput->firstValue;
pOutput->lastValue = pInput->lastValue;
doSaveRateInfo(pOutput, true, pInput->firstKey, pInput->firstPk, pInput->firstValue);
doSaveRateInfo(pOutput, false, pInput->lastKey, pInput->lastPk, pInput->lastValue);
}
static int32_t irateTransferInfo(SRateInfo* pInput, SRateInfo* pOutput) {
@ -6273,11 +6378,13 @@ int32_t irateFunctionMerge(SqlFunctionCtx* pCtx) {
}
SRateInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
initializeRateInfo(pCtx, pInfo, true);
int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
char* data = colDataGetData(pCol, i);
SRateInfo* pInputInfo = (SRateInfo*)varDataVal(data);
initializeRateInfo(pCtx, pInfo, true);
if (pInputInfo->hasResult) {
int32_t code = irateTransferInfo(pInputInfo, pInfo);
if (code != TSDB_CODE_SUCCESS) {
@ -6296,7 +6403,7 @@ int32_t irateFunctionMerge(SqlFunctionCtx* pCtx) {
int32_t iratePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SRateInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = getIrateInfoSize();
int32_t resultBytes = getIrateInfoSize(pInfo->pkBytes);
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
memcpy(varDataVal(res), pInfo, resultBytes);

12
source/libs/function/src/functionMgt.c
View File

@ -115,7 +115,7 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin
return funcMgtBuiltins[pFunc->funcId].dataRequiredFunc(pFunc, pTimeWindow);
}
EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, STimeWindow* pTimeWindow) {
EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo) {
if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return TSDB_CODE_FAILED;
}
@ -128,7 +128,7 @@ EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, STimeWindow*
if (funcMgtBuiltins[funcId].dynDataRequiredFunc == NULL) {
return FUNC_DATA_REQUIRED_DATA_LOAD;
} else {
return funcMgtBuiltins[funcId].dynDataRequiredFunc(pRes, pTimeWindow);
return funcMgtBuiltins[funcId].dynDataRequiredFunc(pRes, pBlockInfo);
}
}
@ -415,6 +415,8 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod
int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", (*pPartialFunc)->functionName, pSrcFunc);
taosCreateMD5Hash(name, len);
strncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN - 1);
(*pPartialFunc)->hasPk = pSrcFunc->hasPk;
(*pPartialFunc)->pkBytes = pSrcFunc->pkBytes;
return TSDB_CODE_SUCCESS;
}
@ -453,7 +455,8 @@ static int32_t createMidFunction(const SFunctionNode* pSrcFunc, const SFunctionN
} else {
nodesDestroyList(pParameterList);
}
(*pMidFunc)->hasPk = pPartialFunc->hasPk;
(*pMidFunc)->pkBytes = pPartialFunc->pkBytes;
return code;
}
@ -482,7 +485,8 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio
} else {
nodesDestroyList(pParameterList);
}
(*pMergeFunc)->hasPk = pPartialFunc->hasPk;
(*pMergeFunc)->pkBytes = pPartialFunc->pkBytes;
return code;
}

2
source/libs/parser/inc/parInsertUtil.h
View File

@ -46,7 +46,7 @@ void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag
SArray *tagName, uint8_t tagNum, int32_t ttl);
int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo);
void insInitColValues(STableMeta *pTableMeta, SArray *aColValues);
void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey);
void insCheckTableDataOrder(STableDataCxt *pTableCxt, SRowKey *rowKey);
int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta,
SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals);
int32_t initTableColSubmitData(STableDataCxt *pTableCxt);

21
source/libs/parser/src/parInsertSml.c
View File

@ -113,7 +113,8 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
SSchema* pTagSchema = &pSchema[tags->pColIndex[i]];
SSmlKv* kv = taosArrayGet(cols, i);
if(kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type){
if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 ||
kv->type != pTagSchema->type) {
code = TSDB_CODE_SML_INVALID_DATA;
uError("SML smlBuildTagRow error col not same %s", pTagSchema->name);
goto end;
@ -200,7 +201,9 @@ int32_t smlBuildRow(STableDataCxt* pTableCxt) {
if (TSDB_CODE_SUCCESS != ret) {
return ret;
}
insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow));
SRowKey key;
tRowGetKey(*pRow, &key);
insCheckTableDataOrder(pTableCxt, &key);
return TSDB_CODE_SUCCESS;
}
@ -209,13 +212,14 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
SSchema* pColSchema = schema + index;
SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);
SSmlKv* kv = (SSmlKv*)data;
if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){
if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 ||
kv->type != pColSchema->type) {
ret = TSDB_CODE_SML_INVALID_DATA;
char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1);
if (tmp) {
memcpy(tmp, kv->key, kv->keyLen);
uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)",
tmp, tDataTypes[kv->type].name, pColSchema->name, tDataTypes[pColSchema->type].name);
uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name,
pColSchema->name, tDataTypes[pColSchema->type].name);
taosMemoryFree(tmp);
} else {
uError("SML smlBuildCol out of memory");
@ -367,7 +371,8 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
}
if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) {
if (errno == E2BIG) {
uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length, pColSchema->bytes, kv->value);
uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length,
pColSchema->bytes, kv->value);
buildInvalidOperationMsg(&pBuf, "value too long");
ret = TSDB_CODE_PAR_VALUE_TOO_LONG;
goto end;
@ -396,7 +401,9 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
buildInvalidOperationMsg(&pBuf, "tRowBuild error");
goto end;
}
insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow));
SRowKey key;
tRowGetKey(*pRow, &key);
insCheckTableDataOrder(pTableCxt, &key);
clearColValArraySml(pTableCxt->pValues);
}

102
source/libs/parser/src/parInsertSql.c
View File

@ -13,12 +13,12 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "geosWrapper.h"
#include "parInsertUtil.h"
#include "parToken.h"
#include "scalar.h"
#include "tglobal.h"
#include "ttime.h"
#include "geosWrapper.h"
typedef struct SInsertParseContext {
SParseContext* pComCxt;
@ -154,19 +154,15 @@ static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModify
return code;
}
typedef enum {
BOUND_TAGS,
BOUND_COLUMNS,
BOUND_ALL_AND_TBNAME
} EBoundColumnsType;
typedef enum { BOUND_TAGS, BOUND_COLUMNS, BOUND_ALL_AND_TBNAME } EBoundColumnsType;
static int32_t getTbnameSchemaIndex(STableMeta* pTableMeta) {
return pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns;
}
// pStmt->pSql -> field1_name, ...)
static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, EBoundColumnsType boundColsType, STableMeta* pTableMeta,
SBoundColInfo* pBoundInfo) {
static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, EBoundColumnsType boundColsType,
STableMeta* pTableMeta, SBoundColInfo* pBoundInfo) {
SSchema* pSchema = NULL;
if (boundColsType == BOUND_TAGS) {
pSchema = getTableTagSchema(pTableMeta);
@ -202,8 +198,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E
token.z = tmpTokenBuf;
token.n = strdequote(token.z);
if (boundColsType == BOUND_ALL_AND_TBNAME &&
token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) {
if (boundColsType == BOUND_ALL_AND_TBNAME && token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) {
pBoundInfo->pColIndex[pBoundInfo->numOfBound] = tbnameSchemaIndex;
pUseCols[tbnameSchemaIndex] = true;
++pBoundInfo->numOfBound;
@ -288,7 +283,8 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t
bool firstIsTS = false, secondIsTs = false;
const char* pTokenEnd = *end;
if (TSDB_CODE_SUCCESS != parseTimestampOrInterval(&pTokenEnd, pToken, timePrec, &ts, &interval, pMsgBuf, &firstIsTS)) {
if (TSDB_CODE_SUCCESS !=
parseTimestampOrInterval(&pTokenEnd, pToken, timePrec, &ts, &interval, pMsgBuf, &firstIsTS)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
}
@ -330,8 +326,7 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t
if (pTokenEnd[i] == ' ' || pTokenEnd[i] == '\t') {
i++;
continue;
}
else if (pTokenEnd[i] == ',' || pTokenEnd[i] == ')') {
} else if (pTokenEnd[i] == ',' || pTokenEnd[i] == ')') {
*end = pTokenEnd + i;
if (!firstIsTS) {
return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
@ -362,7 +357,8 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t
valueToken.n = len;
}
if (TSDB_CODE_SUCCESS != parseTimestampOrInterval(&pTokenEnd, &valueToken, timePrec, &tempTs, &tempInterval, pMsgBuf, &secondIsTs)) {
if (TSDB_CODE_SUCCESS !=
parseTimestampOrInterval(&pTokenEnd, &valueToken, timePrec, &tempTs, &tempInterval, pMsgBuf, &secondIsTs)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
}
@ -759,8 +755,8 @@ static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* p
int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type) {
if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER &&
pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL &&
pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT &&
pToken->type != TK_NK_BIN && pToken->type != TK_NK_VARIABLE) ||
pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && pToken->type != TK_NK_BIN &&
pToken->type != TK_NK_VARIABLE) ||
(pToken->n == 0) || (pToken->type == TK_NK_RP)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z);
}
@ -1114,8 +1110,8 @@ static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache,
return code;
}
static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, STableMeta** pTableMeta,
bool* pMissCache, bool bUsingTable) {
static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, STableMeta** pTableMeta, bool* pMissCache,
bool bUsingTable) {
SParseContext* pComCxt = pCxt->pComCxt;
int32_t code = TSDB_CODE_SUCCESS;
if (pComCxt->async) {
@ -1365,13 +1361,11 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOp
return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z);
}
// pStmt->pSql -> field1_name, ...)
return parseBoundColumns(pCxt, &pStmt->pSql, BOUND_COLUMNS, pStmt->pTableMeta,
&pTableCxt->boundColsInfo);
return parseBoundColumns(pCxt, &pStmt->pSql, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo);
}
if (NULL != pStmt->pBoundCols) {
return parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_COLUMNS, pStmt->pTableMeta,
&pTableCxt->boundColsInfo);
return parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo);
}
return TSDB_CODE_SUCCESS;
@ -1700,7 +1694,8 @@ typedef union SRowsDataContext{
SStbRowsDataContext* pStbRowsCxt;
} SRowsDataContext;
static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, bool* pFoundCtbName) {
static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken,
bool* pFoundCtbName) {
*pFoundCtbName = false;
int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg, TSDB_DATA_TYPE_BINARY);
if (TK_NK_VARIABLE == pToken->type) {
@ -1733,9 +1728,8 @@ static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext*
}
static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt,
SStbRowsDataContext* pStbRowsCxt, bool ctbFirst,
const SToken* tagTokens, SSchema* const* tagSchemas,
int numOfTagTokens) {
SStbRowsDataContext* pStbRowsCxt, bool ctbFirst, const SToken* tagTokens,
SSchema* const* tagSchemas, int numOfTagTokens) {
int32_t code = TSDB_CODE_SUCCESS;
uint8_t precision = pStmt->pTableMeta->tableInfo.precision;
@ -1749,8 +1743,8 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif
}
if (code == TSDB_CODE_SUCCESS) {
code = parseTagValue(&pCxt->msg, NULL, precision, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, pStbRowsCxt->aTagVals,
&pStbRowsCxt->pTag);
code = parseTagValue(&pCxt->msg, NULL, precision, pTagSchema, pTagToken, pStbRowsCxt->aTagNames,
pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag);
}
}
if (code == TSDB_CODE_SUCCESS && !pStbRowsCxt->isJsonTag) {
@ -1765,9 +1759,9 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif
}
static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql,
SStbRowsDataContext* pStbRowsCxt, SToken* pToken,
const SBoundColInfo* pCols, const SSchema* pSchemas,
SToken* tagTokens, SSchema** tagSchemas, int* pNumOfTagTokens, bool* bFoundTbName) {
SStbRowsDataContext* pStbRowsCxt, SToken* pToken, const SBoundColInfo* pCols,
const SSchema* pSchemas, SToken* tagTokens, SSchema** tagSchemas, int* pNumOfTagTokens,
bool* bFoundTbName) {
int32_t code = TSDB_CODE_SUCCESS;
SArray* pTagNames = pStbRowsCxt->aTagNames;
SArray* pTagVals = pStbRowsCxt->aTagVals;
@ -1808,11 +1802,11 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value");
}
if (code == TSDB_CODE_SUCCESS) {
code = parseTagValue(&pCxt->msg, ppSql, precision, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals, &pStbRowsCxt->pTag);
code = parseTagValue(&pCxt->msg, ppSql, precision, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals,
&pStbRowsCxt->pTag);
}
}
}
else if (pCols->pColIndex[i] == tbnameIdx) {
} else if (pCols->pColIndex[i] == tbnameIdx) {
code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, bFoundTbName);
}
@ -1827,8 +1821,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
}
static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql,
SStbRowsDataContext* pStbRowsCxt, bool* pGotRow,
SToken* pToken, bool *pCtbFirst) {
SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken, bool* pCtbFirst) {
SBoundColInfo* pCols = &pStbRowsCxt->boundColsInfo;
SSchema* pSchemas = getTableColumnSchema(pStbRowsCxt->pStbMeta);
@ -1840,8 +1833,8 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS
SSchema* tagSchemas[TSDB_MAX_TAGS] = {0};
int numOfTagTokens = 0;
code = doGetStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pToken, pCols, pSchemas, tagTokens,
tagSchemas, &numOfTagTokens, &bFoundTbName);
code = doGetStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pToken, pCols, pSchemas, tagTokens, tagSchemas,
&numOfTagTokens, &bFoundTbName);
if (code == TSDB_CODE_SUCCESS && !bFoundTbName) {
code = buildSyntaxErrMsg(&pCxt->msg, "tbname value expected", pOrigSql);
@ -1870,7 +1863,8 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS
return code;
}
static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SStbRowsDataContext* pStbRowsCxt) {
static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt,
SStbRowsDataContext* pStbRowsCxt) {
int32_t code = TSDB_CODE_SUCCESS;
pStbRowsCxt->pCreateCtbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
@ -1878,9 +1872,9 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod
code = TSDB_CODE_OUT_OF_MEMORY;
}
if (code == TSDB_CODE_SUCCESS) {
insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, pStbRowsCxt->pStbMeta->uid,
pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, getNumOfTags(pStbRowsCxt->pStbMeta),
TSDB_DEFAULT_TABLE_TTL);
insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag,
pStbRowsCxt->pStbMeta->uid, pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames,
getNumOfTags(pStbRowsCxt->pStbMeta), TSDB_DEFAULT_TABLE_TTL);
pStbRowsCxt->pTag = NULL;
}
@ -1907,7 +1901,6 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod
return code;
}
static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) {
if (pStbRowsCxt == NULL) return;
@ -1948,7 +1941,9 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt
SRow** pRow = taosArrayReserve((*ppTableDataCxt)->pData->aRowP, 1);
code = tRowBuild(pStbRowsCxt->aColVals, (*ppTableDataCxt)->pSchema, pRow);
if (TSDB_CODE_SUCCESS == code) {
insCheckTableDataOrder(*ppTableDataCxt, TD_ROW_KEY(*pRow));
SRowKey key;
tRowGetKey(*pRow, &key);
insCheckTableDataOrder(*ppTableDataCxt, &key);
}
}
@ -1961,7 +1956,8 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt
return TSDB_CODE_SUCCESS;
}
static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, SToken* pToken) {
static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow,
SToken* pToken) {
SBoundColInfo* pCols = &pTableCxt->boundColsInfo;
bool isParseBindParam = false;
SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta);
@ -2014,7 +2010,9 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
code = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
if (TSDB_CODE_SUCCESS == code) {
insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow));
SRowKey key;
tRowGetKey(*pRow, &key);
insCheckTableDataOrder(pTableCxt, &key);
}
}
@ -2112,7 +2110,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt
code = parseOneRow(pCxt, (const char**)&pRow, rowsDataCxt.pTableDataCxt, &gotRow, &token);
} else {
STableDataCxt* pTableDataCxt = NULL;
code = parseOneStbRow(pCxt, pStmt, (const char**)&pRow, rowsDataCxt.pStbRowsCxt, &gotRow, &token, &pTableDataCxt);
code =
parseOneStbRow(pCxt, pStmt, (const char**)&pRow, rowsDataCxt.pStbRowsCxt, &gotRow, &token, &pTableDataCxt);
if (code == TSDB_CODE_SUCCESS) {
SStbRowsDataContext* pStbRowsCxt = rowsDataCxt.pStbRowsCxt;
void* pData = pTableDataCxt;
@ -2149,11 +2148,11 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt
return code;
}
static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) {
static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt,
SRowsDataContext rowsDataCxt) {
// init only for file
if (NULL == pStmt->pTableCxtHashObj) {
pStmt->pTableCxtHashObj =
taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
pStmt->pTableCxtHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
}
int32_t numOfRows = 0;
int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows);
@ -2285,7 +2284,8 @@ static int32_t constructStbRowsDataContext(SVnodeModifyOpStmt* pStmt, SStbRowsDa
static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
int32_t code = TSDB_CODE_SUCCESS;
if (!pStmt->pBoundCols) {
return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion", pStmt->pSql);
return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion",
pStmt->pSql);
}
SStbRowsDataContext* pStbRowsCxt = NULL;

24
source/libs/parser/src/parInsertUtil.c
View File

@ -170,9 +170,7 @@ static void initColValues(STableMeta* pTableMeta, SArray* pValues) {
}
}
void insInitColValues(STableMeta* pTableMeta, SArray* aColValues) {
initColValues(pTableMeta, aColValues);
}
void insInitColValues(STableMeta* pTableMeta, SArray* aColValues) { initColValues(pTableMeta, aColValues); }
int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) {
pInfo->numOfCols = numOfBound;
@ -187,21 +185,22 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) {
return TSDB_CODE_SUCCESS;
}
void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) {
void insCheckTableDataOrder(STableDataCxt* pTableCxt, SRowKey* rowKey) {
// once the data block is disordered, we do NOT keep last timestamp any more
if (!pTableCxt->ordered) {
return;
}
if (tsKey < pTableCxt->lastTs) {
if (tRowKeyCompare(rowKey, &pTableCxt->lastKey) < 0) {
pTableCxt->ordered = false;
}
if (tsKey == pTableCxt->lastTs) {
if (tRowKeyCompare(rowKey, &pTableCxt->lastKey) == 0) {
pTableCxt->duplicateTs = true;
}
pTableCxt->lastTs = tsKey;
// TODO: for variable length data type, we need to copy it out
pTableCxt->lastKey = *rowKey;
return;
}
@ -217,7 +216,7 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat
int32_t code = TSDB_CODE_SUCCESS;
pTableCxt->lastTs = 0;
pTableCxt->lastKey = (SRowKey){0};
pTableCxt->ordered = true;
pTableCxt->duplicateTs = false;
@ -317,7 +316,6 @@ static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) {
return code;
}
static void resetColValues(SArray* pValues) {
int32_t num = taosArrayGetSize(pValues);
for (int32_t i = 0; i < num; ++i) {
@ -644,7 +642,8 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) {
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* tFields,
int numFields, bool needChangeLength) {
void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid));
void* tmp =
taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid));
STableDataCxt* pTableCxt = NULL;
int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false);
@ -663,8 +662,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
}
char* p = (char*)data;
// | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each column
// length |
// | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each
// column length |
int32_t version = *(int32_t*)data;
p += sizeof(int32_t);
p += sizeof(int32_t);
@ -761,7 +760,6 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
break;
}
}
}
for (int c = 0; c < boundInfo->numOfBound; ++c) {

18
source/libs/parser/src/parTranslater.c
View File

@ -927,7 +927,7 @@ static bool isPrimaryKey(STempTableNode* pTable, SNode* pExpr) {
return isPrimaryKeyImpl(pExpr);
}
static bool hasPkInTable(STableMeta* pTableMeta) {
static bool hasPkInTable(const STableMeta* pTableMeta) {
return pTableMeta->tableInfo.numOfColumns>=2 && pTableMeta->schema[1].flags & COL_IS_KEY;
}
@ -6752,7 +6752,13 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable
: TSDB_CODE_PAR_INVALID_TAG_NAME,
pStmt->colName);
}
if (hasPkInTable(pTableMeta) && (pSchema->flags & COL_IS_KEY)) {
if (TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType ||
TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType ||
TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME == pStmt->alterType) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PK_OP, pStmt->colName);
}
}
if ((TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType ||
TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) &&
(!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type ||
@ -10826,6 +10832,14 @@ static int32_t rewriteAlterTableImpl(STranslateContext* pCxt, SAlterTableStmt* p
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
}
const SSchema* pSchema = getNormalColSchema(pTableMeta, pStmt->colName);
if (hasPkInTable(pTableMeta) && pSchema && (pSchema->flags & COL_IS_KEY) &&
(TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType ||
TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType ||
TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME == pStmt->alterType)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PK_OP);
}
SVAlterTbReq req = {0};
int32_t code = buildAlterTbReq(pCxt, pStmt, pTableMeta, &req);

2
source/libs/parser/src/parUtil.c
View File

@ -200,6 +200,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "primary key column must be second column";
case TSDB_CODE_PAR_COL_PK_TYPE:
return "primary key column must be of type int, uint, bigint, ubigint, and varchar";
case TSDB_CODE_PAR_INVALID_PK_OP:
return "primary key column can not be added, modified, and dropped";
default:
return "Unknown error";
}

12
source/libs/planner/src/planLogicCreater.c
View File

@ -341,10 +341,11 @@ static int32_t addDefaultScanCol(const STableMeta* pMeta, SNodeList** pCols) {
if (TSDB_SYSTEM_TABLE == pMeta->tableType) {
return addSystableFirstCol(pMeta->uid, pMeta->schema, pCols, pMeta);
}
if (hasPkInTable(pMeta)) {
addPkCol(pMeta->uid, pMeta->schema + 1, pCols, pMeta);
int32_t code = addPrimaryKeyCol(pMeta->uid, pMeta->schema, pCols, pMeta);
if (code == TSDB_CODE_SUCCESS && hasPkInTable(pMeta)) {
code = addPkCol(pMeta->uid, pMeta->schema + 1, pCols, pMeta);
}
return addPrimaryKeyCol(pMeta->uid, pMeta->schema, pCols, pMeta);
return code;
}
static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealTable, bool hasRepeatScanFuncs,
@ -1695,6 +1696,11 @@ static int32_t createDeleteScanLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* p
}
}
STableMeta* pMeta = ((SRealTableNode*)pDelete->pFromTable)->pMeta;
if (TSDB_CODE_SUCCESS == code && hasPkInTable(pMeta)) {
code = addPkCol(pMeta->uid, pMeta->schema + 1, &pScan->pScanCols, pMeta);
}
if (TSDB_CODE_SUCCESS == code && NULL != pDelete->pTagCond) {
pScan->pTagCond = nodesCloneNode(pDelete->pTagCond);
if (NULL == pScan->pTagCond) {

59
source/libs/planner/src/planSpliter.c
View File

@ -690,19 +690,23 @@ static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent
return code;
}
static int32_t stbSplCreateMergeKeysByExpr(SNode* pExpr, EOrder order, SNodeList** pMergeKeys) {
SOrderByExprNode* pOrderByExpr = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
if (NULL == pOrderByExpr) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pOrderByExpr->pExpr = nodesCloneNode(pExpr);
if (NULL == pOrderByExpr->pExpr) {
nodesDestroyNode((SNode*)pOrderByExpr);
return TSDB_CODE_OUT_OF_MEMORY;
}
pOrderByExpr->order = order;
pOrderByExpr->nullOrder = (order == ORDER_ASC) ? NULL_ORDER_FIRST : NULL_ORDER_LAST;
return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pOrderByExpr);
}
static int32_t stbSplCreateMergeKeysByPrimaryKey(SNode* pPrimaryKey, EOrder order, SNodeList** pMergeKeys) {
SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
if (NULL == pMergeKey) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pMergeKey->pExpr = nodesCloneNode(pPrimaryKey);
if (NULL == pMergeKey->pExpr) {
nodesDestroyNode((SNode*)pMergeKey);
return TSDB_CODE_OUT_OF_MEMORY;
}
pMergeKey->order = order;
pMergeKey->nullOrder = NULL_ORDER_FIRST;
return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey);
return stbSplCreateMergeKeysByExpr(pPrimaryKey, order, pMergeKeys);
}
static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
@ -1357,6 +1361,28 @@ static SNode* stbSplFindPrimaryKeyFromScan(SScanLogicNode* pScan) {
return pCol;
}
static SNode* stbSplFindPkFromScan(SScanLogicNode* pScan) {
bool find = false;
SNode* pCol = NULL;
FOREACH(pCol, pScan->pScanCols) {
if (((SColumnNode*)pCol)->isPk) {
find = true;
break;
}
}
if (!find) {
return NULL;
}
SNode* pTarget = NULL;
FOREACH(pTarget, pScan->node.pTargets) {
if (nodesEqualNode(pTarget, pCol)) {
return pCol;
}
}
nodesListStrictAppend(pScan->node.pTargets, nodesCloneNode(pCol));
return pCol;
}
static int32_t stbSplCreateMergeScanNode(SScanLogicNode* pScan, SLogicNode** pOutputMergeScan,
SNodeList** pOutputMergeKeys) {
SNodeList* pChildren = pScan->node.pChildren;
@ -1374,8 +1400,13 @@ static int32_t stbSplCreateMergeScanNode(SScanLogicNode* pScan, SLogicNode** pOu
pMergeScan->filesetDelimited = true;
pMergeScan->node.pChildren = pChildren;
splSetParent((SLogicNode*)pMergeScan);
code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pMergeScan),
pMergeScan->scanSeq[0] > 0 ? ORDER_ASC : ORDER_DESC, &pMergeKeys);
SNode* pTs = stbSplFindPrimaryKeyFromScan(pMergeScan);
code = stbSplCreateMergeKeysByPrimaryKey(pTs, pMergeScan->scanSeq[0] > 0 ? ORDER_ASC : ORDER_DESC, &pMergeKeys);
SNode* pPk = stbSplFindPkFromScan(pMergeScan);
if (TSDB_CODE_SUCCESS == code && NULL != pPk) {
code = stbSplCreateMergeKeysByExpr(pPk, pMergeScan->scanSeq[0] > 0 ? ORDER_ASC : ORDER_DESC, &pMergeKeys);
}
}
if (TSDB_CODE_SUCCESS == code) {

1
source/util/src/terror.c
View File

@ -627,6 +627,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, "Operator not suppor
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TAG_IS_PRIMARY_KEY, "tag can not be primary key")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SECOND_COL_PK, "primary key must be second column")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COL_PK_TYPE, "primary key column must be of type int, uint, bigint, ubigint, and varchar")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PK_OP, "primary key column can not be added, modified, and dropped")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error")
//planner

5
source/util/test/tbaseCodecTest.cpp
View File

@ -17,11 +17,6 @@ using namespace std;
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) {
int64_t start = taosGetTimestampUs();
char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen);

3
tests/parallel_test/cases.task
View File

@ -45,6 +45,9 @@
#,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4
,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pk_error.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pk_func.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pk_func_group.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_expr.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/project_group.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py

2
tests/script/tsim/query/cache_last.sim
View File

@ -89,7 +89,7 @@ if $data10 != @ -> Merge (columns=3 width=24 input_order=unknown output_order=
return -1
endi
sql explain select count(*), last_row(f1), min(f1) from sta interval(1s);
if $data10 != @ -> Merge (columns=4 width=66 input_order=asc output_order=asc mode=sort)@ then
if $data10 != @ -> Merge (columns=4 width=82 input_order=asc output_order=asc mode=sort)@ then
return -1
endi
sql explain select distinct count(*), last_row(f1), min(f1) from tba1;

267
tests/system-test/2-query/pk_error.py Normal file
View File

@ -0,0 +1,267 @@
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import tdDnodes
from math import inf
class TDTestCase:
def caseDescription(self):
'''
case1<shenglian zhou>: [TD-]
'''
return
def init(self, conn, logSql, replicaVer=1):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), True)
self.conn = conn
def restartTaosd(self, index=1, dbname="db"):
tdDnodes.stop(index)
tdDnodes.startWithoutSleep(index)
tdSql.execute(f"use pk_error")
def run(self):
print("running {}".format(__file__))
tdSql.execute("drop database if exists pk_error")
tdSql.execute("create database if not exists pk_error")
tdSql.execute('use pk_error')
tdSql.execute('drop database IF EXISTS d1;')
tdSql.execute('drop database IF EXISTS d2;')
tdSql.execute('create database d1 vgroups 1')
tdSql.execute('use d1;')
tdSql.execute('create table st(ts timestamp, pk int primary key, f int) tags(t int);')
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1, 1);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 2, 2);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:00', 3, 3);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:00', 4, 4);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1, 1);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 4, 4);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 3, 3);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2, 2);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:02', 6, 6);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:02', 5, 5);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 8, 8);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 7, 7);")
tdSql.query('select first(*) from d1.st partition by t order by t;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 3)
tdSql.checkData(1, 2, 3)
tdSql.query('select last(*) from d1.st partition by t order by t;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 6)
tdSql.checkData(0, 2, 6)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 8)
tdSql.checkData(1, 2, 8)
tdSql.query('select last_row(*) from d1.st partition by t order by t;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 6)
tdSql.checkData(0, 2, 6)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 8)
tdSql.checkData(1, 2, 8)
tdSql.query('select ts,diff(f) from d1.st partition by t order by t;')
tdSql.checkRows(4)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, -1)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 5)
tdSql.query('select irate(f) from d1.st partition by t order by t;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, 4.0)
tdSql.checkData(1, 0, 5.0)
tdSql.query('select ts,derivative(f, 1s, 0) from d1.st partition by t order by t;')
tdSql.checkRows(4)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0.0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4.0)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, -1.0)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 5.0)
tdSql.query('select twa(f) from d1.st partition by t order by t;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, 2.0)
tdSql.checkData(1, 0, 3.5)
tdSql.query('select ts,pk,unique(f) from d1.st partition by t order by t,ts,pk;')
tdSql.checkRows(10)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, 4)
tdSql.checkData(2, 2, 4)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 5)
tdSql.checkData(3, 2, 5)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 6)
tdSql.checkData(4, 2, 6)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(5, 1, 3)
tdSql.checkData(5, 2, 3)
tdSql.checkData(6, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(6, 1, 4)
tdSql.checkData(6, 2, 4)
tdSql.checkData(7, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(7, 1, 2)
tdSql.checkData(7, 2, 2)
tdSql.checkData(8, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(8, 1, 7)
tdSql.checkData(8, 2, 7)
tdSql.checkData(9, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(9, 1, 8)
tdSql.checkData(9, 2, 8)
tdSql.execute('create table nt(ts timestamp, pk int primary key, f int);')
tdSql.execute("insert into nt values('2021-04-19 00:00:00', 1, 1);")
tdSql.execute("insert into nt values('2021-04-19 00:00:00', 2, 2);")
tdSql.execute("insert into nt values('2021-04-19 00:00:00', 3, 3);")
tdSql.execute("insert into nt values('2021-04-19 00:00:00', 4, 4);")
tdSql.execute("insert into nt values('2021-04-19 00:00:01', 1, 1);")
tdSql.execute("insert into nt values('2021-04-19 00:00:01', 4, 4);")
tdSql.execute("insert into nt values('2021-04-19 00:00:01', 3, 3);")
tdSql.execute("insert into nt values('2021-04-19 00:00:01', 2, 2);")
tdSql.execute("insert into nt values('2021-04-19 00:00:02', 6, 6);")
tdSql.execute("insert into nt values('2021-04-19 00:00:02', 5, 5);")
tdSql.execute("insert into nt values('2021-04-19 00:00:02', 8, 8);")
tdSql.execute("insert into nt values('2021-04-19 00:00:02', 7, 7);")
tdSql.query('select first(*) from d1.nt;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.query('select last(*) from d1.nt;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 8)
tdSql.checkData(0, 2, 8)
tdSql.query('select last_row(*) from d1.nt;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 8)
tdSql.checkData(0, 2, 8)
tdSql.query('select ts,diff(f) from d1.nt;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4)
tdSql.query('select irate(f) from d1.nt;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 4.0)
tdSql.query('select ts,derivative(f, 1s, 0) from d1.nt;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0.0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4.0)
tdSql.query('select twa(f) from d1.nt;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 2.0)
tdSql.query('select ts,pk,unique(f) from d1.nt order by ts,pk;')
tdSql.checkRows(8)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(2, 1, 3)
tdSql.checkData(2, 2, 3)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(3, 1, 4)
tdSql.checkData(3, 2, 4)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 5)
tdSql.checkData(4, 2, 5)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(5, 1, 6)
tdSql.checkData(5, 2, 6)
tdSql.checkData(6, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(6, 1, 7)
tdSql.checkData(6, 2, 7)
tdSql.checkData(7, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(7, 1, 8)
tdSql.checkData(7, 2, 8)
tdSql.execute('create table st2(ts timestamp, vcpk varchar(64) primary key, f int);')
tdSql.error('alter table st drop column pk;')
tdSql.error('alter table nt drop column pk;')
tdSql.error('alter table st modify column pk bigint;')
tdSql.error('alter table nt modify column pk bigint;')
tdSql.error('alter table st2 modify column vcpk varchar(32)')
tdSql.execute('drop database pk_error')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

652
tests/system-test/2-query/pk_func.py Normal file
View File

@ -0,0 +1,652 @@
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import tdDnodes
from math import inf
class TDTestCase:
def caseDescription(self):
'''
case1<shenglian zhou>: [TD-]
'''
return
def init(self, conn, logSql, replicaVer=1):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), True)
self.conn = conn
def restartTaosd(self, index=1, dbname="db"):
tdDnodes.stop(index)
tdDnodes.startWithoutSleep(index)
tdSql.execute(f"use pk_func")
def run(self):
print("running {}".format(__file__))
tdSql.execute("drop database if exists pk_func")
tdSql.execute("create database if not exists pk_func")
tdSql.execute('use pk_func')
tdSql.execute('drop database IF EXISTS d1;')
tdSql.execute('drop database IF EXISTS d2;')
tdSql.execute('create database d1 vgroups 1')
tdSql.execute('use d1;')
tdSql.execute('create table st(ts timestamp, pk int primary key, f int) tags(t int);')
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1, 1);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 2, 2);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:00', 3, 3);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:00', 4, 4);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1, 1);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 4, 4);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 3, 3);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2, 2);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:02', 6, 6);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:02', 5, 5);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 8, 8);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 7, 7);")
tdSql.query('select * from d1.ct1 order by ts,pk;')
tdSql.checkRows(6)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, 1)
tdSql.checkData(2, 2, 1)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(3, 1, 4)
tdSql.checkData(3, 2, 4)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 5)
tdSql.checkData(4, 2, 5)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(5, 1, 6)
tdSql.checkData(5, 2, 6)
tdSql.query('select * from d1.ct2 order by ts,pk;')
tdSql.checkRows(6)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 3)
tdSql.checkData(0, 2, 3)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 4)
tdSql.checkData(1, 2, 4)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, 2)
tdSql.checkData(2, 2, 2)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(3, 1, 3)
tdSql.checkData(3, 2, 3)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 7)
tdSql.checkData(4, 2, 7)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(5, 1, 8)
tdSql.checkData(5, 2, 8)
tdSql.query('select * from d1.st order by ts,pk;')
tdSql.checkRows(12)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(2, 1, 3)
tdSql.checkData(2, 2, 3)
tdSql.checkData(2, 3, 2)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(3, 1, 4)
tdSql.checkData(3, 2, 4)
tdSql.checkData(3, 3, 2)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(4, 1, 1)
tdSql.checkData(4, 2, 1)
tdSql.checkData(4, 3, 1)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(5, 1, 2)
tdSql.checkData(5, 2, 2)
tdSql.checkData(5, 3, 2)
tdSql.checkData(6, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(6, 1, 3)
tdSql.checkData(6, 2, 3)
tdSql.checkData(6, 3, 2)
tdSql.checkData(7, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(7, 1, 4)
tdSql.checkData(7, 2, 4)
tdSql.checkData(7, 3, 1)
tdSql.checkData(8, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(8, 1, 5)
tdSql.checkData(8, 2, 5)
tdSql.checkData(8, 3, 1)
tdSql.checkData(9, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(9, 1, 6)
tdSql.checkData(9, 2, 6)
tdSql.checkData(9, 3, 1)
tdSql.checkData(10, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(10, 1, 7)
tdSql.checkData(10, 2, 7)
tdSql.checkData(10, 3, 2)
tdSql.checkData(11, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(11, 1, 8)
tdSql.checkData(11, 2, 8)
tdSql.checkData(11, 3, 2)
tdSql.query('select first(*) from d1.ct1;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.query('select first(*) from d1.ct2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 3)
tdSql.checkData(0, 2, 3)
tdSql.query('select first(*) from d1.st;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.query('select last(*) from d1.ct1;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 6)
tdSql.checkData(0, 2, 6)
tdSql.query('select last(*) from d1.ct2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 8)
tdSql.checkData(0, 2, 8)
tdSql.query('select last(*) from d1.st;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 8)
tdSql.checkData(0, 2, 8)
tdSql.query('select last_row(*) from d1.ct1;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 6)
tdSql.checkData(0, 2, 6)
tdSql.query('select last_row(*) from d1.ct2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 8)
tdSql.checkData(0, 2, 8)
tdSql.query('select last_row(*) from d1.st;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 8)
tdSql.checkData(0, 2, 8)
tdSql.query('select ts,diff(f) from d1.ct1;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4)
tdSql.query('select ts,diff(f) from d1.ct2;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, -1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 5)
tdSql.query('select ts,diff(f) from d1.st;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4)
tdSql.query('select irate(f) from d1.ct1;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 4.0)
tdSql.query('select irate(f) from d1.ct2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 5.0)
tdSql.query('select irate(f) from d1.st;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 4.0)
tdSql.query('select ts,derivative(f, 1s, 0) from d1.ct1;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0.0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4.0)
tdSql.query('select ts,derivative(f, 1s, 0) from d1.ct2;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, -1.0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 5.0)
tdSql.query('select ts,derivative(f, 1s, 0) from d1.st;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0.0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4.0)
tdSql.query('select twa(f) from d1.ct1;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 2.0)
tdSql.query('select twa(f) from d1.ct2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 3.5)
tdSql.query('select twa(f) from d1.st;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 2.0)
tdSql.query('select ts,pk,unique(f) from d1.ct1 order by ts,pk;')
tdSql.checkRows(5)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, 4)
tdSql.checkData(2, 2, 4)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 5)
tdSql.checkData(3, 2, 5)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 6)
tdSql.checkData(4, 2, 6)
tdSql.query('select ts,pk,unique(f) from d1.ct2 order by ts,pk;')
tdSql.checkRows(5)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 3)
tdSql.checkData(0, 2, 3)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 4)
tdSql.checkData(1, 2, 4)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, 2)
tdSql.checkData(2, 2, 2)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 7)
tdSql.checkData(3, 2, 7)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 8)
tdSql.checkData(4, 2, 8)
tdSql.query('select ts,pk,unique(f) from d1.st order by ts,pk;')
tdSql.checkRows(8)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(2, 1, 3)
tdSql.checkData(2, 2, 3)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(3, 1, 4)
tdSql.checkData(3, 2, 4)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 5)
tdSql.checkData(4, 2, 5)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(5, 1, 6)
tdSql.checkData(5, 2, 6)
tdSql.checkData(6, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(6, 1, 7)
tdSql.checkData(6, 2, 7)
tdSql.checkData(7, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(7, 1, 8)
tdSql.checkData(7, 2, 8)
tdSql.execute('create database d2 vgroups 2')
tdSql.execute('use d2;')
tdSql.execute('create table st(ts timestamp, pk int primary key, f int) tags(t int);')
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1, 1);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 2, 2);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:00', 3, 3);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:00', 4, 4);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1, 1);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 4, 4);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 3, 3);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2, 2);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:02', 6, 6);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:02', 5, 5);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 8, 8);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 7, 7);")
tdSql.query('select * from d2.ct1 order by ts,pk;')
tdSql.checkRows(6)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, 1)
tdSql.checkData(2, 2, 1)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(3, 1, 4)
tdSql.checkData(3, 2, 4)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 5)
tdSql.checkData(4, 2, 5)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(5, 1, 6)
tdSql.checkData(5, 2, 6)
tdSql.query('select * from d2.ct2 order by ts,pk;')
tdSql.checkRows(6)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 3)
tdSql.checkData(0, 2, 3)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 4)
tdSql.checkData(1, 2, 4)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, 2)
tdSql.checkData(2, 2, 2)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(3, 1, 3)
tdSql.checkData(3, 2, 3)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 7)
tdSql.checkData(4, 2, 7)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(5, 1, 8)
tdSql.checkData(5, 2, 8)
tdSql.query('select * from d2.st order by ts,pk;')
tdSql.checkRows(12)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(2, 1, 3)
tdSql.checkData(2, 2, 3)
tdSql.checkData(2, 3, 2)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(3, 1, 4)
tdSql.checkData(3, 2, 4)
tdSql.checkData(3, 3, 2)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(4, 1, 1)
tdSql.checkData(4, 2, 1)
tdSql.checkData(4, 3, 1)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(5, 1, 2)
tdSql.checkData(5, 2, 2)
tdSql.checkData(5, 3, 2)
tdSql.checkData(6, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(6, 1, 3)
tdSql.checkData(6, 2, 3)
tdSql.checkData(6, 3, 2)
tdSql.checkData(7, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(7, 1, 4)
tdSql.checkData(7, 2, 4)
tdSql.checkData(7, 3, 1)
tdSql.checkData(8, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(8, 1, 5)
tdSql.checkData(8, 2, 5)
tdSql.checkData(8, 3, 1)
tdSql.checkData(9, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(9, 1, 6)
tdSql.checkData(9, 2, 6)
tdSql.checkData(9, 3, 1)
tdSql.checkData(10, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(10, 1, 7)
tdSql.checkData(10, 2, 7)
tdSql.checkData(10, 3, 2)
tdSql.checkData(11, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(11, 1, 8)
tdSql.checkData(11, 2, 8)
tdSql.checkData(11, 3, 2)
tdSql.query('select first(*) from d2.ct1;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.query('select first(*) from d2.ct2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 3)
tdSql.checkData(0, 2, 3)
tdSql.query('select first(*) from d2.st;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.query('select last(*) from d2.ct1;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 6)
tdSql.checkData(0, 2, 6)
tdSql.query('select last(*) from d2.ct2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 8)
tdSql.checkData(0, 2, 8)
tdSql.query('select last(*) from d2.st;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 8)
tdSql.checkData(0, 2, 8)
tdSql.query('select last_row(*) from d2.ct1;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 6)
tdSql.checkData(0, 2, 6)
tdSql.query('select last_row(*) from d2.ct2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 8)
tdSql.checkData(0, 2, 8)
tdSql.query('select last_row(*) from d2.st;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 8)
tdSql.checkData(0, 2, 8)
tdSql.query('select ts,diff(f) from d2.ct1;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4)
tdSql.query('select ts,diff(f) from d2.ct2;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, -1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 5)
tdSql.query('select ts,diff(f) from d2.st;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4)
tdSql.query('select irate(f) from d2.ct1;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 4.0)
tdSql.query('select irate(f) from d2.ct2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 5.0)
tdSql.query('select irate(f) from d2.st;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 4.0)
tdSql.query('select ts,derivative(f, 1s, 0) from d2.ct1;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0.0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4.0)
tdSql.query('select ts,derivative(f, 1s, 0) from d2.ct2;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, -1.0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 5.0)
tdSql.query('select ts,derivative(f, 1s, 0) from d2.st;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0.0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4.0)
tdSql.query('select twa(f) from d2.ct1;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 2.0)
tdSql.query('select twa(f) from d2.ct2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 3.5)
tdSql.query('select twa(f) from d2.st;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 2.0)
tdSql.query('select ts,pk,unique(f) from d2.ct1 order by ts,pk;')
tdSql.checkRows(5)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, 4)
tdSql.checkData(2, 2, 4)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 5)
tdSql.checkData(3, 2, 5)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 6)
tdSql.checkData(4, 2, 6)
tdSql.query('select ts,pk,unique(f) from d2.ct2 order by ts,pk;')
tdSql.checkRows(5)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 3)
tdSql.checkData(0, 2, 3)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 4)
tdSql.checkData(1, 2, 4)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, 2)
tdSql.checkData(2, 2, 2)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 7)
tdSql.checkData(3, 2, 7)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 8)
tdSql.checkData(4, 2, 8)
tdSql.query('select ts,pk,unique(f) from d2.st order by ts,pk;')
tdSql.checkRows(8)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(2, 1, 3)
tdSql.checkData(2, 2, 3)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(3, 1, 4)
tdSql.checkData(3, 2, 4)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 5)
tdSql.checkData(4, 2, 5)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(5, 1, 6)
tdSql.checkData(5, 2, 6)
tdSql.checkData(6, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(6, 1, 7)
tdSql.checkData(6, 2, 7)
tdSql.checkData(7, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(7, 1, 8)
tdSql.checkData(7, 2, 8)
tdSql.execute('drop database pk_func')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

284
tests/system-test/2-query/pk_func_group.py Normal file
View File

@ -0,0 +1,284 @@
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import tdDnodes
from math import inf
class TDTestCase:
def caseDescription(self):
'''
case1<shenglian zhou>: [TD-]
'''
return
def init(self, conn, logSql, replicaVer=1):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), True)
self.conn = conn
def restartTaosd(self, index=1, dbname="db"):
tdDnodes.stop(index)
tdDnodes.startWithoutSleep(index)
tdSql.execute(f"use pk_func_group")
def run(self):
print("running {}".format(__file__))
tdSql.execute("drop database if exists pk_func_group")
tdSql.execute("create database if not exists pk_func_group")
tdSql.execute('use pk_func_group')
tdSql.execute('drop database IF EXISTS d1;')
tdSql.execute('drop database IF EXISTS d2;')
tdSql.execute('create database d1 vgroups 1')
tdSql.execute('use d1;')
tdSql.execute('create table st(ts timestamp, pk int primary key, f int) tags(t int);')
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1, 1);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 2, 2);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:00', 3, 3);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:00', 4, 4);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1, 1);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 4, 4);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 3, 3);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2, 2);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:02', 6, 6);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:02', 5, 5);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 8, 8);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 7, 7);")
tdSql.query('select first(*) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 3)
tdSql.checkData(1, 2, 3)
tdSql.query('select last(*) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 6)
tdSql.checkData(0, 2, 6)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 8)
tdSql.checkData(1, 2, 8)
tdSql.query('select last_row(*) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 6)
tdSql.checkData(0, 2, 6)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 8)
tdSql.checkData(1, 2, 8)
tdSql.query('select ts,diff(f) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(4)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, -1)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 5)
tdSql.query('select irate(f) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, 4.0)
tdSql.checkData(1, 0, 5.0)
tdSql.query('select ts,derivative(f, 1s, 0) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(4)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0.0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4.0)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, -1.0)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 5.0)
tdSql.query('select twa(f) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, 2.0)
tdSql.checkData(1, 0, 3.5)
tdSql.query('select ts,pk,unique(f) from d1.st partition by tbname order by tbname,ts,pk;')
tdSql.checkRows(10)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, 4)
tdSql.checkData(2, 2, 4)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 5)
tdSql.checkData(3, 2, 5)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 6)
tdSql.checkData(4, 2, 6)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(5, 1, 3)
tdSql.checkData(5, 2, 3)
tdSql.checkData(6, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(6, 1, 4)
tdSql.checkData(6, 2, 4)
tdSql.checkData(7, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(7, 1, 2)
tdSql.checkData(7, 2, 2)
tdSql.checkData(8, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(8, 1, 7)
tdSql.checkData(8, 2, 7)
tdSql.checkData(9, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(9, 1, 8)
tdSql.checkData(9, 2, 8)
tdSql.execute('create database d2 vgroups 2')
tdSql.execute('use d2;')
tdSql.execute('create table st(ts timestamp, pk int primary key, f int) tags(t int);')
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1, 1);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 2, 2);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:00', 3, 3);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:00', 4, 4);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1, 1);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 4, 4);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 3, 3);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2, 2);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:02', 6, 6);")
tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:02', 5, 5);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 8, 8);")
tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 7, 7);")
tdSql.query('select first(*) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 3)
tdSql.checkData(1, 2, 3)
tdSql.query('select last(*) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 6)
tdSql.checkData(0, 2, 6)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 8)
tdSql.checkData(1, 2, 8)
tdSql.query('select last_row(*) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(0, 1, 6)
tdSql.checkData(0, 2, 6)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 8)
tdSql.checkData(1, 2, 8)
tdSql.query('select ts,diff(f) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(4)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, -1)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 5)
tdSql.query('select irate(f) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, 4.0)
tdSql.checkData(1, 0, 5.0)
tdSql.query('select ts,derivative(f, 1s, 0) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(4)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(0, 1, 0.0)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(1, 1, 4.0)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, -1.0)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 5.0)
tdSql.query('select twa(f) from d1.st partition by tbname order by tbname;')
tdSql.checkRows(2)
tdSql.checkData(0, 0, 2.0)
tdSql.checkData(1, 0, 3.5)
tdSql.query('select ts,pk,unique(f) from d1.st partition by tbname order by tbname,ts,pk;')
tdSql.checkRows(10)
tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1)
tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(1, 1, 2)
tdSql.checkData(1, 2, 2)
tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(2, 1, 4)
tdSql.checkData(2, 2, 4)
tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(3, 1, 5)
tdSql.checkData(3, 2, 5)
tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(4, 1, 6)
tdSql.checkData(4, 2, 6)
tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(5, 1, 3)
tdSql.checkData(5, 2, 3)
tdSql.checkData(6, 0, datetime.datetime(2021, 4, 19, 0, 0))
tdSql.checkData(6, 1, 4)
tdSql.checkData(6, 2, 4)
tdSql.checkData(7, 0, datetime.datetime(2021, 4, 19, 0, 0, 1))
tdSql.checkData(7, 1, 2)
tdSql.checkData(7, 2, 2)
tdSql.checkData(8, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(8, 1, 7)
tdSql.checkData(8, 2, 7)
tdSql.checkData(9, 0, datetime.datetime(2021, 4, 19, 0, 0, 2))
tdSql.checkData(9, 1, 8)
tdSql.checkData(9, 2, 8)
tdSql.execute('drop database pk_func_group')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())