[td-225] fix bugs.

This commit is contained in:
Haojun Liao 2020-05-22 16:44:32 +08:00
parent b1d4179d08
commit afb434f07f
6 changed files with 98 additions and 94 deletions

View File

@ -571,7 +571,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid, dfltKey));
pQueryMsg->numOfTables = htonl(1); // set the number of tables
pMsg += sizeof(STableIdInfo);
} else {
int32_t index = pTableMetaInfo->vgroupIndex;
@ -601,8 +600,8 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
}
}
tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
pTableMeta->uid);
tscTrace("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
pTableMeta->sid, pTableMeta->uid);
return pMsg;
}
@ -1863,6 +1862,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
}
free(pTableMeta);
tscTrace("%p recv table meta: %"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->uid, pTableMeta->sid, pTableMetaInfo->name);
return TSDB_CODE_SUCCESS;
}

View File

@ -6041,6 +6041,8 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
isSTableQuery = TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
STableIdInfo *id = taosArrayGet(pTableIdList, 0);
qTrace("qmsg:%p query table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}

View File

@ -485,6 +485,8 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS
static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock,
SArray* sa);
static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey,
STsdbQueryHandle* pQueryHandle);
static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) {
STsdbRepo *pRepo = pQueryHandle->pTsdb;
@ -562,11 +564,33 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1);
// todo refactor:
if ((ASCENDING_TRAVERSE(pQueryHandle->order) && ((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey && k1 >= binfo.window.skey) ||
(k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey && k2 >= binfo.window.skey))) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) && ((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey && k1 >= binfo.window.skey) ||
(k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey && k2 >= binfo.window.skey)))) {
if ((ASCENDING_TRAVERSE(pQueryHandle->order) &&
((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey) || (k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey))) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) &&
((k1 != TSKEY_INITIAL_VAL && k1 >= binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 >= binfo.window.skey)))) {
if ((ASCENDING_TRAVERSE(pQueryHandle->order) &&
((k1 != TSKEY_INITIAL_VAL && k1 < binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 < binfo.window.skey))) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) &&
(((k1 != TSKEY_INITIAL_VAL && k1 > binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 > binfo.window.skey))))) {
// do not load file block into buffer
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
cur->rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, binfo.window.skey - step,
pQueryHandle->outputCapacity, &cur->win.skey, &cur->win.ekey, pQueryHandle);
pQueryHandle->realNumOfRows = cur->rows;
// update the last key value
pCheckInfo->lastKey = cur->win.ekey + step;
if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
SWAP(cur->win.skey, cur->win.ekey, TSKEY);
}
cur->mixBlock = true;
cur->blockCompleted = false;
return;
}
doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, sa);
} else {
@ -917,6 +941,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
} while (numOfRows < pQueryHandle->outputCapacity);
if (numOfRows < pQueryHandle->outputCapacity) {
/**
* if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT
* copy them all to result buffer, since it may be overlapped with file data block.
*/
if (node == NULL ||
((dataRowKey(SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
((dataRowKey(SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
@ -949,28 +977,6 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
pos += (end - start + 1) * step;
} else {
while(numOfRows < pQueryHandle->outputCapacity && node != NULL &&
(((dataRowKey(SL_GET_NODE_DATA(node)) <= pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
((dataRowKey(SL_GET_NODE_DATA(node)) >= pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)))) {
SDataRow row = SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row);
copyOneRowFromMem(pQueryHandle, pCheckInfo, pQueryHandle->outputCapacity, numOfRows, row, pSchema);
numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key;
}
cur->win.ekey = key;
cur->lastKey = key + step;
cur->mixBlock = true;
tSkipListIterNext(pCheckInfo->iter);
node = tSkipListIterGet(pCheckInfo->iter);
}
}
}
}
@ -1275,7 +1281,8 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
(cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
// all data blocks in current file has been checked already, try next file if exists
return getDataBlocksInFilesImpl(pQueryHandle);
} else { // next block of the same file
} else {
// next block of the same file
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
cur->slot += step;
@ -1286,9 +1293,8 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
return loadFileDataBlock(pQueryHandle, pNext->pBlock.compBlock, pNext->pTableCheckInfo);
}
} else {
SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlockInfo->pBlock.compBlock, sa);
return true;
handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->pBlock.compBlock, pCheckInfo);
return pQueryHandle->realNumOfRows > 0;
}
}
}
@ -1472,17 +1478,14 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY
SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle;
STable* pTable = NULL;
int32_t rows = 0;
int32_t step = ASCENDING_TRAVERSE(pHandle->order)? 1:-1;
// there are data in file
if (pHandle->cur.fid >= 0) {
STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot];
STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
pTable = pCheckInfo->pTableObj;
STable* pTable = pCheckInfo->pTableObj;
if (pHandle->cur.mixBlock) {
SDataBlockInfo blockInfo = {
@ -1494,19 +1497,17 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
return blockInfo;
} else {
SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfo->pBlock.compBlock);
return binfo;
return getTrueDataBlockInfo(pCheckInfo, pBlockInfo->pBlock.compBlock);
}
} else {
STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
pTable = pCheckInfo->pTableObj;
if (pTable->mem != NULL) {
// create mem table iterator if it is not created yet
STable* pTable = pCheckInfo->pTableObj;
if (pTable->mem != NULL) { // create mem table iterator if it is not created yet
assert(pCheckInfo->iter != NULL);
STimeWindow* win = &pHandle->cur.win;
rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey,
pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey,
pHandle->outputCapacity, &win->skey, &win->ekey, pHandle); // todo refactor API
// update the last key value
@ -1520,7 +1521,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
SDataBlockInfo blockInfo = {
.uid = pTable->tableId.uid,
.tid = pTable->tableId.tid,
.rows = rows,
.rows = pHandle->cur.rows,
.window = pHandle->cur.win,
};

View File

@ -32,10 +32,10 @@ sql connect
sql create database ir1db days 7
sql use ir1db
sql create table tb(ts timestamp, i int)
sql create table tb(ts timestamp, i bigint)
print ================= step1
sql import into tb values(1520000010000, 10000)
sql import into tb values(1520000010000, 1520000010000)
sql select * from tb;
print $rows
if $rows != 1 then
@ -43,7 +43,7 @@ if $rows != 1 then
endi
print ================= step2
sql insert into tb values(1520000008000, 8000)
sql insert into tb values(1520000008000, 1520000008000)
print $rows
sql select * from tb;
if $rows != 2 then
@ -51,7 +51,7 @@ if $rows != 2 then
endi
print ================= step3
sql insert into tb values(1520000020000, 20000)
sql insert into tb values(1520000020000, 1520000020000)
sql select * from tb;
print $rows
if $rows != 3 then
@ -59,9 +59,9 @@ if $rows != 3 then
endi
print ================= step4
sql import into tb values(1520000009000, 9000)
sql import into tb values(1520000015000, 15000)
sql import into tb values(1520000030000, 30000)
sql import into tb values(1520000009000, 1520000009000)
sql import into tb values(1520000015000, 1520000015000)
sql import into tb values(1520000030000, 1520000030000)
sql select * from tb;
print $rows
if $rows != 6 then
@ -69,10 +69,10 @@ if $rows != 6 then
endi
print ================= step5
sql insert into tb values(1520000008000, 8000)
sql insert into tb values(1520000014000, 14000)
sql insert into tb values(1520000025000, 25000)
sql insert into tb values(1520000040000, 40000)
sql insert into tb values(1520000008000, 1520000008000)
sql insert into tb values(1520000014000, 1520000014000)
sql insert into tb values(1520000025000, 1520000025000)
sql insert into tb values(1520000040000, 1520000040000)
sql select * from tb;
print $rows
if $rows != 9 then
@ -80,11 +80,11 @@ if $rows != 9 then
endi
print ================= step6
sql import into tb values(1520000007000, 7000)
sql import into tb values(1520000012000, 12000)
sql import into tb values(1520000023000, 23000)
sql import into tb values(1520000034000, 34000)
sql import into tb values(1520000050000, 50000)
sql import into tb values(1520000007000, 1520000007000)
sql import into tb values(1520000012000, 1520000012000)
sql import into tb values(1520000023000, 1520000023000)
sql import into tb values(1520000034000, 1520000034000)
sql import into tb values(1520000050000, 1520000050000)
sql select * from tb;
print $rows
if $rows != 14 then
@ -104,11 +104,11 @@ if $rows != 14 then
endi
print ================= step7
sql import into tb values(1520000007001, 7001)
sql import into tb values(1520000012001, 12001)
sql import into tb values(1520000023001, 23001)
sql import into tb values(1520000034001, 34001)
sql import into tb values(1520000050001, 50001)
sql import into tb values(1520000007001, 1520000007001)
sql import into tb values(1520000012001, 1520000012001)
sql import into tb values(1520000023001, 1520000023001)
sql import into tb values(1520000034001, 1520000034001)
sql import into tb values(1520000050001, 1520000050001)
sql select * from tb;
print $rows
if $rows != 19 then
@ -117,10 +117,10 @@ if $rows != 19 then
endi
print ================= step8
sql insert into tb values(1520000008002, 8002)
sql insert into tb values(1520000014002, 14002)
sql insert into tb values(1520000025002, 25002)
sql insert into tb values(1520000060000, 60000)
sql insert into tb values(1520000008002, 1520000008002)
sql insert into tb values(1520000014002, 1520000014002)
sql insert into tb values(1520000025002, 1520000025002)
sql insert into tb values(1520000060000, 1520000060000)
sql select * from tb;
print $rows
if $rows != 23 then
@ -142,18 +142,18 @@ print ================= step9
#sql import into tb values(now+14d, 50001)
#sql import into tb values(now+16d, 500051)
sql import into tb values(1517408000000, 7003)
sql import into tb values(1518272000000, 34003)
sql import into tb values(1519136000000, 34003)
sql import into tb values(1519568000000, 34003)
sql import into tb values(1519654400000, 50001)
sql import into tb values(1519827200000, 50001)
sql import into tb values(1520345600000, 50001)
sql import into tb values(1520691200000, 50002)
sql import into tb values(1520864000000, 50003)
sql import into tb values(1521900800000, 50004)
sql import into tb values(1523110400000, 50001)
sql import into tb values(1521382400000, 500051)
sql import into tb values(1517408000000, 1517408000000)
sql import into tb values(1518272000000, 1518272000000)
sql import into tb values(1519136000000, 1519136000000)
sql import into tb values(1519568000000, 1519568000000)
sql import into tb values(1519654400000, 1519654400000)
sql import into tb values(1519827200000, 1519827200000)
sql import into tb values(1520345600000, 1520345600000)
sql import into tb values(1520691200000, 1520691200000)
sql import into tb values(1520864000000, 1520864000000)
sql import into tb values(1521900800000, 1521900800000)
sql import into tb values(1523110400000, 1523110400000)
sql import into tb values(1521382400000, 1521382400000)
sql select * from tb;
print $rows
if $rows != 35 then
@ -176,7 +176,7 @@ endi
print ================= step11
#sql import into tb values(now-50d, 7003) (now-48d, 7003) (now-46d, 7003) (now-44d, 7003) (now-42d, 7003)
sql import into tb values(1515680000000, 7003) (1515852800000, 7003) (1516025600000, 7003) (1516198400000, 7003) (1516371200000, 7003)
sql import into tb values(1515680000000, 1) (1515852800000, 2) (1516025600000, 3) (1516198400000, 4) (1516371200000, 5)
sql select * from tb;
if $rows != 40 then
return -1
@ -184,8 +184,8 @@ endi
print ================= step12
#1520000000000
#sql import into tb values(now-19d, 7003) (now-18d, 7003) (now-17d, 7003) (now-16d, 7003) (now-15d, 7003) (now-14d, 7003) (now-13d, 7003) (now-12d, 7003) (now-11d, 7003)
sql import into tb values(1518358400000, 7003) (1518444800000, 7003) (1518531200000, 7003) (1518617600000, 7003) (1518704000000, 7003) (1518790400000, 7003) (1518876800000, 7003) (1518963200000, 7003) (1519049600000, 7003)
#sql import into tb values(now-19d, -19) (now-18d, -18) (now-17d, -17) (now-16d, -16) (now-15d, -15) (now-14d, -14) (now-13d, -13) (now-12d, -12) (now-11d, -11)
sql import into tb values(1518358400000, 6) (1518444800000, 7) (1518531200000, 8) (1518617600000, 9) (1518704000000, 10) (1518790400000, 11) (1518876800000, 12) (1518963200000, 13) (1519049600000, 14)
sql select * from tb;
print $rows
if $rows != 49 then
@ -195,13 +195,13 @@ endi
print ================= step14
#1520000000000
#sql import into tb values(now-48d, 34003)
#sql import into tb values(now-38d, 50001)
#sql import into tb values(now-28d, 50001)
#sql import into tb values(now-48d, -48)
#sql import into tb values(now-38d, -38)
#sql import into tb values(now-28d, -28)
sql import into tb values(1515852800001, 34003)
sql import into tb values(1516716800000, 50001)
sql import into tb values(1517580800000, 50001)
sql import into tb values(1515852800001, -48)
sql import into tb values(1516716800000, -38)
sql import into tb values(1517580800000, -28)
sql select * from tb;
if $rows != 52 then

View File

@ -65,7 +65,7 @@ sleep 2000
system sh/exec.sh -n dnode1 -s start
print ================== server restart completed
#run general/parser/limit1_tb.sim
run general/parser/limit1_tb.sim
run general/parser/limit1_stb.sim
system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -107,6 +107,7 @@ echo "monitorDebugFlag 131" >> $TAOS_CFG
echo "udebugFlag 131" >> $TAOS_CFG
echo "jnidebugFlag 131" >> $TAOS_CFG
echo "sdebugFlag 135" >> $TAOS_CFG
echo "qdebugFlag 135" >> $TAOS_CFG
echo "monitor 0" >> $TAOS_CFG
echo "monitorInterval 1" >> $TAOS_CFG
echo "http 0" >> $TAOS_CFG