diff --git a/Jenkinsfile2 b/Jenkinsfile2 index cbf663cdcf..7a6f793399 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -117,27 +117,29 @@ def pre_test(){ def pre_test_win(){ bat ''' hostname + ipconfig + set date /t time /t - taskkill /f /t /im python.exe - taskkill /f /t /im bash.exe rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug - exit 0 ''' bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git reset --hard git fetch || git fetch + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git reset --hard git fetch || git fetch - git checkout -f ''' script { if (env.CHANGE_TARGET == 'master') { bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout master + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout master ''' @@ -145,6 +147,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout 2.0 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout 2.0 ''' @@ -152,6 +156,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout 3.0 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout 3.0 ''' @@ -159,6 +165,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout develop + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout develop ''' @@ -169,30 +177,52 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git pull - git log -5 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git pull - git fetch origin +refs/pull/${CHANGE_ID}/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + git fetch origin +refs/pull/%CHANGE_ID%/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout -qf FETCH_HEAD - git log -5 ''' } else if (env.CHANGE_URL =~ /\/TDinternal\//) { bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git pull - git fetch origin +refs/pull/${CHANGE_ID}/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + git fetch origin +refs/pull/%CHANGE_ID%/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout -qf FETCH_HEAD - git log -5 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git pull - git log -5 ''' } else { - sh ''' - echo "unmatched reposiotry ${CHANGE_URL}" + bat ''' + echo "unmatched reposiotry %CHANGE_URL%" ''' } } + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + git branch + git log -5 + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + git branch + git log -5 + ''' bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git submodule update --init --recursive @@ -205,10 +235,15 @@ def pre_test_build_win() { cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal mkdir debug cd debug + time /t call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 set CL=/MP8 - cmake .. -G "NMake Makefiles JOM" - jom -j 4 || exit 8 + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake" + time /t + cmake .. -G "NMake Makefiles JOM" || exit 7 + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" + time /t + jom -j 6 || exit 8 time /t ''' return 1 @@ -226,6 +261,13 @@ pipeline { stages { stage('run test') { parallel { + stage('windows test') { + agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "} + steps { + pre_test_win() + pre_test_build_win() + } + } stage('linux test') { agent{label " slave3_0 || slave15 || slave16 || slave17 "} options { skipDefaultCheckout() } diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index eeae1b47da..10d3dcdbaa 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -56,10 +56,10 @@ typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutS typedef struct SSkipList { uint32_t seed; + uint16_t len; __compar_fn_t comparFn; __sl_key_fn_t keyFn; TdThreadRwlock *lock; - uint16_t len; uint8_t maxLevel; uint8_t flags; uint8_t type; // static info above diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d1516403c1..22bdd960ea 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -1063,7 +1063,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { if (isAllRowsNone(pCol)) { - pVal->valType = TD_VTYPE_NULL; + pVal->valType = TD_VTYPE_NONE; #ifdef TD_SUPPORT_READ2 pVal->val = (void *)getNullValue(pCol->type); #else diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 6a5ad63822..b1760e6dae 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -104,6 +104,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeep int tsdbClose(STsdb** pTsdb); int tsdbBegin(STsdb* pTsdb); int tsdbCommit(STsdb* pTsdb); +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index bebdfb3b63..c1a1e7570e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -330,12 +330,12 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, ASSERT(pReadh->pDCols[0]->bitmapMode != 0); } - if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { for (int i = 0; i < numOfColsIds; ++i) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - if (pDataCol->bitmap) { + if (pDataCol->len > 0 && pDataCol->bitmap) { ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(pDataCol->pBitmap); tdMergeBitmap(pDataCol->pBitmap, pReadh->pDCols[0]->numOfRows, pDataCol->pBitmap); tdDataColsSetBitmapI(pReadh->pDCols[0]); } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 341ab94ca4..a67f413ba7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -15,7 +15,7 @@ #include "tsdb.h" -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); +// static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) { SSubmitMsgIter msgIter = {0}; @@ -54,7 +54,38 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * return 0; } -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { +#if 0 +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, STable *pTable, STSRow *row, TSKEY minKey, TSKEY maxKey, + TSKEY now) { + TSKEY rowKey = TD_ROW_KEY(row); + if (rowKey < minKey || rowKey > maxKey) { + tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 + " maxKey %" PRId64 " row key %" PRId64, + REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, + rowKey); + terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + return -1; + } + + return 0; +} +#endif + +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *row, TSKEY minKey, TSKEY maxKey, + TSKEY now) { + TSKEY rowKey = TD_ROW_KEY(row); + if (rowKey < minKey || rowKey > maxKey) { + tsdbError("vgId:%d table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 + " maxKey %" PRId64 " row key %" PRId64, + REPO_ID(pTsdb), uid, now, minKey, maxKey, rowKey); + terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + return -1; + } + + return 0; +} + +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; SSubmitMsgIter msgIter = {0}; @@ -112,14 +143,14 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { return -1; } } - - tsdbInitSubmitBlkIter(pBlock, &blkIter); - while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { - if (tsdbCheckRowRange(pTsdb, pTable, row, minKey, maxKey, now) < 0) { +#endif + tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); + while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { + if (tsdbCheckRowRange(pTsdb, msgIter.uid, row, minKey, maxKey, now) < 0) { return -1; } } -#endif + } if (terrno != TSDB_CODE_SUCCESS) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 5c8cd362fd..a1a8e7bc88 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -631,6 +631,11 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); #endif + if (tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq) < 0) { + pRsp->code = terrno; + goto _exit; + } + // handle the request if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { pRsp->code = TSDB_CODE_INVALID_MSG; diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 13637b8fe4..4ce668e6ba 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -185,10 +185,10 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it pKey = SL_GET_NODE_KEY(pSkipList, p); compare = pSkipList->comparFn(pKey, pDataKey); - if (compare >= 0) { - if (compare == 0 && !hasDup) hasDup = true; + if (compare > 0) { break; } else { + if (compare == 0 && !hasDup) hasDup = true; px = p; p = SL_NODE_GET_FORWARD_POINTER(px, i); } diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index adbab04e07..bdda7c453b 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -57,7 +57,7 @@ class TDSql: tdLog.notice("'reset query cache' is not supported") s = 'drop database if exists db' self.cursor.execute(s) - s = 'create database db' + s = 'create database db days 300' self.cursor.execute(s) s = 'use db' self.cursor.execute(s) diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index 68860dc2cb..dcd88e5a0c 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -5,7 +5,7 @@ sleep 500 sql connect print =============== create database -sql create database d0 +sql create database d0 days 300 sql use d0 print =============== create super table and child table @@ -254,4 +254,4 @@ endi #sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT