From aae9b3c1bc5db5d2007eb467c35a5a7eafdea3a6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 17 Jul 2020 17:32:56 +0800 Subject: [PATCH 01/63] add a idx file for query speed --- src/tsdb/inc/tsdbMain.h | 34 +++-- src/tsdb/src/tsdbFile.c | 18 +-- src/tsdb/src/tsdbMain.c | 12 +- src/tsdb/src/tsdbMemTable.c | 7 +- src/tsdb/src/tsdbRWHelper.c | 261 +++++++++++++++++++----------------- 5 files changed, 178 insertions(+), 154 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 40f2dac660..da5f03a4fb 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -132,21 +132,23 @@ typedef struct { // ------------------ tsdbFile.c extern const char* tsdbFileSuffix[]; typedef enum { - TSDB_FILE_TYPE_HEAD = 0, + TSDB_FILE_TYPE_IDX = 0, + TSDB_FILE_TYPE_HEAD, TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX, + TSDB_FILE_TYPE_NIDX, TSDB_FILE_TYPE_NHEAD, TSDB_FILE_TYPE_NLAST } TSDB_FILE_TYPE; typedef struct { - uint32_t offset; + uint32_t magic; uint32_t len; - uint64_t size; // total size of the file - uint64_t tombSize; // unused file size uint32_t totalBlocks; uint32_t totalSubBlocks; + uint64_t size; // total size of the file + uint64_t tombSize; // unused file size } STsdbFileInfo; typedef struct { @@ -249,16 +251,12 @@ typedef struct { typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t; typedef struct { - int fid; - TSKEY minKey; - TSKEY maxKey; - // For read/write purpose - SFile headF; - SFile dataF; - SFile lastF; - // For write purpose only - SFile nHeadF; - SFile nLastF; + TSKEY minKey; + TSKEY maxKey; + SFileGroup fGroup; + SFile nIdxF; + SFile nHeadF; + SFile nLastF; } SHelperFile; typedef struct { @@ -444,6 +442,14 @@ void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); #define helperRepo(h) (h)->pRepo #define helperState(h) (h)->state #define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0) +#define helperFileId(h) ((h)->files.fGroup.fileId) +#define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX])) +#define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD])) +#define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA])) +#define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST])) +#define helperNewIdxF(h) (&((h)->files.nIdxF)) +#define helperNewHeadF(h) (&((h)->files.nHeadF)) +#define helperNewLastF(h) (&((h)->files.nLastF)) int tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo); int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 95cc47292b..7b7788c4d9 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -30,7 +30,7 @@ #include "ttime.h" #include "tfile.h" -const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; +const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"}; static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); @@ -108,7 +108,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { memset((void *)(&fileGroup), 0, sizeof(SFileGroup)); fileGroup.fileId = fid; - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbInitFile(&fileGroup.files[type], pRepo, fid, type) < 0) { tsdbError("vgId:%d failed to init file fid %d type %d", REPO_ID(pRepo), fid, type); goto _err; @@ -126,7 +126,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { return 0; _err: - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); tfree(tDataDir); if (dir != NULL) closedir(dir); @@ -139,7 +139,7 @@ void tsdbCloseFileH(STsdbRepo *pRepo) { for (int i = 0; i < pFileH->nFGroups; i++) { SFileGroup *pFGroup = pFileH->pFGroup + i; - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { tsdbDestroyFile(&pFGroup->files[type]); } } @@ -156,7 +156,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pGroup == NULL) { // if not exists, create one pFGroup->fileId = fid; - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) goto _err; } @@ -169,7 +169,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int return pGroup; _err: - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); return NULL; } @@ -323,7 +323,7 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { int tlen = 0; - tlen += taosEncodeFixedU32(buf, pInfo->offset); + tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->len); tlen += taosEncodeFixedU64(buf, pInfo->size); tlen += taosEncodeFixedU64(buf, pInfo->tombSize); @@ -334,7 +334,7 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { } void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { - buf = taosDecodeFixedU32(buf, &(pInfo->offset)); + buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->len)); buf = taosDecodeFixedU64(buf, &(pInfo->size)); buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); @@ -358,7 +358,7 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { pFileH->nFGroups--; ASSERT(pFileH->nFGroups >= 0); - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { if (remove(fileGroup.files[type].fname) < 0) { tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index ffaab375a3..848cd4e85a 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -214,7 +214,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ char *prefix = dirname(sdup); if (name[0] == 0) { // get the file from index or after, but not larger than eindex - int fid = (*index) / 3; + int fid = (*index) / TSDB_FILE_TYPE_MAX; if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { @@ -228,11 +228,11 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ SFileGroup *pFGroup = taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); if (pFGroup->fileId == fid) { - fname = strdup(pFGroup->files[(*index) % 3].fname); + fname = strdup(pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].fname); } else { - if (pFGroup->fileId * 3 + 2 < eindex) { + if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < eindex) { fname = strdup(pFGroup->files[0].fname); - *index = pFGroup->fileId * 3; + *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; } else { tfree(sdup); return 0; @@ -244,14 +244,14 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ if (*index == TSDB_META_FILE_INDEX) { // get meta file fname = tsdbGetMetaFileName(pRepo->rootDir); } else { - int fid = (*index) / 3; + int fid = (*index) / TSDB_FILE_TYPE_MAX; SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pFGroup == NULL) { // not found tfree(sdup); return 0; } - SFile *pFile = &pFGroup->files[(*index) % 3]; + SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; fname = strdup(pFile->fname); } } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index b29cec3cf9..69e6ab3c0f 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -628,9 +628,10 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe tsdbCloseHelperFile(pHelper, 0); pthread_rwlock_wrlock(&(pFileH->fhlock)); - pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF; - pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF; - pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF; + pGroup->files[TSDB_FILE_TYPE_IDX] = *(helperIdxF(pHelper)); + pGroup->files[TSDB_FILE_TYPE_HEAD] = *(helperHeadF(pHelper)); + pGroup->files[TSDB_FILE_TYPE_DATA] = *(helperDataF(pHelper)); + pGroup->files[TSDB_FILE_TYPE_LAST] = *(helperLastF(pHelper)); pthread_rwlock_unlock(&(pFileH->fhlock)); return 0; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 0d52b7ae33..79b4a8ef64 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -106,44 +106,36 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { ASSERT(pHelper->state == TSDB_HELPER_CLEAR_STATE); // Set the files - pHelper->files.fid = pGroup->fileId; - pHelper->files.headF = pGroup->files[TSDB_FILE_TYPE_HEAD]; - pHelper->files.dataF = pGroup->files[TSDB_FILE_TYPE_DATA]; - pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; + pHelper->files.fGroup = *pGroup; if (helperType(pHelper) == TSDB_WRITE_HELPER) { - tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, pHelper->files.nHeadF.fname); - tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, pHelper->files.nLastF.fname); + tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NIDX, helperNewIdxF(pHelper)->fname); + tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, helperNewHeadF(pHelper)->fname); + tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, helperNewLastF(pHelper)->fname); } // Open the files - if (tsdbOpenFile(&(pHelper->files.headF), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) goto _err; if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (tsdbOpenFile(&(pHelper->files.dataF), O_RDWR) < 0) goto _err; - if (tsdbOpenFile(&(pHelper->files.lastF), O_RDWR) < 0) goto _err; + if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) goto _err; + if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) goto _err; + + // Create and open .i file + if (tsdbOpenFile(helperNewIdxF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; + if (tsdbUpdateFileHeader(helperNewIdxF(pHelper), 0) < 0) return -1; // Create and open .h - if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) return -1; - // size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM); - if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), - TSDB_FILE_HEAD_SIZE, pHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + if (tsdbOpenFile(helperNewHeadF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; + if (tsdbUpdateFileHeader(helperNewHeadF(pHelper), 0) < 0) return -1; // Create and open .l file if should if (tsdbShouldCreateNewLast(pHelper)) { - if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err; - if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), - TSDB_FILE_HEAD_SIZE, pHelper->files.lastF.fname, pHelper->files.nLastF.fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + if (tsdbOpenFile(helperNewLastF(pHelper), O_WRONLY | O_CREAT) < 0) goto _err; + if (tsdbUpdateFileHeader(helperNewLastF(pHelper), 0) < 0) return -1; } } else { - if (tsdbOpenFile(&(pHelper->files.dataF), O_RDONLY) < 0) goto _err; - if (tsdbOpenFile(&(pHelper->files.lastF), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperDataF(pHelper), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperLastF(pHelper), O_RDONLY) < 0) goto _err; } helperSetState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN); @@ -155,59 +147,94 @@ _err: } int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { - if (pHelper->files.headF.fd > 0) { - close(pHelper->files.headF.fd); - pHelper->files.headF.fd = -1; + SFile *pFile = NULL; + + pFile = helperIdxF(pHelper); + if (pFile->fd > 0) { + close(pFile->fd); + pFile->fd = -1; } - if (pHelper->files.dataF.fd > 0) { + + pFile = helperHeadF(pHelper); + if (pFile->fd > 0) { + close(pFile->fd); + pFile->fd = -1; + } + + pFile = helperDataF(pHelper); + if (pFile->fd > 0) { if (helperType(pHelper) == TSDB_WRITE_HELPER) { - tsdbUpdateFileHeader(&(pHelper->files.dataF), 0); - fsync(pHelper->files.dataF.fd); + tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); } - close(pHelper->files.dataF.fd); - pHelper->files.dataF.fd = -1; + close(pFile->fd); + pFile->fd = -1; } - if (pHelper->files.lastF.fd > 0) { - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - fsync(pHelper->files.lastF.fd); + + pFile = helperLastF(pHelper); + if (pFile->fd > 0) { + if (helperType(pHelper) == TSDB_WRITE_HELPER && !TSDB_NLAST_FILE_OPENED(pHelper)) { + fsync(pFile->fd); } - close(pHelper->files.lastF.fd); - pHelper->files.lastF.fd = -1; + close(pFile->fd); + pFile->fd = -1; } + if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (pHelper->files.nHeadF.fd > 0) { - if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nHeadF), 0); - fsync(pHelper->files.nHeadF.fd); - close(pHelper->files.nHeadF.fd); - pHelper->files.nHeadF.fd = -1; + pFile = helperNewIdxF(pHelper); + if (pFile->fd > 0) { + if (!hasError) tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + close(pFile->fd); + pFile->fd = -1; if (hasError) { - (void)remove(pHelper->files.nHeadF.fname); + (void)remove(pFile->fname); } else { - if (rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname) < 0) { - tsdbError("failed to rename file from %s to %s since %s", pHelper->files.nHeadF.fname, - pHelper->files.headF.fname, strerror(errno)); + if (rename(pFile->fname, helperIdxF(pHelper)->fname) < 0) { + tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperIdxF(pHelper)->fname, + strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pHelper->files.headF.info = pHelper->files.nHeadF.info; + helperIdxF(pHelper)->info = pFile->info; } } - if (pHelper->files.nLastF.fd > 0) { - if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nLastF), 0); - fsync(pHelper->files.nLastF.fd); - close(pHelper->files.nLastF.fd); - pHelper->files.nLastF.fd = -1; + pFile = helperNewHeadF(pHelper); + if (pFile->fd > 0) { + if (!hasError) tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + close(pFile->fd); + pFile->fd = -1; if (hasError) { - (void)remove(pHelper->files.nLastF.fname); + (void)remove(pFile->fname); } else { - if (rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname) < 0) { - tsdbError("failed to rename file from %s to %s since %s", pHelper->files.nLastF.fname, - pHelper->files.lastF.fname, strerror(errno)); + if (rename(pFile->fname, helperHeadF(pHelper)->fname) < 0) { + tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperHeadF(pHelper)->fname, + strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pHelper->files.lastF.info = pHelper->files.nLastF.info; + helperHeadF(pHelper)->info = pFile->info; + } + } + + pFile = helperNewLastF(pHelper); + if (pFile->fd > 0) { + if (!hasError) tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + close(pFile->fd); + pFile->fd = -1; + if (hasError) { + (void)remove(pFile->fname); + } else { + if (rename(pFile->fname, helperLastF(pHelper)->fname) < 0) { + tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperLastF(pHelper)->fname, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + helperLastF(pHelper)->info = helperNewLastF(pHelper)->info; } } } @@ -283,28 +310,28 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], &compBlock, true, true) < 0) + if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0) return -1; if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; } else { - if (lseek(pHelper->files.lastF.fd, pCompBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.lastF.fname, + if (lseek(helperLastF(pHelper)->fd, pCompBlock->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperLastF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pCompBlock->offset = lseek(pHelper->files.nLastF.fd, 0, SEEK_END); + pCompBlock->offset = lseek(helperNewLastF(pHelper)->fd, 0, SEEK_END); if (pCompBlock->offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nLastF.fname, + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewLastF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, pCompBlock->len) < pCompBlock->len) { + if (tsendfile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) { tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo), - pHelper->files.lastF.fname, pHelper->files.nLastF.fname, strerror(errno)); + helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -321,9 +348,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { if (pIdx->offset > 0) { - offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); + offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); if (offset < 0) { - tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname, + tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -332,9 +359,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->offset = offset; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) { + if (tsendfile(helperNewHeadF(pHelper)->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to send %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno)); + helperHeadF(pHelper)->fname, helperNewHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -347,9 +374,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) && (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); - offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); + offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname, + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -358,9 +385,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->uid = pHelper->tableInfo.uid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { + if (twrite(helperNewHeadF(pHelper)->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pHelper->files.nHeadF.fname, strerror(errno)); + helperNewHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -371,19 +398,10 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } int tsdbWriteCompIdx(SRWHelper *pHelper) { + ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); STsdbCfg *pCfg = &pHelper->pRepo->config; - ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - off_t offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s to end since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - SFile *pFile = &(pHelper->files.nHeadF); - pFile->info.offset = offset; + SFile *pFile = helperNewIdxF(pHelper); void *buf = pHelper->pBuffer; for (uint32_t i = 0; i < pCfg->maxTables; i++) { @@ -406,9 +424,9 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { int tsize = (char *)buf - (char *)pHelper->pBuffer + sizeof(TSCKSUM); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pBuffer, tsize); - if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pBuffer, tsize) < tsize) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), tsize, - pHelper->files.nHeadF.fname, strerror(errno)); + if (twrite(pFile->fd, (void *)pHelper->pBuffer, tsize) < tsize) { + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname, + strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -423,23 +441,21 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object - SFile *pFile = &(pHelper->files.headF); + SFile *pFile = helperIdxF(pHelper); int fd = pFile->fd; memset(pHelper->pCompIdx, 0, tsizeof(pHelper->pCompIdx)); - if (pFile->info.offset > 0) { - ASSERT(pFile->info.offset > TSDB_FILE_HEAD_SIZE); - - if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s to %u since %s", REPO_ID(pHelper->pRepo), pFile->fname, - pFile->info.offset, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + if (pFile->info.len > 0) { if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } + if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, pFile->fname, strerror(errno)); @@ -447,8 +463,8 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { return -1; } if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pBuffer), pFile->info.len)) { - tsdbError("vgId:%d file %s SCompIdx part is corrupted. offset %u len %u", REPO_ID(pHelper->pRepo), pFile->fname, - pFile->info.offset, pFile->info.len); + tsdbError("vgId:%d file %s SCompIdx part is corrupted. len %u", REPO_ID(pHelper->pRepo), pFile->fname, + pFile->info.len); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } @@ -484,13 +500,13 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - int fd = pHelper->files.headF.fd; + int fd = helperHeadF(pHelper)->fd; if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { if (pIdx->offset > 0) { ASSERT(pIdx->uid == pHelper->tableInfo.uid); if (lseek(fd, pIdx->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.headF.fname, + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -499,13 +515,13 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len); if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pHelper->files.headF.fname, strerror(errno)); + helperHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompInfo, pIdx->len)) { tsdbError("vgId:%d file %s SCompInfo part is corrupted, tid %d uid %" PRIu64, REPO_ID(pHelper->pRepo), - pHelper->files.headF.fname, pHelper->tableInfo.tid, pHelper->tableInfo.uid); + helperHeadF(pHelper)->fname, pHelper->tableInfo.tid, pHelper->tableInfo.uid); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } @@ -523,7 +539,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { ASSERT(pCompBlock->numOfSubBlocks <= 1); - SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); if (lseek(pFile->fd, pCompBlock->offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); @@ -642,9 +658,9 @@ _err: // ---------------------- INTERNAL FUNCTIONS ---------------------- static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { - ASSERT(pHelper->files.lastF.fd > 0); + ASSERT(helperLastF(pHelper)->fd > 0); struct stat st; - if (fstat(pHelper->files.lastF.fd, &st) < 0) return true; + if (fstat(helperLastF(pHelper)->fd, &st) < 0) return true; if (st.st_size > 32 * 1024 + TSDB_FILE_HEAD_SIZE) return true; return false; } @@ -972,12 +988,13 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); - pHelper->files.fid = -1; - pHelper->files.headF.fd = -1; - pHelper->files.dataF.fd = -1; - pHelper->files.lastF.fd = -1; - pHelper->files.nHeadF.fd = -1; - pHelper->files.nLastF.fd = -1; + helperIdxF(pHelper)->fd = -1; + helperHeadF(pHelper)->fd = -1; + helperDataF(pHelper)->fd = -1; + helperLastF(pHelper)->fd = -1; + helperNewIdxF(pHelper)->fd = -1; + helperNewHeadF(pHelper)->fd = -1; + helperNewLastF(pHelper)->fd = -1; } static int tsdbInitHelperFile(SRWHelper *pHelper) { @@ -1154,7 +1171,7 @@ static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, ASSERT(pCompBlock->numOfSubBlocks <= 1); ASSERT(colIds[0] == 0); - SFile * pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + SFile * pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); SCompCol compCol = {0}; // If only load timestamp column, no need to load SCompData part @@ -1215,7 +1232,7 @@ _err: static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) { ASSERT(pCompBlock->numOfSubBlocks <= 1); - SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); pHelper->pBuffer = trealloc(pHelper->pBuffer, pCompBlock->len); if (pHelper->pBuffer == NULL) { @@ -1362,7 +1379,7 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, &compBlock, true, false) < 0) return -1; + if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1; } else { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; @@ -1427,7 +1444,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols, pDataCols0->cols[0].pData, pDataCols0->numOfRows); ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, &compBlock, true, false) < 0) return -1; + if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; } else { @@ -1466,7 +1483,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (rowsRead == 0) break; ASSERT(rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, &compBlock, false, true) < 0) return -1; + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; tblkIdx++; } @@ -1493,7 +1510,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols, pDataCols0->cols[0].pData, pDataCols0->numOfRows); ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, &compBlock, false, false) < 0) + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; @@ -1506,7 +1523,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock); if (rowsRead == 0) break; - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, &compBlock, false, true) < 0) + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (round == 0) { if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; @@ -1577,10 +1594,10 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, ASSERT(pDataCols->numOfRows > 0); if (pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { - pFile = &(pHelper->files.dataF); + pFile = helperDataF(pHelper); } else { isLast = true; - pFile = TSDB_NLAST_FILE_OPENED(pHelper) ? &(pHelper->files.nLastF) : &(pHelper->files.lastF); + pFile = TSDB_NLAST_FILE_OPENED(pHelper) ? helperNewLastF(pHelper) : helperLastF(pHelper); } ASSERT(pFile->fd > 0); From 31824d9ffa0c05b52d0d1d21ddd806b9bc372267 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 20 Jul 2020 15:01:37 +0800 Subject: [PATCH 02/63] get rid of maxTables in TSDB --- src/tsdb/inc/tsdbMain.h | 13 ++- src/tsdb/src/tsdbMain.c | 2 +- src/tsdb/src/tsdbRWHelper.c | 186 +++++++++++++++++++++++------------- src/tsdb/src/tsdbRead.c | 6 +- 4 files changed, 131 insertions(+), 76 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index da5f03a4fb..9346230e02 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -199,6 +199,7 @@ typedef struct { // ------------------ tsdbRWHelper.c typedef struct { + int32_t tid; uint32_t len; uint32_t offset; uint32_t hasLast : 2; @@ -262,9 +263,14 @@ typedef struct { typedef struct { uint64_t uid; int32_t tid; - int32_t sversion; } SHelperTable; +typedef struct { + SCompIdx* pIdxArray; + int numOfIdx; + int curIdx; +} SIdxH; + typedef struct { tsdb_rw_helper_t type; @@ -272,7 +278,9 @@ typedef struct { int8_t state; // For file set usage SHelperFile files; - SCompIdx* pCompIdx; + SIdxH idxH; + SCompIdx curCompIdx; + void* pWIdx; // For table set usage SHelperTable tableInfo; SCompInfo* pCompInfo; @@ -284,7 +292,6 @@ typedef struct { void* compBuffer; // Buffer for temperary compress/decompress purpose } SRWHelper; - // Operations // ------------------ tsdbMeta.c #define TABLE_TYPE(t) (t)->type diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index c8b31af4f0..82dbe5f498 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -793,7 +793,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { for (int i = 1; i < pRepo->config.maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; - SCompIdx *pIdx = &rhelper.pCompIdx[i]; + SCompIdx *pIdx = &(rhelper.curCompIdx); if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; } diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 79b4a8ef64..19cb791ce8 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -60,6 +60,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter int *blkIdx); static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows); +static int tsdbCompareTidIdx(const void *key1, const void *key2); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -251,17 +252,35 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { pHelper->tableInfo.tid = pTable->tableId.tid; pHelper->tableInfo.uid = pTable->tableId.uid; STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - pHelper->tableInfo.sversion = schemaVersion(pSchema); tdInitDataCols(pHelper->pDataCols[0], pSchema); tdInitDataCols(pHelper->pDataCols[1], pSchema); - SCompIdx *pIdx = pHelper->pCompIdx + pTable->tableId.tid; - if (pIdx->offset > 0) { - if (pIdx->uid != TABLE_UID(pTable)) { - memset((void *)pIdx, 0, sizeof(SCompIdx)); + if (helperType(pHelper) == TSDB_WRITE_HELPER) { + if (pHelper->idxH.numOfIdx > 0) { + if (pHelper->idxH.curIdx >= pHelper->idxH.numOfIdx) { + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + } else { + SCompIdx *pIdx = &(pHelper->idxH.pIdxArray[pHelper->idxH.curIdx]); + if (pIdx->tid == TABLE_TID(pTable)) { + pHelper->curCompIdx = *pIdx; + pHelper->idxH.curIdx++; + } else { + ASSERT(pIdx->tid > TABLE_TID(pTable)); + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + } + } } else { - if (pIdx->hasLast) pHelper->hasOldLastBlock = true; + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + } + } else { + // TODO: make it more efficient + void *ptr = bsearch(&TABLE_TID(pTable), (void *)pHelper->idxH.pIdxArray, pHelper->idxH.numOfIdx, sizeof(SCompIdx), + tsdbCompareTidIdx); + if (ptr == NULL) { + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + } else { + pHelper->curCompIdx = *(SCompIdx *)ptr; } } @@ -272,8 +291,8 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - SCompIdx * pIdx = &(pHelper->pCompIdx[TABLE_TID(pCommitIter->pTable)]); - int blkIdx = 0; + SCompIdx *pIdx = &(pHelper->curCompIdx); + int blkIdx = 0; ASSERT(pIdx->offset == 0 || pIdx->uid == TABLE_UID(pCommitIter->pTable)); if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; @@ -298,7 +317,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { STsdbCfg *pCfg = &pHelper->pRepo->config; ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx * pIdx = &(pHelper->curCompIdx); SCompBlock compBlock = {0}; if (TSDB_NLAST_FILE_OPENED(pHelper) && (pHelper->hasOldLastBlock)) { if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; @@ -344,10 +363,11 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { } int tsdbWriteCompInfo(SRWHelper *pHelper) { - off_t offset = 0; - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - if (pIdx->offset > 0) { + SCompIdx *pIdx = &(pHelper->curCompIdx); + off_t offset = 0; + + if (pIdx->len > 0) { + if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); if (offset < 0) { tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, @@ -357,6 +377,8 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } pIdx->offset = offset; + pIdx->uid = pHelper->tableInfo.uid; + pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); if (tsendfile(helperNewHeadF(pHelper)->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { @@ -365,9 +387,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - } - } else { - if (pIdx->len > 0) { + } else { pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER; pHelper->pCompInfo->uid = pHelper->tableInfo.uid; pHelper->pCompInfo->checksum = 0; @@ -383,6 +403,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } pIdx->offset = offset; pIdx->uid = pHelper->tableInfo.uid; + pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); if (twrite(helperNewHeadF(pHelper)->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { @@ -392,6 +413,17 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { return -1; } } + + if (tsizeof(pHelper->pWIdx) < helperNewIdxF(pHelper)->info.len + sizeof(SCompIdx) + 12) { + pHelper->pWIdx = trealloc(pHelper->pWIdx, tsizeof(pHelper->pWIdx) == 0 ? 1024 : tsizeof(pHelper->pWIdx) * 2); + if (pHelper->pWIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } + + void *pBuf = POINTER_SHIFT(pHelper->pWIdx, helperNewIdxF(pHelper)->info.len); + helperNewIdxF(pHelper)->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); } return 0; @@ -399,57 +431,43 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { int tsdbWriteCompIdx(SRWHelper *pHelper) { ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - STsdbCfg *pCfg = &pHelper->pRepo->config; + // STsdbCfg *pCfg = &pHelper->pRepo->config; SFile *pFile = helperNewIdxF(pHelper); - void *buf = pHelper->pBuffer; - for (uint32_t i = 0; i < pCfg->maxTables; i++) { - SCompIdx *pCompIdx = pHelper->pCompIdx + i; - if (pCompIdx->offset > 0) { - int drift = POINTER_DISTANCE(buf, pHelper->pBuffer); - if (tsizeof(pHelper->pBuffer) - drift < 128) { - pHelper->pBuffer = trealloc(pHelper->pBuffer, tsizeof(pHelper->pBuffer) * 2); - if (pHelper->pBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - buf = POINTER_SHIFT(pHelper->pBuffer, drift); - taosEncodeVariantU32(&buf, i); - tsdbEncodeSCompIdx(&buf, pCompIdx); + pFile->info.len += sizeof(TSCKSUM); + if (tsizeof(pHelper->pWIdx) < pFile->info.len) { + pHelper->pWIdx = trealloc(pHelper->pWIdx, pFile->info.len); + if (pHelper->pWIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; } } + taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len); - int tsize = (char *)buf - (char *)pHelper->pBuffer + sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)pHelper->pBuffer, tsize); - - if (twrite(pFile->fd, (void *)pHelper->pBuffer, tsize) < tsize) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname, - strerror(errno)); + if (twrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) { + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, + pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pFile->info.len = tsize; + return 0; } int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { - STsdbCfg *pCfg = &(pHelper->pRepo->config); - ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); + SFile *pFile = helperIdxF(pHelper); + int fd = pFile->fd; if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object - SFile *pFile = helperIdxF(pHelper); - int fd = pFile->fd; - - memset(pHelper->pCompIdx, 0, tsizeof(pHelper->pCompIdx)); if (pFile->info.len > 0) { if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } + if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -462,6 +480,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pBuffer), pFile->info.len)) { tsdbError("vgId:%d file %s SCompIdx part is corrupted. len %u", REPO_ID(pHelper->pRepo), pFile->fname, pFile->info.len); @@ -470,27 +489,49 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { } // Decode it + pHelper->idxH.numOfIdx = 0; void *ptr = pHelper->pBuffer; while (POINTER_DISTANCE(ptr, pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) { - uint32_t tid = 0; - if ((ptr = taosDecodeVariantU32(ptr, &tid)) == NULL) return -1; - ASSERT(tid > 0 && tid < pCfg->maxTables); + size_t tlen = tsizeof(pHelper->idxH.pIdxArray); + pHelper->idxH.numOfIdx++; - if ((ptr = tsdbDecodeSCompIdx(ptr, pHelper->pCompIdx + tid)) == NULL) return -1; + if (tlen < pHelper->idxH.numOfIdx) { + pHelper->idxH.pIdxArray = (SCompIdx *)trealloc(pHelper->idxH.pIdxArray, (tlen == 0) ? 1024 : tlen * 2); + if (pHelper->idxH.pIdxArray == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } + + ptr = tsdbDecodeSCompIdx(ptr, &(pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 1])); + if (ptr == NULL) { + tsdbError("vgId:%d file %s SCompIdx part is corrupted. len %u", REPO_ID(pHelper->pRepo), pFile->fname, + pFile->info.len); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + + ASSERT(pHelper->idxH.numOfIdx == 1 || pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 1].tid > + pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 2].tid); ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= pFile->info.len - sizeof(TSCKSUM)); } - if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + // if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { + // terrno = TAOS_SYSTEM_ERROR(errno); + // return -1; + // } } } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); + if (helperType(pHelper) == TSDB_WRITE_HELPER) { + pFile->info.len = 0; + } + // Copy the memory for outside usage - if (target) memcpy(target, pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)); + if (target && pHelper->idxH.numOfIdx > 0) + memcpy(target, pHelper->idxH.pIdxArray, sizeof(SCompIdx) * pHelper->idxH.numOfIdx); return 0; } @@ -498,7 +539,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); int fd = helperHeadF(pHelper)->fd; @@ -820,7 +861,7 @@ static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) { } static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfBlocks); ASSERT(pCompBlock->numOfSubBlocks == 1); @@ -867,7 +908,7 @@ _err: static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) { ASSERT(pCompBlock->numOfSubBlocks == 0); - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks); SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; @@ -951,7 +992,7 @@ _err: static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { ASSERT(pCompBlock->numOfSubBlocks == 1); - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks); @@ -987,6 +1028,8 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int } static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { + pHelper->idxH.numOfIdx = 0; + pHelper->idxH.curIdx = 0; memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); helperIdxF(pHelper)->fd = -1; helperHeadF(pHelper)->fd = -1; @@ -998,14 +1041,6 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { } static int tsdbInitHelperFile(SRWHelper *pHelper) { - STsdbCfg *pCfg = &pHelper->pRepo->config; - size_t tsize = sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM); - pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize); - if (pHelper->pCompIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - tsdbResetHelperFileImpl(pHelper); return 0; } @@ -1013,7 +1048,8 @@ static int tsdbInitHelperFile(SRWHelper *pHelper) { static void tsdbDestroyHelperFile(SRWHelper *pHelper) { tsdbCloseHelperFile(pHelper, false); tsdbResetHelperFileImpl(pHelper); - tzfree(pHelper->pCompIdx); + tzfree(pHelper->idxH.pIdxArray); + tzfree(pHelper->pWIdx); } // ---------- Operations on Helper Table part @@ -1331,6 +1367,7 @@ _err: static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx) { int tlen = 0; + tlen += taosEncodeVariantI32(buf, pIdx->tid); tlen += taosEncodeVariantU32(buf, pIdx->len); tlen += taosEncodeVariantU32(buf, pIdx->offset); tlen += taosEncodeFixedU8(buf, pIdx->hasLast); @@ -1346,6 +1383,7 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { uint32_t numOfBlocks = 0; uint64_t value = 0; + if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL; if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; @@ -1363,7 +1401,7 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { STsdbCfg * pCfg = &(pHelper->pRepo->config); STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = pHelper->pCompIdx + TABLE_TID(pTable); + SCompIdx * pIdx = &(pHelper->curCompIdx); TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; SCompBlock compBlock = {0}; @@ -1410,7 +1448,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int *blkIdx) { STsdbCfg * pCfg = &(pHelper->pRepo->config); STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = pHelper->pCompIdx + TABLE_TID(pTable); + SCompIdx * pIdx = &(pHelper->curCompIdx); SCompBlock compBlock = {0}; TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; @@ -1605,4 +1643,14 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, pCompBlock, isLast, true) < 0) return -1; return 0; +} + +static int tsdbCompareTidIdx(const void *key1, const void *key2) { + if (*(int32_t *)key1 > ((SCompIdx *)key2)->tid) { + return 1; + } else if (*(int32_t *)key1 < ((SCompIdx *)key2)->tid) { + return -1; + } else { + return 0; + } } \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 5b1afe3da8..0d76c7f7bf 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -555,7 +555,9 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); pCheckInfo->numOfBlocks = 0; - SCompIdx* compIndex = &pQueryHandle->rhelper.pCompIdx[pCheckInfo->tableId.tid]; + tsdbSetHelperTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj, pQueryHandle->pTsdb); + + SCompIdx* compIndex = &pQueryHandle->rhelper.curCompIdx; // no data block in this file, try next file if (compIndex->len == 0 || compIndex->numOfBlocks == 0 || compIndex->uid != pCheckInfo->tableId.uid) { @@ -572,8 +574,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo pCheckInfo->compSize = compIndex->len; } - tsdbSetHelperTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj, pQueryHandle->pTsdb); - tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo)); SCompInfo* pCompInfo = pCheckInfo->pCompInfo; From 9bb9d7a2a857eac311907bf81f23d1d8d5bbc8d5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 20 Jul 2020 15:19:15 +0800 Subject: [PATCH 03/63] use malloc for more efficent --- src/common/src/tdataformat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 719d80aa77..12ea4ad78d 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -318,7 +318,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { pCols->maxPoints = maxRows; pCols->bufSize = maxRowSize * maxRows; - pCols->buf = calloc(1, pCols->bufSize); + pCols->buf = malloc(pCols->bufSize); if (pCols->buf == NULL) { free(pCols); return NULL; From e57e4a6d654d968b5588b620d3a867eaa4782f2a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 20 Jul 2020 16:06:00 +0800 Subject: [PATCH 04/63] add gcov to support coverage test. --- src/query/tests/CMakeLists.txt | 4 ++-- src/util/tests/CMakeLists.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index 0ae8600756..1856223391 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -11,5 +11,5 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread) -ENDIF() \ No newline at end of file + TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread gcov) +ENDIF() diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt index 9f66eba37a..b0b5d3013b 100644 --- a/src/util/tests/CMakeLists.txt +++ b/src/util/tests/CMakeLists.txt @@ -11,5 +11,5 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread) -ENDIF() \ No newline at end of file + TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread gcov) +ENDIF() From dd9f62cfe3fd52a70f27e5852b1a9cc07e1474e1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 20 Jul 2020 16:14:55 +0800 Subject: [PATCH 05/63] remove idx file back --- src/tsdb/inc/tsdbMain.h | 11 +++++- src/tsdb/src/tsdbFile.c | 26 ++++++++------ src/tsdb/src/tsdbMemTable.c | 2 ++ src/tsdb/src/tsdbRWHelper.c | 71 ++++++++++++++++++++++++++----------- 4 files changed, 78 insertions(+), 32 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 9346230e02..8ab89d72ed 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -132,12 +132,18 @@ typedef struct { // ------------------ tsdbFile.c extern const char* tsdbFileSuffix[]; typedef enum { +#ifdef TSDB_IDX TSDB_FILE_TYPE_IDX = 0, TSDB_FILE_TYPE_HEAD, +#else + TSDB_FILE_TYPE_HEAD = 0, +#endif TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX, +#ifdef TSDB_IDX TSDB_FILE_TYPE_NIDX, +#endif TSDB_FILE_TYPE_NHEAD, TSDB_FILE_TYPE_NLAST } TSDB_FILE_TYPE; @@ -147,6 +153,7 @@ typedef struct { uint32_t len; uint32_t totalBlocks; uint32_t totalSubBlocks; + uint32_t offset; uint64_t size; // total size of the file uint64_t tombSize; // unused file size } STsdbFileInfo; @@ -450,11 +457,13 @@ void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); #define helperState(h) (h)->state #define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0) #define helperFileId(h) ((h)->files.fGroup.fileId) +#ifdef TSDB_IDX #define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX])) +#define helperNewIdxF(h) (&((h)->files.nIdxF)) +#endif #define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD])) #define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA])) #define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST])) -#define helperNewIdxF(h) (&((h)->files.nIdxF)) #define helperNewHeadF(h) (&((h)->files.nHeadF)) #define helperNewLastF(h) (&((h)->files.nLastF)) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 7b7788c4d9..0a18d15773 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -30,7 +30,11 @@ #include "ttime.h" #include "tfile.h" +#ifdef TSDB_IDX const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"}; +#else +const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; +#endif static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); @@ -108,7 +112,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { memset((void *)(&fileGroup), 0, sizeof(SFileGroup)); fileGroup.fileId = fid; - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbInitFile(&fileGroup.files[type], pRepo, fid, type) < 0) { tsdbError("vgId:%d failed to init file fid %d type %d", REPO_ID(pRepo), fid, type); goto _err; @@ -126,7 +130,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { return 0; _err: - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); tfree(tDataDir); if (dir != NULL) closedir(dir); @@ -139,7 +143,7 @@ void tsdbCloseFileH(STsdbRepo *pRepo) { for (int i = 0; i < pFileH->nFGroups; i++) { SFileGroup *pFGroup = pFileH->pFGroup + i; - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { tsdbDestroyFile(&pFGroup->files[type]); } } @@ -156,7 +160,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pGroup == NULL) { // if not exists, create one pFGroup->fileId = fid; - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) goto _err; } @@ -169,7 +173,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int return pGroup; _err: - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); return NULL; } @@ -325,10 +329,11 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { int tlen = 0; tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->len); - tlen += taosEncodeFixedU64(buf, pInfo->size); - tlen += taosEncodeFixedU64(buf, pInfo->tombSize); tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks); tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks); + tlen += taosEncodeFixedU32(buf, pInfo->offset); + tlen += taosEncodeFixedU64(buf, pInfo->size); + tlen += taosEncodeFixedU64(buf, pInfo->tombSize); return tlen; } @@ -336,10 +341,11 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->len)); - buf = taosDecodeFixedU64(buf, &(pInfo->size)); - buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); + buf = taosDecodeFixedU32(buf, &(pInfo->offset)); + buf = taosDecodeFixedU64(buf, &(pInfo->size)); + buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); return buf; } @@ -358,7 +364,7 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { pFileH->nFGroups--; ASSERT(pFileH->nFGroups >= 0); - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (remove(fileGroup.files[type].fname) < 0) { tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 69e6ab3c0f..03fa9541c9 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -628,7 +628,9 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe tsdbCloseHelperFile(pHelper, 0); pthread_rwlock_wrlock(&(pFileH->fhlock)); +#ifdef TSDB_IDX pGroup->files[TSDB_FILE_TYPE_IDX] = *(helperIdxF(pHelper)); +#endif pGroup->files[TSDB_FILE_TYPE_HEAD] = *(helperHeadF(pHelper)); pGroup->files[TSDB_FILE_TYPE_DATA] = *(helperDataF(pHelper)); pGroup->files[TSDB_FILE_TYPE_LAST] = *(helperLastF(pHelper)); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 19cb791ce8..22182694de 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -109,21 +109,27 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Set the files pHelper->files.fGroup = *pGroup; if (helperType(pHelper) == TSDB_WRITE_HELPER) { +#ifdef TSDB_IDX tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NIDX, helperNewIdxF(pHelper)->fname); +#endif tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, helperNewHeadF(pHelper)->fname); tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, helperNewLastF(pHelper)->fname); } // Open the files +#ifdef TSDB_IDX if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) goto _err; +#endif if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) goto _err; if (helperType(pHelper) == TSDB_WRITE_HELPER) { if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) goto _err; if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) goto _err; +#ifdef TSDB_IDX // Create and open .i file if (tsdbOpenFile(helperNewIdxF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; if (tsdbUpdateFileHeader(helperNewIdxF(pHelper), 0) < 0) return -1; +#endif // Create and open .h if (tsdbOpenFile(helperNewHeadF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; @@ -150,11 +156,13 @@ _err: int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { SFile *pFile = NULL; +#ifdef TSDB_IDX pFile = helperIdxF(pHelper); if (pFile->fd > 0) { close(pFile->fd); pFile->fd = -1; } +#endif pFile = helperHeadF(pHelper); if (pFile->fd > 0) { @@ -182,6 +190,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { } if (helperType(pHelper) == TSDB_WRITE_HELPER) { +#ifdef TSDB_IDX pFile = helperNewIdxF(pHelper); if (pFile->fd > 0) { if (!hasError) tsdbUpdateFileHeader(pFile, 0); @@ -200,6 +209,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { helperIdxF(pHelper)->info = pFile->info; } } +#endif pFile = helperNewHeadF(pHelper); if (pFile->fd > 0) { @@ -365,12 +375,13 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { int tsdbWriteCompInfo(SRWHelper *pHelper) { SCompIdx *pIdx = &(pHelper->curCompIdx); off_t offset = 0; + SFile * pFile = helperNewHeadF(pHelper); if (pIdx->len > 0) { if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); + offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) { - tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, + tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -381,9 +392,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (tsendfile(helperNewHeadF(pHelper)->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { + if (tsendfile(pFile->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to send %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - helperHeadF(pHelper)->fname, helperNewHeadF(pHelper)->fname, strerror(errno)); + helperHeadF(pHelper)->fname, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -394,9 +405,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) && (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); - offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); + offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -406,15 +417,19 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (twrite(helperNewHeadF(pHelper)->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { + if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - helperNewHeadF(pHelper)->fname, strerror(errno)); + pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - if (tsizeof(pHelper->pWIdx) < helperNewIdxF(pHelper)->info.len + sizeof(SCompIdx) + 12) { +#ifdef TSDB_IDX + pFile = helperNewIdxF(pHelper); +#endif + + if (tsizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) { pHelper->pWIdx = trealloc(pHelper->pWIdx, tsizeof(pHelper->pWIdx) == 0 ? 1024 : tsizeof(pHelper->pWIdx) * 2); if (pHelper->pWIdx == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -422,8 +437,8 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } } - void *pBuf = POINTER_SHIFT(pHelper->pWIdx, helperNewIdxF(pHelper)->info.len); - helperNewIdxF(pHelper)->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); + void *pBuf = POINTER_SHIFT(pHelper->pWIdx, pFile->info.len); + pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); } return 0; @@ -431,9 +446,13 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { int tsdbWriteCompIdx(SRWHelper *pHelper) { ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - // STsdbCfg *pCfg = &pHelper->pRepo->config; + off_t offset = 0; +#ifdef TSDB_IDX SFile *pFile = helperNewIdxF(pHelper); +#else + SFile *pFile = helperNewHeadF(pHelper); +#endif pFile->info.len += sizeof(TSCKSUM); if (tsizeof(pHelper->pWIdx) < pFile->info.len) { @@ -445,6 +464,15 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { } taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len); + offset = lseek(pFile->fd, 0, SEEK_END); + if (offset < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + pFile->info.offset = offset; + if (twrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, pFile->fname, strerror(errno)); @@ -457,8 +485,12 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); +#ifdef TSDB_IDX SFile *pFile = helperIdxF(pHelper); - int fd = pFile->fd; +#else + SFile *pFile = helperHeadF(pHelper); +#endif + int fd = pFile->fd; if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object @@ -468,7 +500,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { return -1; } - if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { + if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -516,11 +548,6 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= pFile->info.len - sizeof(TSCKSUM)); } - - // if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { - // terrno = TAOS_SYSTEM_ERROR(errno); - // return -1; - // } } } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); @@ -1031,13 +1058,15 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { pHelper->idxH.numOfIdx = 0; pHelper->idxH.curIdx = 0; memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); - helperIdxF(pHelper)->fd = -1; helperHeadF(pHelper)->fd = -1; helperDataF(pHelper)->fd = -1; helperLastF(pHelper)->fd = -1; - helperNewIdxF(pHelper)->fd = -1; helperNewHeadF(pHelper)->fd = -1; helperNewLastF(pHelper)->fd = -1; +#ifdef TSDB_IDX + helperIdxF(pHelper)->fd = -1; + helperNewIdxF(pHelper)->fd = -1; +#endif } static int tsdbInitHelperFile(SRWHelper *pHelper) { From 975f8c2c97f97652e1c4bb8f36ce640c3d5cdcdf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 20 Jul 2020 18:32:26 +0800 Subject: [PATCH 06/63] update file magic with other method --- src/tsdb/inc/tsdbMain.h | 4 +- src/tsdb/src/tsdbFile.c | 1 + src/tsdb/src/tsdbMain.c | 16 +++--- src/tsdb/src/tsdbRWHelper.c | 97 ++++++++++++++++++++----------------- src/util/inc/tkvstore.h | 11 +++-- src/util/src/tkvstore.c | 8 ++- 6 files changed, 80 insertions(+), 57 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 8ab89d72ed..156a408059 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -42,6 +42,7 @@ extern int tsdbDebugFlag; #define TSDB_MAX_TABLE_SCHEMAS 16 #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F +#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF // Definitions // ------------------ tsdbMeta.c @@ -230,7 +231,7 @@ typedef struct { typedef struct { int32_t delimiter; // For recovery usage - int32_t checksum; // TODO: decide if checksum logic in this file or make it one API + int32_t tid; uint64_t uid; SCompBlock blocks[]; } SCompInfo; @@ -308,6 +309,7 @@ typedef struct { #define TABLE_TID(t) (t)->tableId.tid #define TABLE_SUID(t) (t)->suid #define TABLE_LASTKEY(t) (t)->lastKey +#define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore) STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); void tsdbFreeMeta(STsdbMeta* pMeta); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 0a18d15773..767fbc8252 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -264,6 +264,7 @@ int tsdbCreateFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { } pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; if (tsdbUpdateFileHeader(pFile, 0) < 0) { tsdbCloseFile(pFile); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 82dbe5f498..7fe69021bf 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -212,6 +212,8 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ char *sdup = strdup(pRepo->rootDir); char *prefix = dirname(sdup); + int prefixLen = strlen(prefix); + tfree(sdup); if (name[0] == 0) { // get the file from index or after, but not larger than eindex int fid = (*index) / TSDB_FILE_TYPE_MAX; @@ -220,8 +222,8 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { fname = tsdbGetMetaFileName(pRepo->rootDir); *index = TSDB_META_FILE_INDEX; + magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); } else { - tfree(sdup); return 0; } } else { @@ -229,42 +231,42 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); if (pFGroup->fileId == fid) { fname = strdup(pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].fname); + magic = pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].info.magic; } else { if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < eindex) { fname = strdup(pFGroup->files[0].fname); *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; + magic = pFGroup->files[0].info.magic; } else { - tfree(sdup); return 0; } } } - strcpy(name, fname + strlen(prefix)); + strcpy(name, fname + prefixLen); } else { // get the named file at the specified index. If not there, return 0 if (*index == TSDB_META_FILE_INDEX) { // get meta file fname = tsdbGetMetaFileName(pRepo->rootDir); + magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); } else { int fid = (*index) / TSDB_FILE_TYPE_MAX; SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pFGroup == NULL) { // not found - tfree(sdup); return 0; } SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; fname = strdup(pFile->fname); + magic = pFile->info.magic; } } if (stat(fname, &fState) < 0) { - tfree(sdup); tfree(fname); return 0; } - tfree(sdup); *size = fState.st_size; - magic = *size; + // magic = *size; tfree(fname); return magic; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 22182694de..0b8d1fec49 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -100,6 +100,7 @@ void tsdbResetHelper(SRWHelper *pHelper) { int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { ASSERT(pHelper != NULL && pGroup != NULL); + SFile *pFile = NULL; // Clear the helper object tsdbResetHelper(pHelper); @@ -127,18 +128,27 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { #ifdef TSDB_IDX // Create and open .i file - if (tsdbOpenFile(helperNewIdxF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; - if (tsdbUpdateFileHeader(helperNewIdxF(pHelper), 0) < 0) return -1; + pFile = helperNewIdxF(pHelper); + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; + pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; + if (tsdbUpdateFileHeader(pFile, 0) < 0) return -1; #endif // Create and open .h - if (tsdbOpenFile(helperNewHeadF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; - if (tsdbUpdateFileHeader(helperNewHeadF(pHelper), 0) < 0) return -1; + pFile = helperNewHeadF(pHelper); + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; + pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; + if (tsdbUpdateFileHeader(pFile, 0) < 0) return -1; // Create and open .l file if should if (tsdbShouldCreateNewLast(pHelper)) { - if (tsdbOpenFile(helperNewLastF(pHelper), O_WRONLY | O_CREAT) < 0) goto _err; - if (tsdbUpdateFileHeader(helperNewLastF(pHelper), 0) < 0) return -1; + pFile = helperNewLastF(pHelper); + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) goto _err; + pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; + if (tsdbUpdateFileHeader(pFile, 0) < 0) return -1; } } else { if (tsdbOpenFile(helperDataF(pHelper), O_RDONLY) < 0) goto _err; @@ -334,7 +344,15 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); ASSERT(pCompBlock->last); + if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; + ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && + pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); + if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0) + return -1; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; + +#if 0 if (pCompBlock->numOfSubBlocks > 1) { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && @@ -365,6 +383,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { return -1; } } +#endif pHelper->hasOldLastBlock = false; } @@ -379,50 +398,34 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { if (pIdx->len > 0) { if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - pIdx->offset = offset; - pIdx->uid = pHelper->tableInfo.uid; - pIdx->tid = pHelper->tableInfo.tid; - ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - - if (tsendfile(pFile->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { - tsdbError("vgId:%d failed to send %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - helperHeadF(pHelper)->fname, pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; } else { pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER; pHelper->pCompInfo->uid = pHelper->tableInfo.uid; - pHelper->pCompInfo->checksum = 0; + pHelper->pCompInfo->tid = pHelper->tableInfo.tid; ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) && (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); - offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pIdx->offset = offset; - pIdx->uid = pHelper->tableInfo.uid; - pIdx->tid = pHelper->tableInfo.tid; - ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); + } - if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + pFile->info.magic = taosCalcChecksum( + pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pCompInfo, pIdx->len - sizeof(TSCKSUM)), sizeof(TSCKSUM)); + offset = lseek(pFile->fd, 0, SEEK_END); + if (offset < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pIdx->offset = offset; + pIdx->uid = pHelper->tableInfo.uid; + pIdx->tid = pHelper->tableInfo.tid; + ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); + + if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, + pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } #ifdef TSDB_IDX @@ -463,6 +466,8 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { } } taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len); + pFile->info.magic = taosCalcChecksum( + pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pWIdx, pFile->info.len - sizeof(TSCKSUM)), sizeof(TSCKSUM)); offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) { @@ -594,7 +599,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { return -1; } - ASSERT(pIdx->uid == pHelper->pCompInfo->uid); + ASSERT(pIdx->uid == pHelper->pCompInfo->uid && pIdx->tid == pHelper->pCompInfo->tid); } helperSetState(pHelper, TSDB_HELPER_INFO_LOAD); @@ -813,6 +818,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa ASSERT(flen > 0); flen += sizeof(TSCKSUM); taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); + pFile->info.magic = + taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)), sizeof(TSCKSUM)); if (ncol != 0) { pCompCol->offset = toffset; @@ -831,6 +838,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa pCompData->numOfCols = nColsNotAllNull; taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize); + pFile->info.magic = taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(pCompData, tsize - sizeof(TSCKSUM)), + sizeof(TSCKSUM)); // Write the whole block to file if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) { diff --git a/src/util/inc/tkvstore.h b/src/util/inc/tkvstore.h index 346e567c41..6d67607e24 100644 --- a/src/util/inc/tkvstore.h +++ b/src/util/inc/tkvstore.h @@ -25,10 +25,11 @@ typedef int (*iterFunc)(void *, void *cont, int contLen); typedef void (*afterFunc)(void *); typedef struct { - int64_t size; // including 512 bytes of header size - int64_t tombSize; - int64_t nRecords; - int64_t nDels; + int64_t size; // including 512 bytes of header size + int64_t tombSize; + int64_t nRecords; + int64_t nDels; + uint32_t magic; } SStoreInfo; typedef struct { @@ -45,6 +46,8 @@ typedef struct { SStoreInfo info; } SKVStore; +#define KVSTORE_MAGIC(s) (s)->info.magic + int tdCreateKVStore(char *fname); int tdDestroyKVStore(char *fname); SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH); diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index ab2aa738c6..a2022569bf 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -34,6 +34,7 @@ #define TD_KVSTORE_MAINOR_VERSION 0 #define TD_KVSTORE_SNAP_SUFFIX ".snap" #define TD_KVSTORE_NEW_SUFFIX ".new" +#define TD_KVSTORE_INIT_MAGIC 0xFFFFFFFF typedef struct { uint64_t uid; @@ -251,6 +252,8 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe return -1; } + pStore->info.magic = + taosCalcChecksum(pStore->info.magic, (uint8_t *)POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)), sizeof(TSCKSUM)); pStore->info.size += (sizeof(SKVRecord) + contLen); SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid)); if (pRecord != NULL) { // just to insert @@ -288,6 +291,7 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { return -1; } + pStore->info.magic = taosCalcChecksum(pStore->info.magic, (uint8_t *)buf, POINTER_DISTANCE(pBuf, buf)); pStore->info.size += POINTER_DISTANCE(pBuf, buf); pStore->info.nDels++; pStore->info.nRecords--; @@ -371,7 +375,7 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) { } static int tdInitKVStoreHeader(int fd, char *fname) { - SStoreInfo info = {TD_KVSTORE_HEADER_SIZE, 0, 0, 0}; + SStoreInfo info = {TD_KVSTORE_HEADER_SIZE, 0, 0, 0, TD_KVSTORE_INIT_MAGIC}; return tdUpdateKVStoreHeader(fd, fname, &info); } @@ -382,6 +386,7 @@ static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo) { tlen += taosEncodeVariantI64(buf, pInfo->tombSize); tlen += taosEncodeVariantI64(buf, pInfo->nRecords); tlen += taosEncodeVariantI64(buf, pInfo->nDels); + tlen += taosEncodeFixedU32(buf, pInfo->magic); return tlen; } @@ -391,6 +396,7 @@ static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) { buf = taosDecodeVariantI64(buf, &(pInfo->tombSize)); buf = taosDecodeVariantI64(buf, &(pInfo->nRecords)); buf = taosDecodeVariantI64(buf, &(pInfo->nDels)); + buf = taosDecodeFixedU32(buf, &(pInfo->magic)); return buf; } From 76e36a8d4e2a1d8c4a594c04ba64a31d9add6786 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 21 Jul 2020 03:05:47 +0000 Subject: [PATCH 07/63] fix defect found in coverity scan --- src/mnode/src/mnodeDb.c | 18 +++++++++++++----- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeVgroup.c | 17 ++++++++++++----- src/plugins/http/src/gcJson.c | 4 ++++ src/plugins/http/src/httpContext.c | 2 +- src/plugins/http/src/restJson.c | 4 ++++ src/plugins/mqtt/src/mqttSystem.c | 2 +- 7 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index fb97d6f380..a159e98ed5 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -67,8 +67,11 @@ static int32_t mnodeDbActionInsert(SSdbOper *pOper) { SAcctObj *pAcct = mnodeGetAcct(pDb->acct); pthread_mutex_init(&pDb->mutex, NULL); + pthread_mutex_lock(&pDb->mutex); pDb->vgListSize = VG_LIST_SIZE; pDb->vgList = calloc(pDb->vgListSize, sizeof(SVgObj *)); + pthread_mutex_unlock(&pDb->mutex); + pDb->numOfVgroups = 0; pDb->numOfTables = 0; pDb->numOfSuperTables = 0; @@ -395,8 +398,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { - mnodeDestroyDb(pDb); mLInfo("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code)); + mnodeDestroyDb(pDb); return code; } else { return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -605,7 +608,9 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn static char *mnodeGetDbStr(char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); - return ++pos; + if (pos != NULL) ++pos; + + return pos; } static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn) { @@ -622,10 +627,13 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void cols = 0; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; char* name = mnodeGetDbStr(pDb->name); - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]); + if (name != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]); + } else { + STR_TO_VARSTR(pWrite, "NULL"); + } cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 42ded7ed06..dee82f53be 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -453,7 +453,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { keySize = strlen((char *)key); } - taosHashPut(pTable->iHandle, key, keySize, &pOper->pObj, sizeof(void **)); + taosHashPut(pTable->iHandle, key, keySize, &pOper->pObj, sizeof(int64_t)); sdbIncRef(pTable, pOper->pObj); atomic_add_fetch_32(&pTable->numOfRows, 1); diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 1de591df7c..46255a6c8f 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -434,15 +434,22 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi } if (pDb->numOfVgroups < maxVgroupsPerDb) { - mDebug("app:%p:%p, db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg->rpcMsg.ahandle, pMsg, - pDb->name, pDb->numOfVgroups, maxVgroupsPerDb); + mDebug("app:%p:%p, db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg->rpcMsg.ahandle, + pMsg, pDb->name, pDb->numOfVgroups, maxVgroupsPerDb); pthread_mutex_unlock(&pDb->mutex); int32_t code = mnodeCreateVgroup(pMsg); - if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return code; + if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + return code; + } else { + pthread_mutex_lock(&pDb->mutex); + } } SVgObj *pVgroup = pDb->vgList[0]; - if (pVgroup == NULL) return TSDB_CODE_MND_NO_ENOUGH_DNODES; + if (pVgroup == NULL) { + pthread_mutex_unlock(&pDb->mutex); + return TSDB_CODE_MND_NO_ENOUGH_DNODES; + } int32_t code = mnodeAllocVgroupIdPool(pVgroup); if (code != TSDB_CODE_SUCCESS) { @@ -483,7 +490,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { } else { pVgroup->status = TAOS_VG_STATUS_READY; SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; - sdbUpdateRow(&desc); + (void)sdbUpdateRow(&desc); } mInfo("app:%p:%p, vgId:%d, is created in mnode, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, diff --git a/src/plugins/http/src/gcJson.c b/src/plugins/http/src/gcJson.c index 544a11b5fc..94d53db6ef 100644 --- a/src/plugins/http/src/gcJson.c +++ b/src/plugins/http/src/gcJson.c @@ -121,6 +121,10 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, for (int k = 0; k < numOfRows; ++k) { TAOS_ROW row = taos_fetch_row(result); + if (row == NULL) { + cmd->numOfRows--; + continue; + } int32_t* length = taos_fetch_lengths(result); // for group by diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index cefcca7821..225977abae 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -108,7 +108,7 @@ HttpContext *httpCreateContext(int32_t fd) { pContext->lastAccessTime = taosGetTimestampSec(); pContext->state = HTTP_CONTEXT_STATE_READY; - HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &pContext, sizeof(void *), &pContext, sizeof(void *), 3); + HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &pContext, sizeof(int64_t), &pContext, sizeof(int64_t), 3); pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); diff --git a/src/plugins/http/src/restJson.c b/src/plugins/http/src/restJson.c index 53b0248149..7a73f6559f 100644 --- a/src/plugins/http/src/restJson.c +++ b/src/plugins/http/src/restJson.c @@ -94,6 +94,10 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, for (int k = 0; k < numOfRows; ++k) { TAOS_ROW row = taos_fetch_row(result); + if (row == NULL) { + cmd->numOfRows--; + continue; + } int32_t* length = taos_fetch_lengths(result); // data row array begin diff --git a/src/plugins/mqtt/src/mqttSystem.c b/src/plugins/mqtt/src/mqttSystem.c index 2687106124..0259ea23eb 100644 --- a/src/plugins/mqtt/src/mqttSystem.c +++ b/src/plugins/mqtt/src/mqttSystem.c @@ -64,7 +64,7 @@ int32_t mqttInitSystem() { } char* _begin_hostname = strstr(url, recntStatus.hostname); - if (strstr(_begin_hostname, ":") != NULL) { + if (_begin_hostname != NULL && strstr(_begin_hostname, ":") != NULL) { recntStatus.port = strbetween(_begin_hostname, ":", "/"); } else { recntStatus.port = strbetween("'1883'", "'", "'"); From ce0ffb408090d3ae64398e8d9fb77e1f9e12e33f Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Tue, 21 Jul 2020 03:36:34 +0000 Subject: [PATCH 08/63] [TD-970] fix taosdemo bugs --- src/kit/taosdemo/taosdemo.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 305302b71a..79cfbe2a37 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -361,7 +361,7 @@ int main(int argc, char *argv[]) { arguments.num_of_DPT = 100000; arguments.num_of_RPR = 1000; arguments.use_metric = true; - arguments.insert_only = true; + arguments.insert_only = false; // end change argp_parse(&argp, argc, argv, 0, 0, &arguments); @@ -954,13 +954,13 @@ void *readMetric(void *sarg) { for (int i = 1; i <= m; i++) { if (i == 1) { - sprintf(tempS, "index = %d", i); + sprintf(tempS, "areaid = %d", i); } else { - sprintf(tempS, " or index = %d ", i); + sprintf(tempS, " or areaid = %d ", i); } strcat(condition, tempS); - sprintf(command, "select %s from m1 where %s", aggreFunc[j], condition); + sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); printf("Where condition: %s\n", condition); fprintf(fp, "%s\n", command); From a0ca29dcfa3635079e9f646f9b7cfd7d22c3a7f0 Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Tue, 21 Jul 2020 04:02:53 +0000 Subject: [PATCH 09/63] [TD-971] change create db sql --- tests/examples/python/read_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/python/read_example.py b/tests/examples/python/read_example.py index a96115dfb2..73c4b95214 100644 --- a/tests/examples/python/read_example.py +++ b/tests/examples/python/read_example.py @@ -29,7 +29,7 @@ if __name__ == '__main__': # Create a database named db try: - c1.execute('create database db') + c1.execute('create database if not exists db ') except Exception as err: conn.close() raise(err) From 88ba2f49359c6b5fbfed26e942d6d8b1944584fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 21 Jul 2020 13:43:46 +0800 Subject: [PATCH 10/63] add missed compile option --- src/tsdb/inc/tsdbMain.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index ba5711a81c..762d2253e2 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -263,7 +263,9 @@ typedef struct { TSKEY minKey; TSKEY maxKey; SFileGroup fGroup; - SFile nIdxF; +#ifdef TSDB_IDX + SFile nIdxF; +#endif SFile nHeadF; SFile nLastF; } SHelperFile; From 36f4948b2598b8d5b049d853feca577b73e21e5b Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 21 Jul 2020 05:45:44 +0000 Subject: [PATCH 11/63] set link secured except code is NOT_READY --- src/rpc/src/rpcMain.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 2325d12d92..4790c22668 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -446,7 +446,10 @@ void rpcSendResponse(const SRpcMsg *pRsp) { // set the idle timer to monitor the activity taosTmrReset(rpcProcessIdleTimer, pRpc->idleTime, pConn, pRpc->tmrCtrl, &pConn->pIdleTimer); rpcSendMsgToPeer(pConn, msg, msgLen); - pConn->secured = 1; // connection shall be secured + + // if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured + if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) + pConn->secured = 1; // connection shall be secured if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); pConn->pReqMsg = NULL; From 08fe5fbfa66c3869223c3adab63ae0d1de92492f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 21 Jul 2020 13:55:31 +0800 Subject: [PATCH 12/63] increase crash_gen step to 1000. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4d7a809e29..f2cd83343e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,7 +63,7 @@ matrix: pkill -TERM -x taosd fuser -k -n tcp 6030 sleep 1 - ./crash_gen.sh -a -p -t 4 -s 25|| travis_terminate $? + ./crash_gen.sh -a -p -t 4 -s 1000|| travis_terminate $? sleep 1 cd ${TRAVIS_BUILD_DIR}/tests/pytest From fac8cf100bfd774dd30c5dbd3c7afcf25009668a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 21 Jul 2020 14:17:05 +0800 Subject: [PATCH 13/63] refactor and debug --- src/tsdb/src/tsdbRWHelper.c | 44 ++++++++++++++----------------------- src/tsdb/src/tsdbRead.c | 24 +++++++++++++++----- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 0b8d1fec49..cec3b0d36b 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -60,7 +60,6 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter int *blkIdx); static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows); -static int tsdbCompareTidIdx(const void *key1, const void *key2); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -276,32 +275,31 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { tdInitDataCols(pHelper->pDataCols[0], pSchema); tdInitDataCols(pHelper->pDataCols[1], pSchema); - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (pHelper->idxH.numOfIdx > 0) { + if (pHelper->idxH.numOfIdx > 0) { + while (true) { if (pHelper->idxH.curIdx >= pHelper->idxH.numOfIdx) { memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - } else { - SCompIdx *pIdx = &(pHelper->idxH.pIdxArray[pHelper->idxH.curIdx]); - if (pIdx->tid == TABLE_TID(pTable)) { + break; + } + + SCompIdx *pIdx = &(pHelper->idxH.pIdxArray[pHelper->idxH.curIdx]); + if (pIdx->tid == TABLE_TID(pTable)) { + if (pIdx->uid == TABLE_UID(pTable)) { pHelper->curCompIdx = *pIdx; - pHelper->idxH.curIdx++; } else { - ASSERT(pIdx->tid > TABLE_TID(pTable)); memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); } + pHelper->idxH.curIdx++; + break; + } else if (pIdx->tid > TABLE_TID(pTable)) { + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + break; + } else { + pHelper->idxH.curIdx++; } - } else { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); } } else { - // TODO: make it more efficient - void *ptr = bsearch(&TABLE_TID(pTable), (void *)pHelper->idxH.pIdxArray, pHelper->idxH.numOfIdx, sizeof(SCompIdx), - tsdbCompareTidIdx); - if (ptr == NULL) { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - } else { - pHelper->curCompIdx = *(SCompIdx *)ptr; - } + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); } helperSetState(pHelper, TSDB_HELPER_TABLE_SET); @@ -1681,14 +1679,4 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, pCompBlock, isLast, true) < 0) return -1; return 0; -} - -static int tsdbCompareTidIdx(const void *key1, const void *key2) { - if (*(int32_t *)key1 > ((SCompIdx *)key2)->tid) { - return 1; - } else if (*(int32_t *)key1 < ((SCompIdx *)key2)->tid) { - return -1; - } else { - return 0; - } } \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index b2a3887bed..b2204948d0 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -126,12 +126,13 @@ typedef struct STsdbQueryHandle { SIOCostSummary cost; } STsdbQueryHandle; -static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); +static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); +static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); +static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle); +static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, + STsdbQueryHandle* pQueryHandle); +static int tsdbCheckInfoCompar(const void* key1, const void* key2); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -236,7 +237,8 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab taosArrayPush(pQueryHandle->pTableCheckInfo, &info); } } - + + taosArraySort(pQueryHandle->pTableCheckInfo, tsdbCheckInfoCompar); pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); tsdbDebug("%p total numOfTable:%zu in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo); @@ -2431,3 +2433,13 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); } +static int tsdbCheckInfoCompar(const void* key1, const void* key2) { + if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + return -1; + } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + return 1; + } else { + ASSERT(false); + return 0; + } +} \ No newline at end of file From bfe15fe0927b975f1cdf0726ec7649336f73e9bf Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 21 Jul 2020 14:18:06 +0800 Subject: [PATCH 14/63] increase crash_gen step to 2000. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f2cd83343e..6a4acce451 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,7 +63,7 @@ matrix: pkill -TERM -x taosd fuser -k -n tcp 6030 sleep 1 - ./crash_gen.sh -a -p -t 4 -s 1000|| travis_terminate $? + ./crash_gen.sh -a -p -t 4 -s 2000|| travis_terminate $? sleep 1 cd ${TRAVIS_BUILD_DIR}/tests/pytest From 51797a45d5ecb05943935b025e5b3570c2439921 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 21 Jul 2020 15:39:58 +0800 Subject: [PATCH 15/63] fix invalid write --- src/tsdb/src/tsdbRWHelper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index cec3b0d36b..9d0efd0de2 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -530,7 +530,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { size_t tlen = tsizeof(pHelper->idxH.pIdxArray); pHelper->idxH.numOfIdx++; - if (tlen < pHelper->idxH.numOfIdx) { + if (tlen < pHelper->idxH.numOfIdx * sizoef(SCompIdx)) { pHelper->idxH.pIdxArray = (SCompIdx *)trealloc(pHelper->idxH.pIdxArray, (tlen == 0) ? 1024 : tlen * 2); if (pHelper->idxH.pIdxArray == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; From 8a3616898f1836d06b7330a3bb06d2661535fc21 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 21 Jul 2020 15:45:04 +0800 Subject: [PATCH 16/63] fix typo --- src/tsdb/src/tsdbRWHelper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 9d0efd0de2..f8792469cb 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -530,7 +530,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { size_t tlen = tsizeof(pHelper->idxH.pIdxArray); pHelper->idxH.numOfIdx++; - if (tlen < pHelper->idxH.numOfIdx * sizoef(SCompIdx)) { + if (tlen < pHelper->idxH.numOfIdx * sizeof(SCompIdx)) { pHelper->idxH.pIdxArray = (SCompIdx *)trealloc(pHelper->idxH.pIdxArray, (tlen == 0) ? 1024 : tlen * 2); if (pHelper->idxH.pIdxArray == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; From 43b2f41fbbcbf7ee8c1620354dd7ff3441ac3caa Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 21 Jul 2020 16:24:59 +0800 Subject: [PATCH 17/63] TD-819: increase test coverage --- tests/pytest/fulltest.sh | 20 +++ tests/pytest/functions/function_avg.py | 73 +++++++++ tests/pytest/functions/function_bottom.py | 98 ++++++++++++ tests/pytest/functions/function_count.py | 84 +++++++++++ tests/pytest/functions/function_diff.py | 103 +++++++++++++ tests/pytest/functions/function_first.py | 124 ++++++++++++++++ tests/pytest/functions/function_last.py | 124 ++++++++++++++++ tests/pytest/functions/function_last_row.py | 133 +++++++++++++++++ .../pytest/functions/function_leastsquares.py | 82 ++++++++++ tests/pytest/functions/function_max.py | 78 ++++++++++ tests/pytest/functions/function_min.py | 78 ++++++++++ tests/pytest/functions/function_operations.py | 86 +++++++++++ tests/pytest/functions/function_percentile.py | 140 ++++++++++++++++++ tests/pytest/functions/function_spread.py | 111 ++++++++++++++ tests/pytest/functions/function_stddev.py | 80 ++++++++++ tests/pytest/functions/function_sum.py | 70 +++++++++ tests/pytest/functions/function_top.py | 98 ++++++++++++ tests/pytest/functions/function_twa.py | 135 +++++++++++++++++ tests/pytest/query/filterOtherTypes.py | 24 ++- tests/pytest/regressiontest.sh | 21 +++ tests/pytest/util/sql.py | 20 ++- 21 files changed, 1768 insertions(+), 14 deletions(-) create mode 100644 tests/pytest/functions/function_avg.py create mode 100644 tests/pytest/functions/function_bottom.py create mode 100644 tests/pytest/functions/function_count.py create mode 100644 tests/pytest/functions/function_diff.py create mode 100644 tests/pytest/functions/function_first.py create mode 100644 tests/pytest/functions/function_last.py create mode 100644 tests/pytest/functions/function_last_row.py create mode 100644 tests/pytest/functions/function_leastsquares.py create mode 100644 tests/pytest/functions/function_max.py create mode 100644 tests/pytest/functions/function_min.py create mode 100644 tests/pytest/functions/function_operations.py create mode 100644 tests/pytest/functions/function_percentile.py create mode 100644 tests/pytest/functions/function_spread.py create mode 100644 tests/pytest/functions/function_stddev.py create mode 100644 tests/pytest/functions/function_sum.py create mode 100644 tests/pytest/functions/function_top.py create mode 100644 tests/pytest/functions/function_twa.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 9d1aef0dc5..904e712fa4 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -144,6 +144,7 @@ python3 ./test.py -f query/querySort.py python3 ./test.py -f query/queryJoin.py python3 ./test.py -f query/select_last_crash.py python3 ./test.py -f query/queryNullValueTest.py +python3 ./test.py -f query/queryInsertValue.py #stream python3 ./test.py -f stream/metric_1.py @@ -161,3 +162,22 @@ python3 ./test.py -f client/client.py # Misc python3 testCompress.py python3 testNoCompress.py + +# functions +python3 ./test.py -f functions/function_avg.py +python3 ./test.py -f functions/function_bottom.py +python3 ./test.py -f functions/function_count.py +python3 ./test.py -f functions/function_diff.py +python3 ./test.py -f functions/function_first.py +python3 ./test.py -f functions/function_last.py +python3 ./test.py -f functions/function_last_row.py +python3 ./test.py -f functions/function_leastsquares.py +python3 ./test.py -f functions/function_max.py +python3 ./test.py -f functions/function_min.py +python3 ./test.py -f functions/function_operations.py +python3 ./test.py -f functions/function_percentile.py +python3 ./test.py -f functions/function_spread.py +python3 ./test.py -f functions/function_stddev.py +python3 ./test.py -f functions/function_sum.py +python3 ./test.py -f functions/function_top.py +python3 ./test.py -f functions/function_twa.py \ No newline at end of file diff --git a/tests/pytest/functions/function_avg.py b/tests/pytest/functions/function_avg.py new file mode 100644 index 0000000000..9481550ba3 --- /dev/null +++ b/tests/pytest/functions/function_avg.py @@ -0,0 +1,73 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # average verifacation + tdSql.error("select avg(ts) from test") + tdSql.error("select avg(ts) from test1") + tdSql.error("select avg(col7) from test") + tdSql.error("select avg(col7) from test1") + tdSql.error("select avg(col8) from test") + tdSql.error("select avg(col8) from test1") + tdSql.error("select avg(col9) from test") + tdSql.error("select avg(col9) from test1") + + tdSql.query("select avg(col1) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col2) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col3) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col4) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col5) from test") + tdSql.checkData(0, 0, np.average(floatData)) + tdSql.query("select avg(col6) from test") + tdSql.checkData(0, 0, np.average(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py new file mode 100644 index 0000000000..cb008d522a --- /dev/null +++ b/tests/pytest/functions/function_bottom.py @@ -0,0 +1,98 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # bottom verifacation + tdSql.error("select bottom(ts, 10) from test") + tdSql.error("select bottom(col1, 0) from test") + tdSql.error("select bottom(col1, 101) from test") + tdSql.error("select bottom(col2, 0) from test") + tdSql.error("select bottom(col2, 101) from test") + tdSql.error("select bottom(col3, 0) from test") + tdSql.error("select bottom(col3, 101) from test") + tdSql.error("select bottom(col4, 0) from test") + tdSql.error("select bottom(col4, 101) from test") + tdSql.error("select bottom(col5, 0) from test") + tdSql.error("select bottom(col5, 101) from test") + tdSql.error("select bottom(col6, 0) from test") + tdSql.error("select bottom(col6, 101) from test") + tdSql.error("select bottom(col7, 10) from test") + tdSql.error("select bottom(col8, 10) from test") + tdSql.error("select bottom(col9, 10) from test") + + tdSql.query("select bottom(col1, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col2, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col3, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col4, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col5, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 0.1) + tdSql.checkData(1, 1, 1.1) + + tdSql.query("select bottom(col6, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 0.1) + tdSql.checkData(1, 1, 1.1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_count.py b/tests/pytest/functions/function_count.py new file mode 100644 index 0000000000..e5ef71bf5a --- /dev/null +++ b/tests/pytest/functions/function_count.py @@ -0,0 +1,84 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # Count verifacation + tdSql.query("select count(*) from test") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(ts) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col1) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col2) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col3) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col4) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col5) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col6) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col7) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col8) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col9) from test") + tdSql.checkData(0, 0, 10) + + tdSql.execute("alter table test add column col10 int") + tdSql.query("select count(col10) from test") + tdSql.checkRows(0) + + tdSql.execute("insert into test1 values(now, 1, 2, 3, 4, 1.1, 2.2, false, 'test', 'test' 1)") + tdSql.query("select count(col10) from test") + tdSql.checkData(0, 0, 1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_diff.py b/tests/pytest/functions/function_diff.py new file mode 100644 index 0000000000..62daa5fa82 --- /dev/null +++ b/tests/pytest/functions/function_diff.py @@ -0,0 +1,103 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) + tdSql.query("select diff(col1) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col2) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col3) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col4) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col5) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col6) from test1") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # diff verifacation + tdSql.error("select diff(ts) from test") + tdSql.error("select diff(ts) from test1") + tdSql.error("select diff(col1) from test") + tdSql.error("select diff(col2) from test") + tdSql.error("select diff(col3) from test") + tdSql.error("select diff(col4) from test") + tdSql.error("select diff(col5) from test") + tdSql.error("select diff(col6) from test") + tdSql.error("select diff(col7) from test") + tdSql.error("select diff(col7) from test1") + tdSql.error("select diff(col8) from test") + tdSql.error("select diff(col8) from test1") + tdSql.error("select diff(col9) from test") + tdSql.error("select diff(col9) from test1") + + tdSql.query("select diff(col1) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col2) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col3) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col4) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col5) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col6) from test1") + tdSql.checkRows(10) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_first.py b/tests/pytest/functions/function_first.py new file mode 100644 index 0000000000..27ce09b309 --- /dev/null +++ b/tests/pytest/functions/function_first.py @@ -0,0 +1,124 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + + # first verifacation + tdSql.query("select first(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + tdSql.query("select first(col1) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col2) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col3) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col4) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col5) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col6) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col7) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col8) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col9) from test1") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + tdSql.query("select first(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + tdSql.query("select first(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0.1) + + tdSql.query("select first(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0.1) + + tdSql.query("select first(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, False) + + tdSql.query("select first(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata1') + + tdSql.query("select first(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据1') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_last.py b/tests/pytest/functions/function_last.py new file mode 100644 index 0000000000..6412b7ce6a --- /dev/null +++ b/tests/pytest/functions/function_last.py @@ -0,0 +1,124 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + + # first verifacation + tdSql.query("select last(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + tdSql.query("select last(col1) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col2) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col3) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col4) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col5) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col6) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col7) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col8) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col9) from test1") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + tdSql.query("select last(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + + tdSql.query("select last(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + + tdSql.query("select last(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + + tdSql.query("select last(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据10') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_last_row.py b/tests/pytest/functions/function_last_row.py new file mode 100644 index 0000000000..19db865a3d --- /dev/null +++ b/tests/pytest/functions/function_last_row.py @@ -0,0 +1,133 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + + # first verifacation + tdSql.query("select last_row(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + tdSql.query("select last_row(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + tdSql.query("select last_row(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + + tdSql.query("select last_row(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last_row(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last_row(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + + tdSql.query("select last_row(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + + tdSql.query("select last_row(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据10') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_leastsquares.py b/tests/pytest/functions/function_leastsquares.py new file mode 100644 index 0000000000..66187926f5 --- /dev/null +++ b/tests/pytest/functions/function_leastsquares.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # leastsquares verifacation + tdSql.error("select leastsquares(ts, 1, 1) from test1") + tdSql.error("select leastsquares(col1, 1, 1) from test") + tdSql.error("select leastsquares(col2, 1, 1) from test") + tdSql.error("select leastsquares(col3, 1, 1) from test") + tdSql.error("select leastsquares(col4, 1, 1) from test") + tdSql.error("select leastsquares(col5, 1, 1) from test") + tdSql.error("select leastsquares(col6, 1, 1) from test") + tdSql.error("select leastsquares(col7, 1, 1) from test1") + tdSql.error("select leastsquares(col8, 1, 1) from test1") + tdSql.error("select leastsquares(col9, 1, 1) from test1") + + tdSql.query("select leastsquares(col1, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col2, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col3, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col4, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col5, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}') + + tdSql.query("select leastsquares(col6, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}') + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_max.py b/tests/pytest/functions/function_max.py new file mode 100644 index 0000000000..02364e0fde --- /dev/null +++ b/tests/pytest/functions/function_max.py @@ -0,0 +1,78 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # min verifacation + tdSql.error("select max(ts) from test") + tdSql.error("select max(ts) from test1") + tdSql.error("select max(col7) from test") + tdSql.error("select max(col7) from test1") + tdSql.error("select max(col8) from test") + tdSql.error("select max(col8) from test1") + tdSql.error("select max(col9) from test") + tdSql.error("select max(col9) from test1") + + tdSql.query("select max(col1) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col2) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col3) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col4) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col5) from test1") + tdSql.checkData(0, 0, np.max(floatData)) + + tdSql.query("select max(col6) from test1") + tdSql.checkData(0, 0, np.max(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_min.py b/tests/pytest/functions/function_min.py new file mode 100644 index 0000000000..bc180bc224 --- /dev/null +++ b/tests/pytest/functions/function_min.py @@ -0,0 +1,78 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # min verifacation + tdSql.error("select min(ts) from test") + tdSql.error("select min(ts) from test1") + tdSql.error("select min(col7) from test") + tdSql.error("select min(col7) from test1") + tdSql.error("select min(col8) from test") + tdSql.error("select min(col8) from test1") + tdSql.error("select min(col9) from test") + tdSql.error("select min(col9) from test1") + + tdSql.query("select min(col1) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col2) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col3) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col4) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col5) from test1") + tdSql.checkData(0, 0, np.min(floatData)) + + tdSql.query("select min(col6) from test1") + tdSql.checkData(0, 0, np.min(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_operations.py b/tests/pytest/functions/function_operations.py new file mode 100644 index 0000000000..ec8da92c95 --- /dev/null +++ b/tests/pytest/functions/function_operations.py @@ -0,0 +1,86 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # min verifacation + tdSql.error("select ts + col1 from test") + tdSql.error("select ts + col1 from test1") + tdSql.error("select col1 + col7 from test") + tdSql.error("select col1 + col7 from test1") + tdSql.error("select col1 + col8 from test") + tdSql.error("select col1 + col8 from test1") + tdSql.error("select col1 + col9 from test") + tdSql.error("select col1 + col9 from test1") + + tdSql.query("select col1 + col2 from test1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 2.0) + + tdSql.query("select col1 + col2 * col3 from test1") + tdSql.checkRows(10) + tdSql.checkData(1, 0, 6.0) + + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 3.2) + + tdSql.execute("insert into test1(ts, col1) values(%d, 11)" % (self.ts + 11)) + tdSql.query("select col1 + col2 from test1") + tdSql.checkRows(11) + tdSql.checkData(10, 0, None) + + tdSql.query("select col1 + col2 * col3 from test1") + tdSql.checkRows(11) + tdSql.checkData(10, 0, None) + + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") + tdSql.checkRows(11) + tdSql.checkData(10, 0, None) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_percentile.py b/tests/pytest/functions/function_percentile.py new file mode 100644 index 0000000000..aaeb94372e --- /dev/null +++ b/tests/pytest/functions/function_percentile.py @@ -0,0 +1,140 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20))''') + for i in range(self.rowNum): + tdSql.execute("insert into test values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # percentile verifacation + tdSql.error("select percentile(ts 20) from test") + tdSql.error("select apercentile(ts 20) from test") + tdSql.error("select percentile(col7 20) from test") + tdSql.error("select apercentile(col7 20) from test") + tdSql.error("select percentile(col8 20) from test") + tdSql.error("select apercentile(col8 20) from test") + tdSql.error("select percentile(col9 20) from test") + tdSql.error("select apercentile(col9 20) from test") + + tdSql.query("select percentile(col1, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col1, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col1, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col1, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col1, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col1, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col2, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col2, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col2, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col2, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col2, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col2, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col3, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col3, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col3, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col3, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col3, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col3, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col4, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col4, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col4, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col4, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col4, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col4, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col5, 0) from test") + print("query result: %s" % tdSql.getData(0, 0)) + print("array result: %s" % np.percentile(floatData, 0)) + tdSql.query("select apercentile(col5, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col5, 50) from test") + print("query result: %s" % tdSql.getData(0, 0)) + print("array result: %s" % np.percentile(floatData, 50)) + tdSql.query("select apercentile(col5, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col5, 100) from test") + print("query result: %s" % tdSql.getData(0, 0)) + print("array result: %s" % np.percentile(floatData, 100)) + tdSql.query("select apercentile(col5, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col6, 0) from test") + tdSql.checkData(0, 0, np.percentile(floatData, 0)) + tdSql.query("select apercentile(col6, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col6, 50) from test") + tdSql.checkData(0, 0, np.percentile(floatData, 50)) + tdSql.query("select apercentile(col6, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col6, 100) from test") + tdSql.checkData(0, 0, np.percentile(floatData, 100)) + tdSql.query("select apercentile(col6, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_spread.py b/tests/pytest/functions/function_spread.py new file mode 100644 index 0000000000..62b604a279 --- /dev/null +++ b/tests/pytest/functions/function_spread.py @@ -0,0 +1,111 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) + + # spread verifacation + tdSql.query("select spread(ts) from test1") + tdSql.checkRows(1) + + tdSql.query("select spread(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + tdSql.error("select spread(col7) from test") + tdSql.error("select spread(col7) from test1") + tdSql.error("select spread(col8) from test") + tdSql.error("select spread(col8) from test1") + tdSql.error("select spread(col9) from test") + tdSql.error("select spread(col9) from test1") + + tdSql.query("select spread(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select spread(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_stddev.py b/tests/pytest/functions/function_stddev.py new file mode 100644 index 0000000000..23df415aa3 --- /dev/null +++ b/tests/pytest/functions/function_stddev.py @@ -0,0 +1,80 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # stddev verifacation + tdSql.error("select stddev(ts) from test1") + tdSql.error("select stddev(col1) from test") + tdSql.error("select stddev(col2) from test") + tdSql.error("select stddev(col3) from test") + tdSql.error("select stddev(col4) from test") + tdSql.error("select stddev(col5) from test") + tdSql.error("select stddev(col6) from test") + tdSql.error("select stddev(col7) from test1") + tdSql.error("select stddev(col8) from test1") + tdSql.error("select stddev(col9) from test1") + + tdSql.query("select stddev(col1) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col2) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col3) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col4) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col5) from test1") + tdSql.checkData(0, 0, np.std(floatData)) + + tdSql.query("select stddev(col6) from test1") + tdSql.checkData(0, 0, np.std(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_sum.py b/tests/pytest/functions/function_sum.py new file mode 100644 index 0000000000..8e4e289c7e --- /dev/null +++ b/tests/pytest/functions/function_sum.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # sum verifacation + tdSql.error("select sum(ts) from test") + tdSql.error("select sum(col7) from test") + tdSql.error("select sum(col8) from test") + tdSql.error("select sum(col9) from test") + + tdSql.query("select sum(col1) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col2) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col3) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col4) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col5) from test") + print("query result for folat sum function: %f" % tdSql.getData(0, 0)) + print("calculation result for numpy sum function: %f" % np.sum(floatData)) + tdSql.query("select sum(col6) from test") + tdSql.checkData(0, 0, np.sum(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_top.py b/tests/pytest/functions/function_top.py new file mode 100644 index 0000000000..e24ff1cc53 --- /dev/null +++ b/tests/pytest/functions/function_top.py @@ -0,0 +1,98 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # top verifacation + tdSql.error("select top(ts, 10) from test") + tdSql.error("select top(col1, 0) from test") + tdSql.error("select top(col1, 101) from test") + tdSql.error("select top(col2, 0) from test") + tdSql.error("select top(col2, 101) from test") + tdSql.error("select top(col3, 0) from test") + tdSql.error("select top(col3, 101) from test") + tdSql.error("select top(col4, 0) from test") + tdSql.error("select top(col4, 101) from test") + tdSql.error("select top(col5, 0) from test") + tdSql.error("select top(col5, 101) from test") + tdSql.error("select top(col6, 0) from test") + tdSql.error("select top(col6, 101) from test") + tdSql.error("select top(col7, 10) from test") + tdSql.error("select top(col8, 10) from test") + tdSql.error("select top(col9, 10) from test") + + tdSql.query("select top(col1, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col2, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col3, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col4, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col5, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 8.1) + tdSql.checkData(1, 1, 9.1) + + tdSql.query("select top(col6, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 8.1) + tdSql.checkData(1, 1, 9.1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_twa.py b/tests/pytest/functions/function_twa.py new file mode 100644 index 0000000000..1ce4c99b60 --- /dev/null +++ b/tests/pytest/functions/function_twa.py @@ -0,0 +1,135 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # twa verifacation + tdSql.error("select twa(ts) from test") + tdSql.error("select twa(ts) from test1") + + tdSql.error("select twa(col1) from test") + tdSql.error("select twa(col1) from test1") + + tdSql.error("select twa(col2) from test") + tdSql.error("select twa(col2) from test1") + + tdSql.error("select twa(col3) from test") + tdSql.error("select twa(col3) from test1") + + tdSql.error("select twa(col4) from test") + tdSql.error("select twa(col4) from test1") + + tdSql.error("select twa(col5) from test") + tdSql.error("select twa(col5) from test1") + + tdSql.error("select twa(col6) from test") + tdSql.error("select twa(col6) from test1") + + tdSql.error("select twa(col7) from test") + tdSql.error("select twa(col7) from test1") + + tdSql.error("select twa(col8) from test") + tdSql.error("select twa(col8) from test1") + + tdSql.error("select twa(col9) from test") + tdSql.error("select twa(col9) from test1") + + tdSql.error("select twa(col1) from test where ts > %d" % self.ts) + tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col2) from test where ts > %d" % self.ts) + tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col3) from test where ts > %d" % self.ts) + tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col4) from test where ts > %d" % self.ts) + tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col5) from test where ts > %d" % self.ts) + tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col6) from test where ts > %d" % self.ts) + tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.query("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col1) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col2) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col3) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col6) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/filterOtherTypes.py b/tests/pytest/query/filterOtherTypes.py index 1db5497604..5033ffdb48 100644 --- a/tests/pytest/query/filterOtherTypes.py +++ b/tests/pytest/query/filterOtherTypes.py @@ -231,10 +231,12 @@ class TDTestCase: tdSql.error("select * from st where tagcol1 like '____'") # > for nchar type on tag - tdSql.error("select * from st where tagcol2 > 'table'") + tdSql.query("select * from st where tagcol2 > 'table1'") + tdSql.checkRows(5) # >= for nchar type on tag - tdSql.error("select * from st where tagcol2 >= 'table'") + tdSql.query("select * from st where tagcol2 >= 'table1'") + tdSql.checkRows(10) # = for nchar type on tag tdSql.query("select * from st where tagcol2 = 'table1'") @@ -249,10 +251,12 @@ class TDTestCase: tdSql.checkRows(10) # > for nchar type on tag - tdSql.error("select * from st where tagcol2 < 'table'") + tdSql.query("select * from st where tagcol2 < 'table'") + tdSql.checkRows(0) # >= for nchar type on tag - tdSql.error("select * from st where tagcol2 <= 'table'") + tdSql.query("select * from st where tagcol2 <= 'table'") + tdSql.checkRows(0) # % for nchar type on tag case 1 tdSql.query("select * from st where tagcol2 like '%'") @@ -291,10 +295,12 @@ class TDTestCase: tdSql.checkRows(10) # > for binary type on tag - tdSql.error("select * from st where tagcol3 > '表'") + tdSql.query("select * from st where tagcol3 > '表'") + tdSql.checkRows(10) # >= for binary type on tag - tdSql.error("select * from st where tagcol3 >= '表'") + tdSql.query("select * from st where tagcol3 >= '表'") + tdSql.checkRows(10) # = for binary type on tag tdSql.query("select * from st where tagcol3 = '水表'") @@ -309,10 +315,12 @@ class TDTestCase: tdSql.checkRows(5) # > for binary type on tag - tdSql.error("select * from st where tagcol3 < '水表'") + tdSql.query("select * from st where tagcol3 < '水表'") + tdSql.checkRows(0) # >= for binary type on tag - tdSql.error("select * from st where tagcol3 <= '水表'") + tdSql.query("select * from st where tagcol3 <= '水表'") + tdSql.checkRows(5) # % for binary type on tag case 1 tdSql.query("select * from st where tagcol3 like '%'") diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh index 24cd93f0fc..61ec491f5d 100755 --- a/tests/pytest/regressiontest.sh +++ b/tests/pytest/regressiontest.sh @@ -141,6 +141,7 @@ python3 ./test.py -f query/filterCombo.py python3 ./test.py -f query/queryNormal.py python3 ./test.py -f query/select_last_crash.py python3 ./test.py -f query/queryNullValueTest.py +python3 ./test.py -f query/queryInsertValue.py #stream python3 ./test.py -f stream/stream1.py @@ -155,3 +156,23 @@ python3 ./test.py -f client/client.py # Misc python3 testCompress.py python3 testNoCompress.py + + +# functions +python3 ./test.py -f functions/function_avg.py +python3 ./test.py -f functions/function_bottom.py +python3 ./test.py -f functions/function_count.py +python3 ./test.py -f functions/function_diff.py +python3 ./test.py -f functions/function_first.py +python3 ./test.py -f functions/function_last.py +python3 ./test.py -f functions/function_last_row.py +python3 ./test.py -f functions/function_leastsquares.py +python3 ./test.py -f functions/function_max.py +python3 ./test.py -f functions/function_min.py +python3 ./test.py -f functions/function_operations.py +python3 ./test.py -f functions/function_percentile.py +python3 ./test.py -f functions/function_spread.py +python3 ./test.py -f functions/function_stddev.py +python3 ./test.py -f functions/function_sum.py +python3 ./test.py -f functions/function_top.py +python3 ./test.py -f functions/function_twa.py diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index ec39ab61b9..1e1d02959f 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -122,11 +122,16 @@ class TDSql: return self.cursor.istype(col, dataType) def checkData(self, row, col, data): - self.checkRowCol(row, col) - if self.queryResult[row][col] != data: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) - tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + self.checkRowCol(row, col) + if self.queryResult[row][col] != data: + if isinstance(data, float) and abs(self.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (self.sql, row, col, self.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) if data is None: tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % @@ -135,9 +140,12 @@ class TDSql: tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (self.sql, row, col, self.queryResult[row][col], data)) + elif isinstance(data, float): tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) - else: + else: tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % (self.sql, row, col, self.queryResult[row][col], data)) From 2867f3082942b3b27fa5e573e530ad66a8e1c2d8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 21 Jul 2020 17:29:31 +0800 Subject: [PATCH 18/63] fix sh/exec signal for coverage test. [TD-976] --- tests/script/fullGeneralSuite.sim | 4 ---- tests/script/sh/exec-no-random-fail.sh | 12 +++++++----- tests/script/sh/exec-random-fail.sh | 15 +++++++++------ tests/script/sh/exec.sh | 8 ++++---- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index 2f45ab077c..15cc2954e8 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -135,7 +135,6 @@ run general/parser/set_tag_vals.sim #unsupport run general/parser/repeatAlter.sim #unsupport run general/parser/slimit_alter_tags.sim #unsupport run general/parser/stream_on_sys.sim -run general/parser/stream.sim #unsupport run general/parser/repeatStream.sim run general/stable/disk.sim run general/stable/dnode3.sim @@ -212,12 +211,9 @@ run general/vector/table_mix.sim run general/vector/table_query.sim run general/vector/table_time.sim run general/stream/restart_stream.sim -run general/stream/stream_1.sim -run general/stream/stream_2.sim run general/stream/stream_3.sim run general/stream/stream_restart.sim run general/stream/table_1.sim -run general/stream/metrics_1.sim run general/stream/table_n.sim run general/stream/metrics_n.sim run general/stream/table_del.sim diff --git a/tests/script/sh/exec-no-random-fail.sh b/tests/script/sh/exec-no-random-fail.sh index 04a663bc5a..2bd0a64923 100755 --- a/tests/script/sh/exec-no-random-fail.sh +++ b/tests/script/sh/exec-no-random-fail.sh @@ -88,7 +88,9 @@ if [ "$EXEC_OPTON" = "start" ]; then echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR if [ "$SHELL_OPTION" = "true" ]; then - nohup valgrind --log-file=${LOG_DIR}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + TT=`date +%s` + mkdir ${LOG_DIR}/${TT} + nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & else nohup $EXE_DIR/taosd -c $CFG_DIR --random-file-fail-factor 0 > /dev/null 2>&1 & fi @@ -99,12 +101,12 @@ else PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` while [ -n "$PID" ] do - if [ "$SIGNAL" = "SIGINT" ]; then - echo try to kill by signal SIGINT - kill -SIGINT $PID - else + if [ "$SIGNAL" = "SIGKILL" ]; then echo try to kill by signal SIGKILL kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID fi sleep 1 PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` diff --git a/tests/script/sh/exec-random-fail.sh b/tests/script/sh/exec-random-fail.sh index a354021684..3761498859 100755 --- a/tests/script/sh/exec-random-fail.sh +++ b/tests/script/sh/exec-random-fail.sh @@ -88,9 +88,12 @@ if [ "$EXEC_OPTON" = "start" ]; then echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR if [ "$SHELL_OPTION" = "true" ]; then - nohup valgrind --log-file=${LOG_DIR}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + TT=`date +%s` + mkdir ${LOG_DIR}/${TT} + nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & else - nohup $EXE_DIR/taosd -c $CFG_DIR --alloc-random-fail --random-file-fail-factor 5 > /dev/null 2>&1 & + nohup $EXE_DIR/taosd -c $CFG_DIR --alloc-random-fail \ + --random-file-fail-factor 5 > /dev/null 2>&1 & fi else @@ -99,12 +102,12 @@ else PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` while [ -n "$PID" ] do - if [ "$SIGNAL" = "SIGINT" ]; then - echo try to kill by signal SIGINT - kill -SIGINT $PID - else + if [ "$SIGNAL" = "SIGKILL" ]; then echo try to kill by signal SIGKILL kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID fi sleep 1 PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 2f294075a1..1c84a6b10e 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -101,12 +101,12 @@ else PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` while [ -n "$PID" ] do - if [ "$SIGNAL" = "SIGINT" ]; then - echo try to kill by signal SIGINT - kill -SIGINT $PID - else + if [ "$SIGNAL" = "SIGKILL" ]; then echo try to kill by signal SIGKILL kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID fi sleep 1 PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` From 5e8b11415a8e3241668bc875638346027f25d1b2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 21 Jul 2020 18:08:46 +0800 Subject: [PATCH 19/63] need to restore magic number when kvstore open --- src/util/src/tkvstore.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index a2022569bf..0704285da5 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -141,6 +141,7 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err; pStore->info.size = TD_KVSTORE_HEADER_SIZE; + pStore->info.magic = info.magic; if (tdRestoreKVStore(pStore) < 0) goto _err; From 52fb827f0a5f42f33f952c313b8bcd02009ec928 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 21 Jul 2020 20:40:19 +0800 Subject: [PATCH 20/63] [TD-972] --- src/mnode/inc/mnodeDef.h | 32 +++++++++++++++----------------- src/mnode/src/mnodeSdb.c | 14 ++++++++++---- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 3154a441ca..a454f413f0 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -50,8 +50,8 @@ typedef struct SDnodeObj { int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode int8_t status; // set in balance function int8_t isMgmt; - int8_t reserve1[14]; - int8_t updateEnd[1]; + int8_t reserve1[11]; + int8_t updateEnd[4]; int32_t refCount; uint32_t moduleStatus; uint32_t lastReboot; // time stamp for last reboot @@ -68,8 +68,8 @@ typedef struct SMnodeObj { int32_t mnodeId; int8_t reserved0[4]; int64_t createdTime; - int8_t reserved1[7]; - int8_t updateEnd[1]; + int8_t reserved1[4]; + int8_t updateEnd[4]; int32_t refCount; int8_t role; int8_t reserved2[3]; @@ -90,8 +90,7 @@ typedef struct SSuperTableObj { int32_t tversion; int32_t numOfColumns; int32_t numOfTags; - int8_t reserved1[3]; - int8_t updateEnd[1]; + int8_t updateEnd[4]; int32_t refCount; int32_t numOfTables; SSchema * schema; @@ -111,8 +110,7 @@ typedef struct { int32_t sid; int32_t vgId; int32_t sqlLen; - int8_t updateEnd[1]; - int8_t reserved1[1]; + int8_t updateEnd[4]; int32_t refCount; char* sql; //used by normal table SSchema* schema; //used by normal table @@ -138,8 +136,8 @@ typedef struct SVgObj { int8_t status; int8_t reserved0[4]; SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; - int8_t reserved1[7]; - int8_t updateEnd[1]; + int8_t reserved1[4]; + int8_t updateEnd[4]; int32_t refCount; int32_t numOfTables; int64_t totalStorage; @@ -176,8 +174,8 @@ typedef struct SDbObj { int32_t cfgVersion; SDbCfg cfg; int8_t status; - int8_t reserved1[14]; - int8_t updateEnd[1]; + int8_t reserved1[11]; + int8_t updateEnd[4]; int32_t refCount; int32_t numOfVgroups; int32_t numOfTables; @@ -196,8 +194,8 @@ typedef struct SUserObj { int64_t createdTime; int8_t superAuth; int8_t writeAuth; - int8_t reserved[13]; - int8_t updateEnd[1]; + int8_t reserved[10]; + int8_t updateEnd[4]; int32_t refCount; struct SAcctObj * pAcct; } SUserObj; @@ -228,11 +226,11 @@ typedef struct SAcctObj { int64_t createdTime; int32_t acctId; int8_t status; - int8_t reserved0[10]; - int8_t updateEnd[1]; - SAcctInfo acctInfo; + int8_t reserved0[7]; + int8_t updateEnd[4]; int32_t refCount; int8_t reserved1[4]; + SAcctInfo acctInfo; pthread_mutex_t mutex; } SAcctObj; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index dee82f53be..26b0f60fcd 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -406,7 +406,7 @@ void sdbDecRef(void *handle, void *pObj) { int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); sdbTrace("def ref of table:%s record:%p:%s:%d", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount); - int8_t *updateEnd = pObj + pTable->refCountPos - 1; + int32_t *updateEnd = pObj + pTable->refCountPos - 4; if (refCount <= 0 && *updateEnd) { sdbTrace("table:%s, record:%p:%s:%d is destroyed", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount); SSdbOper oper = {.pObj = pObj}; @@ -472,6 +472,14 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { } static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { + int32_t *updateEnd = pOper->pObj + pTable->refCountPos - 4; + bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; + if (!set) { + sdbError("table:%s, failed to delete record:%s from hash, for it already removed", pTable->tableName, + sdbGetKeyStrFromObj(pTable, pOper->pObj)); + return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; + } + (*pTable->deleteFp)(pOper); void * key = sdbGetObjKey(pTable, pOper->pObj); @@ -486,8 +494,6 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { sdbDebug("table:%s, delete record:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg); - int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1; - *updateEnd = 1; sdbDecRef(pTable, pOper->pObj); return TSDB_CODE_SUCCESS; @@ -654,7 +660,7 @@ bool sdbCheckRowDeleted(void *pTableInput, void *pRow) { SSdbTable *pTable = pTableInput; if (pTable == NULL) return false; - int8_t *updateEnd = pRow + pTable->refCountPos - 1; + int32_t *updateEnd = pRow + pTable->refCountPos - 4; return (*updateEnd == 1); } From 946f6cb87f111936e8d54a742453ed1700ea0d59 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 22 Jul 2020 10:07:57 +0800 Subject: [PATCH 21/63] fix restart bug --- src/tsdb/src/tsdbMain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 7fe69021bf..772bcf48d6 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -795,6 +795,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { for (int i = 1; i < pRepo->config.maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; + tsdbSetHelperTable(&rhelper, pTable, pRepo); SCompIdx *pIdx = &(rhelper.curCompIdx); if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; From 2104d465863922c9352bb9778f88cc3f66cb1fdb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 22 Jul 2020 10:26:25 +0800 Subject: [PATCH 22/63] change kill to term signal for coverage flush. --- tests/pytest/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/test.py b/tests/pytest/test.py index a9da8e5671..678bd87336 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -96,7 +96,7 @@ if __name__ == "__main__": processID = subprocess.check_output(usePortPID, shell=True) if processID: - killCmd = "kill -9 %s" % processID + killCmd = "kill -TERM %s" % processID os.system(killCmd) fuserCmd = "fuser -k -n tcp %d" % port os.system(fuserCmd) From 9463982cecc1c00c2c47b3dc3448d8d148114c62 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 22 Jul 2020 11:27:02 +0800 Subject: [PATCH 23/63] [TD-972] drop table before rsp --- src/mnode/src/mnodeSdb.c | 3 +- src/mnode/src/mnodeTable.c | 68 +++++++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 26b0f60fcd..4b2945152b 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -661,7 +661,8 @@ bool sdbCheckRowDeleted(void *pTableInput, void *pRow) { if (pTable == NULL) return false; int32_t *updateEnd = pRow + pTable->refCountPos - 4; - return (*updateEnd == 1); + return atomic_val_compare_exchange_32(updateEnd, 1, 1) == 1; + // return (*updateEnd == 1); } int32_t sdbDeleteRow(SSdbOper *pOper) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 7478d7cd78..12ab58d949 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -72,7 +72,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg); static int32_t mnodeProcessDropTableMsg(SMnodeMsg *mnodeMsg); static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg); static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg); -static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn); +static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg); static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg); static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *mnodeMsg); @@ -759,7 +759,7 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { SChildTableObj *pCTable = (SChildTableObj *)pMsg->pTable; mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pCTable->vgId, pCTable->sid, pCTable->uid); - return mnodeProcessDropChildTableMsg(pMsg, true); + return mnodeProcessDropChildTableMsg(pMsg); } } @@ -882,7 +882,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) { SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; if (code != TSDB_CODE_SUCCESS) { - mError("app:%p:%p, table:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); + mError("app:%p:%p, stable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); } else { mLInfo("app:%p:%p, stable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); } @@ -1765,18 +1765,13 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { } } -static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { +static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; - if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); - if (pMsg->pVgroup == NULL) { - mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg, - pTable->info.tableId); - return TSDB_CODE_MND_APP_ERROR; - } + mLInfo("app:%p:%p, ctable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg)); if (pDrop == NULL) { - mError("app:%p:%p, table:%s, failed to drop ctable, no enough memory", pMsg->rpcMsg.ahandle, pMsg, + mError("app:%p:%p, ctable:%s, failed to drop ctable, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -1789,7 +1784,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); - mInfo("app:%p:%p, table:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, + mInfo("app:%p:%p, ctable:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pTable->vgId, pTable->sid, pTable->uid); SRpcMsg rpcMsg = { @@ -1807,6 +1802,40 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) { + if (code != TSDB_CODE_SUCCESS) { + SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + mError("app:%p:%p, ctable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); + return code; + } + + return mnodeSendDropChildTableMsg(pMsg, true); +} + +static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { + SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); + if (pMsg->pVgroup == NULL) { + mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId); + return TSDB_CODE_MND_APP_ERROR; + } + + SSdbOper oper = { + .type = SDB_OPER_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable, + .pMsg = pMsg, + .cb = mnodeDropChildTableCb + }; + + int32_t code = sdbDeleteRow(&oper); + if (code == TSDB_CODE_SUCCESS) { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + return code; +} + static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) { SSchema *schema = (SSchema *) pTable->schema; for (int32_t col = 0; col < pTable->numOfColumns; col++) { @@ -2220,19 +2249,6 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { return; } - SSdbOper oper = { - .type = SDB_OPER_GLOBAL, - .table = tsChildTableSdb, - .pObj = pTable - }; - - int32_t code = sdbDeleteRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - mError("app:%p:%p, table:%s, update ctables sdb error", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_MND_SDB_ERROR); - return; - } - if (mnodeMsg->pVgroup->numOfTables <= 0) { mInfo("app:%p:%p, vgId:%d, all tables is dropped, drop vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, mnodeMsg->pVgroup->vgId); @@ -2259,7 +2275,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (sdbCheckRowDeleted(tsChildTableSdb, pTable)) { mDebug("app:%p:%p, table:%s, create table rsp received, but a deleting opertion incoming, vgId:%d sid:%d uid:%" PRIu64, mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid); - mnodeProcessDropChildTableMsg(mnodeMsg, false); + mnodeSendDropChildTableMsg(mnodeMsg, false); rpcMsg->code = TSDB_CODE_SUCCESS; } From 9397bbe48a4bd18c5dceff6c0bc244dae011981b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 22 Jul 2020 11:47:02 +0800 Subject: [PATCH 24/63] [TD-972] jump or move depends on uninitialised value --- src/mnode/src/mnodeShow.c | 4 ++-- src/mnode/src/mnodeUser.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index e3d5b41be3..1d85a8cacd 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -236,7 +236,7 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { } SCMHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont; - SRpcConnInfo connInfo; + SRpcConnInfo connInfo = {0}; rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo); int32_t connId = htonl(pHBMsg->connId); @@ -284,7 +284,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { SCMConnectRsp *pConnectRsp = NULL; int32_t code = TSDB_CODE_SUCCESS; - SRpcConnInfo connInfo; + SRpcConnInfo connInfo = {0}; if (rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo) != 0) { mError("thandle:%p is already released while process connect msg", pMsg->rpcMsg.handle); code = TSDB_CODE_MND_INVALID_CONNECTION; diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 84f5d6aa58..8c783eebaf 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -358,7 +358,7 @@ static int32_t mnodeRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, voi } SUserObj *mnodeGetUserFromConn(void *pConn) { - SRpcConnInfo connInfo; + SRpcConnInfo connInfo = {0}; if (rpcGetConnInfo(pConn, &connInfo) == 0) { return mnodeGetUser(connInfo.user); } else { From 7ff6bc2eb02ad0c9367e5db1673f631461e49a36 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jul 2020 13:08:32 +0800 Subject: [PATCH 25/63] [td-225] add checks. --- src/tsdb/src/tsdbRead.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 3b6e90b740..6107c73921 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -721,7 +721,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock SQueryFilePos* cur = &pQueryHandle->cur; if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - // query ended in current block + // query ended in/started from current block if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { return false; @@ -737,6 +737,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock cur->pos = 0; } + assert(pCheckInfo->lastKey <= pBlock->keyLast); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { // the whole block is loaded in to buffer handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); @@ -754,6 +755,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock cur->pos = pBlock->numOfRows - 1; } + assert(pCheckInfo->lastKey >= pBlock->keyFirst); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); @@ -1059,7 +1061,9 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* initTableMemIterator(pQueryHandle, pCheckInfo); SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; - assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); + assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && + cur->pos >= 0 && cur->pos < pBlock->numOfRows); + TSKEY* tsArray = pCols->cols[0].pData; // for search the endPos, so the order needs to reverse @@ -1069,8 +1073,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); STable* pTable = pCheckInfo->pTableObj; - int32_t endPos = cur->pos; + if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { endPos = blockInfo.rows - 1; cur->mixBlock = (cur->pos != 0); From 7e7f91b8fc20f3128d9d741b3ee72b8565eabecd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jul 2020 13:09:53 +0800 Subject: [PATCH 26/63] [td-225] add checks. --- src/tsdb/src/tsdbRead.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 6107c73921..9962019591 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1490,6 +1490,7 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex return code; } + assert(pQueryHandle->pFileGroup != NULL); cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; cur->fid = pQueryHandle->pFileGroup->fileId; From 1f3571926e41a573146bbd373ca2354a987030d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jul 2020 13:21:49 +0800 Subject: [PATCH 27/63] [td-225] refactor codes. --- src/tsdb/src/tsdbRead.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 9962019591..7577a8e8fc 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1450,9 +1450,11 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex // current file are not overlapped with query time window, ignore remain files if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - tsdbDebug("%p remain files are not qualified for qrange:%"PRId64"-%"PRId64", ignore, %p", pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo) + (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle, + pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo); pQueryHandle->pFileGroup = NULL; + assert(pQueryHandle->numOfBlocks == 0); break; } @@ -1460,8 +1462,8 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, - numOfTables, pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, numOfTables, + pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo); assert(numOfBlocks >= 0); if (numOfBlocks == 0) { @@ -1469,7 +1471,7 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex } // todo return error code to query engine - if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { + if ((code = createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } @@ -1480,7 +1482,7 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex } // no data in file anymore - if (pQueryHandle->numOfBlocks <= 0) { + if (pQueryHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) { assert(pQueryHandle->pFileGroup == NULL); } From e8b0eb7f224d96acc7ba7bdcd5e62c1af717d50f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jul 2020 13:29:31 +0800 Subject: [PATCH 28/63] [td-225] fix compiler errors. --- src/query/inc/qResultbuf.h | 9 ++------- src/query/src/qResultbuf.c | 8 ++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h index 9d60870962..2e1126b517 100644 --- a/src/query/inc/qResultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -97,13 +97,8 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); * @param id * @return */ -static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { - if (id < pResultBuf->inMemPages) { - return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize); - } else { - return (tFilePage*) ((char*) pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize); - } -} +tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id); + /** * get the total buffer size in the format of disk file * @param pResultBuf diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 9e597aea5c..a4696ea62c 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -144,6 +144,14 @@ static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { return num; } +tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { + if (id < pResultBuf->inMemPages) { + return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize); + } else { + return (tFilePage*) ((char*) pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize); + } +} + static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { int32_t slot = getGroupIndex(pResultBuf, groupId); if (slot < 0) { From 85def32807a982751c3aa760dbf9577699c01a4c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jul 2020 14:18:18 +0800 Subject: [PATCH 29/63] [td-225] add check. --- src/tsdb/src/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7577a8e8fc..63ad8701ff 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1492,7 +1492,7 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex return code; } - assert(pQueryHandle->pFileGroup != NULL); + assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0); cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; cur->fid = pQueryHandle->pFileGroup->fileId; From b6fb79dc0dd43518442ef0a02ca71e5e905b495a Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 22 Jul 2020 14:32:36 +0800 Subject: [PATCH 30/63] fix td-900: support both config dir & file --- src/util/src/tconfig.c | 53 ++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 74acb7ce35..d561e8ba5f 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -308,38 +308,47 @@ bool taosReadGlobalCfg() { sprintf(fileName, "%s/taos.cfg", configDir); FILE* fp = fopen(fileName, "r"); + if (fp == NULL) { + struct stat s; + if (stat(configDir, &s) != 0 || (!S_ISREG(s.st_mode) && !S_ISLNK(s.st_mode))) { + //return true to follow behavior before file support + return true; + } + fp = fopen(configDir, "r"); + if (fp == NULL) { + return false; + } + } size_t len = 1024; line = calloc(1, len); - if (fp != NULL) { - while (!feof(fp)) { - memset(line, 0, len); + while (!feof(fp)) { + memset(line, 0, len); - option = value = NULL; - olen = vlen = 0; + option = value = NULL; + olen = vlen = 0; - getline(&line, &len, fp); - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; + getline(&line, &len, fp); + line[len - 1] = 0; + + paGetToken(line, &option, &olen); + if (olen == 0) continue; + option[olen] = 0; - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; + paGetToken(option + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; - // For dataDir, the format is: - // dataDir /mnt/disk1 0 - paGetToken(value + vlen + 1, &value1, &vlen1); - - taosReadConfigOption(option, value); - } - - fclose(fp); + // For dataDir, the format is: + // dataDir /mnt/disk1 0 + paGetToken(value + vlen + 1, &value1, &vlen1); + + taosReadConfigOption(option, value); } + fclose(fp); + tfree(line); return true; From ec47ae9ccab2d0a0695a66dd9b6cd8b7ad67cb5d Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 22 Jul 2020 14:38:06 +0800 Subject: [PATCH 31/63] TD-978: adjust kill command --- .../java/com/taosdata/jdbc/utils/TDNode.java | 51 ++++++++++++++++--- .../java/com/taosdata/jdbc/utils/TDNodes.java | 32 ++---------- .../test/java/com/taosdata/jdbc/BaseTest.java | 28 +++++----- 3 files changed, 62 insertions(+), 49 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java index df25076a95..800265868d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java @@ -1,6 +1,8 @@ package com.taosdata.jdbc.utils; +import java.io.BufferedReader; import java.io.File; +import java.io.InputStreamReader; import java.util.*; import java.util.concurrent.TimeUnit; @@ -31,6 +33,10 @@ public class TDNode { this.testCluster = testCluster; } + public void setRunning(int running) { + this.running = running; + } + public void searchTaosd(File dir, ArrayList taosdPath) { File[] fileList = dir.listFiles(); @@ -102,15 +108,46 @@ public class TDNode { this.running = 1; } - public void stop() { - String toBeKilled = "taosd"; + public Integer getTaosdPid() { + String cmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"; + String[] cmds = {"sh", "-c", cmd}; + try { + Process process = Runtime.getRuntime().exec(cmds); + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line = null; + Integer res = null; + while((line = reader.readLine()) != null) { + if(!line.isEmpty()) { + res = Integer.valueOf(line); + break; + } + } + + return res; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public void stop() { if (this.running != 0) { - String killCmd = "pkill -kill -x " + toBeKilled; - String[] killCmds = {"sh", "-c", killCmd}; - try { - Runtime.getRuntime().exec(killCmds).waitFor(); + Integer pid = null; + while((pid = getTaosdPid()) != null) { + + String killCmd = "kill -term " + pid; + String[] killCmds = {"sh", "-c", killCmd}; + try { + Runtime.getRuntime().exec(killCmds).waitFor(); + TimeUnit.SECONDS.sleep(2); + } catch (Exception e) { + e.printStackTrace(); + } + } + + try { for(int port = 6030; port < 6041; port ++) { String fuserCmd = "fuser -k -n tcp " + port; Runtime.getRuntime().exec(fuserCmd).waitFor(); @@ -120,7 +157,7 @@ public class TDNode { } this.running = 0; - System.out.println("dnode:" + this.index + " is stopped by pkill"); + System.out.println("dnode:" + this.index + " is stopped by kill -term"); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java index ea15ae9863..efc4c53e28 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java @@ -14,33 +14,6 @@ public class TDNodes { } } - public void setPath(String path) { - try { - String killCmd = "pkill -kill -x taosd"; - String[] killCmds = {"sh", "-c", killCmd}; - Runtime.getRuntime().exec(killCmds).waitFor(); - - String binPath = System.getProperty("user.dir"); - binPath += "/../../../debug"; - System.out.println("binPath: " + binPath); - - File file = new File(path); - binPath = file.getCanonicalPath(); - System.out.println("binPath real path: " + binPath); - - if(path.isEmpty()){ - file = new File(path + "/../../"); - path = file.getCanonicalPath(); - } - - for(int i = 0; i < tdNodes.size(); i++) { - tdNodes.get(i).setPath(path); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - public void setTestCluster(boolean testCluster) { this.testCluster = testCluster; } @@ -70,6 +43,11 @@ public class TDNodes { check(index); tdNodes.get(index - 1).setCfgConfig(option, value); } + + public TDNode getTDNode(int index) { + check(index); + return tdNodes.get(index - 1); + } public void start(int index) { check(index); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java index 5f105fb782..6c3437186f 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java @@ -1,6 +1,5 @@ package com.taosdata.jdbc; -import java.io.File; import com.taosdata.jdbc.utils.TDNodes; import org.junit.AfterClass; @@ -8,31 +7,30 @@ import org.junit.BeforeClass; public class BaseTest { - private static boolean testCluster = false; - private static String deployPath = System.getProperty("user.dir"); - private static TDNodes tdNodes = new TDNodes(); + private static boolean testCluster = false; + private static TDNodes nodes = new TDNodes(); @BeforeClass public static void setupEnv() { - try{ - File file = new File(deployPath + "/../../../"); - String rootPath = file.getCanonicalPath(); - - tdNodes.setPath(rootPath); - tdNodes.setTestCluster(testCluster); + try{ + if (nodes.getTDNode(1).getTaosdPid() != null) { + System.out.println("Kill taosd before running JDBC test"); + nodes.getTDNode(1).setRunning(1); + nodes.stop(1); + } - tdNodes.deploy(1); - tdNodes.start(1); + nodes.setTestCluster(testCluster); + nodes.deploy(1); + nodes.start(1); } catch (Exception e) { - e.printStackTrace(); - System.out.println("Base Test Exception"); + e.printStackTrace(); } } @AfterClass public static void cleanUpEnv() { - tdNodes.stop(1); + nodes.stop(1); } } \ No newline at end of file From e41aeb834903d2dc49fdf9400d7eb989925ef536 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jul 2020 15:15:37 +0800 Subject: [PATCH 32/63] [td-225] update intermediate buf page size. --- src/query/inc/qExtbuffer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h index b57c48933f..bde5fc35ac 100644 --- a/src/query/inc/qExtbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -28,9 +28,9 @@ extern "C" { #include "tdataformat.h" #include "talgo.h" -#define DEFAULT_PAGE_SIZE (1024L*4) // 16k larger than the SHistoInfo #define MAX_TMPFILE_PATH_LENGTH PATH_MAX -#define INITIAL_ALLOCATION_BUFFER_SIZE 64 +#define INITIAL_ALLOCATION_BUFFER_SIZE 64L +#define DEFAULT_PAGE_SIZE (1024L*(INITIAL_ALLOCATION_BUFFER_SIZE)) // 16k larger than the SHistoInfo typedef enum EXT_BUFFER_FLUSH_MODEL { /* From 01e348cc0182111863b2d819700719255c721726 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 22 Jul 2020 16:23:52 +0800 Subject: [PATCH 33/63] fix td-980 --- src/query/src/qExecutor.c | 6 ++++- src/tsdb/src/tsdbMeta.c | 5 +++- src/tsdb/src/tsdbRead.c | 53 ++++++++++++++++++++++++++------------- 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 906d0cfe67..1220c5ca31 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6419,8 +6419,12 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo); *contLen = size + sizeof(SRetrieveTableRsp); - // todo handle failed to allocate memory + // todo proper handle failed to allocate memory, + // current solution only avoid crash, but cannot return error code to client *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); + if (*pRsp == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } (*pRsp)->numOfRows = htonl(pQuery->rec.rows); int32_t code = pQInfo->code; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index c1923f5235..b25e734694 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -123,7 +123,10 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { int tlen2 = tsdbGetTableEncodeSize(TSDB_UPDATE_META, table); int tlen = tlen1 + tlen2; void *buf = tsdbAllocBytes(pRepo, tlen); - ASSERT(buf != NULL); + if (buf == NULL) { + goto _err; + } + if (newSuper) { void *pBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, super); ASSERT(POINTER_DISTANCE(pBuf, buf) == tlen1); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index b2204948d0..fc67c64e2e 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -201,18 +201,34 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab int32_t numOfCols = pCond->numOfCols; pQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis)); + if (pQueryHandle->statis == NULL) { + tsdbCleanupQueryHandle(pQueryHandle); + return NULL; + } pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); // todo: use list instead of array? + if (pQueryHandle->pColumns == NULL) { + tsdbCleanupQueryHandle(pQueryHandle); + return NULL; + } for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {{0}, 0}; colInfo.info = pCond->colList[i]; colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes); + if (colInfo.pData == NULL) { + tsdbCleanupQueryHandle(pQueryHandle); + return NULL; + } taosArrayPush(pQueryHandle->pColumns, &colInfo); pQueryHandle->statis[i].colId = colInfo.info.colId; } pQueryHandle->pTableCheckInfo = taosArrayInit(groupList->numOfTables, sizeof(STableCheckInfo)); + if (pQueryHandle->pTableCheckInfo == NULL) { + tsdbCleanupQueryHandle(pQueryHandle); + return NULL; + } STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); @@ -2372,28 +2388,31 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { return; } - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - destroyTableMemIterator(pTableCheckInfo); + if (pQueryHandle->pTableCheckInfo != NULL) { + size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + for (int32_t i = 0; i < size; ++i) { + STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + destroyTableMemIterator(pTableCheckInfo); - if (pTableCheckInfo->pDataCols != NULL) { - tfree(pTableCheckInfo->pDataCols->buf); + if (pTableCheckInfo->pDataCols != NULL) { + tfree(pTableCheckInfo->pDataCols->buf); + } + + tfree(pTableCheckInfo->pDataCols); + tfree(pTableCheckInfo->pCompInfo); } - - tfree(pTableCheckInfo->pDataCols); - tfree(pTableCheckInfo->pCompInfo); + taosArrayDestroy(pQueryHandle->pTableCheckInfo); } - taosArrayDestroy(pQueryHandle->pTableCheckInfo); + if (pQueryHandle->pColumns != NULL) { + size_t cols = taosArrayGetSize(pQueryHandle->pColumns); + for (int32_t i = 0; i < cols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + tfree(pColInfo->pData); + } + taosArrayDestroy(pQueryHandle->pColumns); + } - size_t cols = taosArrayGetSize(pQueryHandle->pColumns); - for (int32_t i = 0; i < cols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - tfree(pColInfo->pData); - } - - taosArrayDestroy(pQueryHandle->pColumns); taosArrayDestroy(pQueryHandle->defaultLoadColumn); tfree(pQueryHandle->pDataBlockInfo); tfree(pQueryHandle->statis); From 41c76edcc3c7e356e181dac66e64c40a8428746e Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 22 Jul 2020 17:06:53 +0800 Subject: [PATCH 34/63] fix crash in executing compress2.sim --- src/tsdb/src/tsdbRead.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index fc67c64e2e..c82d1f905a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -188,8 +188,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->allocSize = 0; if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) { - free(pQueryHandle); - return NULL; + goto out_of_memory; } tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &pQueryHandle->mem, &pQueryHandle->imem); @@ -202,13 +201,11 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis)); if (pQueryHandle->statis == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); - return NULL; + goto out_of_memory; } pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); // todo: use list instead of array? if (pQueryHandle->pColumns == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); - return NULL; + goto out_of_memory; } for (int32_t i = 0; i < numOfCols; ++i) { @@ -217,8 +214,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab colInfo.info = pCond->colList[i]; colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes); if (colInfo.pData == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); - return NULL; + goto out_of_memory; } taosArrayPush(pQueryHandle->pColumns, &colInfo); pQueryHandle->statis[i].colId = colInfo.info.colId; @@ -226,8 +222,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->pTableCheckInfo = taosArrayInit(groupList->numOfTables, sizeof(STableCheckInfo)); if (pQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); - return NULL; + goto out_of_memory; } STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); @@ -263,6 +258,11 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); return (TsdbQueryHandleT) pQueryHandle; + +out_of_memory: + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pQueryHandle); + return NULL; } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { From b337c07e8dc670fbbd7135bca005a39a04603950 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jul 2020 17:08:56 +0800 Subject: [PATCH 35/63] [td-225] fix concurrent exec query. --- src/query/inc/qExtbuffer.h | 1 - src/query/src/qExecutor.c | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h index bde5fc35ac..a992f6153c 100644 --- a/src/query/inc/qExtbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -19,7 +19,6 @@ extern "C" { #endif - #include "os.h" #include "taosmsg.h" diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 30d6cc288f..458ce228bb 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5963,8 +5963,6 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ pQuery->window.ekey, pQuery->order.order); setQueryStatus(pQuery, QUERY_COMPLETED); pQInfo->tableqinfoGroupInfo.numOfTables = 0; - - sem_post(&pQInfo->dataReady); return TSDB_CODE_SUCCESS; } @@ -5973,8 +5971,6 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo); setQueryStatus(pQuery, QUERY_COMPLETED); - - sem_post(&pQInfo->dataReady); return TSDB_CODE_SUCCESS; } @@ -6319,6 +6315,8 @@ void qTableQuery(qinfo_t qinfo) { } if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { + setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED); + qDebug("QInfo:%p no table exists for query, abort", pQInfo); sem_post(&pQInfo->dataReady); return; From 8428a1afbc85afc734dd4b18fb155f942bb9fe49 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 22 Jul 2020 09:56:08 +0000 Subject: [PATCH 36/63] [TD-926] refactor alter dnode func --- src/client/src/tscSQLParser.c | 44 ++++++++++++++------- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 41 +++++++++++++++++++- src/inc/taoserror.h | 1 + src/inc/tbalance.h | 1 + src/mnode/src/mnodeBalance.c | 1 + src/mnode/src/mnodeDnode.c | 56 +++++++++++++-------------- src/plugins/monitor/src/monitorMain.c | 43 ++++++-------------- 8 files changed, 113 insertions(+), 75 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 09db65a105..32d44dfb81 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -358,7 +358,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_CFG_DNODE: { - const char* msg2 = "invalid configure options or values"; + const char* msg2 = "invalid configure options or values, such as resetlog / debugFlag 135 / balance 'vnode:1-dnode:2' / monitor 1 "; const char* msg3 = "invalid dnode ep"; /* validate the ip address */ @@ -4674,26 +4674,42 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_TSC_INVALID_SQL; } - const int DNODE_DYNAMIC_CFG_OPTIONS_SIZE = 19; - const SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[] = { - {"resetLog", 8}, {"resetQueryCache", 15}, {"debugFlag", 9}, {"mDebugFlag", 10}, - {"dDebugFlag", 10}, {"sdbDebugFlag", 12}, {"vDebugFlag", 10}, {"cDebugFlag", 10}, - {"httpDebugFlag", 13}, {"monitorDebugFlag", 16}, {"rpcDebugFlag", 12}, {"uDebugFlag", 10}, - {"tmrDebugFlag", 12}, {"qDebugflag", 10}, {"sDebugflag", 10}, {"tsdbDebugFlag", 13}, - {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"monitor", 7}}; + const int tokenLogEnd = 2; + const int tokenBalance = 2; + const int tokenMonitor = 3; + const int tokenDebugFlag = 4; + const int tokenDebugFlagEnd = 20; + const SDNodeDynConfOption cfgOptions[] = { + {"resetLog", 8}, {"resetQueryCache", 15}, {"balance", 7}, {"monitor", 7}, + {"debugFlag", 9}, {"monitorDebugFlag", 16}, {"vDebugFlag", 10}, {"mDebugFlag", 10}, + {"cDebugFlag", 10}, {"httpDebugFlag", 13}, {"qDebugflag", 10}, {"sdbDebugFlag", 12}, + {"uDebugFlag", 10}, {"tsdbDebugFlag", 13}, {"sDebugflag", 10}, {"rpcDebugFlag", 12}, + {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, + }; SSQLToken* pOptionToken = &pOptions->a[1]; if (pOptions->nTokens == 2) { // reset log and reset query cache does not need value - for (int32_t i = 0; i < 2; ++i) { - const SDNodeDynConfOption* pOption = &DNODE_DYNAMIC_CFG_OPTIONS[i]; + for (int32_t i = 0; i < tokenLogEnd; ++i) { + const SDNodeDynConfOption* pOption = &cfgOptions[i]; if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { return TSDB_CODE_SUCCESS; } } - } else if ((strncasecmp(DNODE_DYNAMIC_CFG_OPTIONS[DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1].name, pOptionToken->z, pOptionToken->n) == 0) && - (DNODE_DYNAMIC_CFG_OPTIONS[DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1].len == pOptionToken->n)) { + } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && + (cfgOptions[tokenBalance].len == pOptionToken->n)) { + SSQLToken* pValToken = &pOptions->a[2]; + int32_t vnodeIndex = 0; + int32_t dnodeIndex = 0; + strdequote(pValToken->z); + bool parseOk = taosCheckBalanceCfgOptions(pValToken->z, &vnodeIndex, &dnodeIndex); + if (!parseOk) { + return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid + } + return TSDB_CODE_SUCCESS; + } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && + (cfgOptions[tokenMonitor].len == pOptionToken->n)) { SSQLToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val != 0 && val != 1) { @@ -4709,8 +4725,8 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_TSC_INVALID_SQL; } - for (int32_t i = 2; i < DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1; ++i) { - const SDNodeDynConfOption* pOption = &DNODE_DYNAMIC_CFG_OPTIONS[i]; + for (int32_t i = tokenDebugFlag; i < tokenDebugFlagEnd; ++i) { + const SDNodeDynConfOption* pOption = &cfgOptions[i]; if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { /* options is valid */ diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 6506707457..7ba96ceb60 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -174,6 +174,7 @@ bool taosCheckGlobalCfg(); void taosSetAllDebugFlag(); bool taosCfgDynamicOptions(char *msg); int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); +bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeIndex, int32_t *dnodeIndex); #ifdef __cplusplus } diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 56c63ee49d..c79b016b93 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -198,6 +198,7 @@ int32_t tsdbDebugFlag = 131; int32_t (*monitorStartSystemFp)() = NULL; void (*monitorStopSystemFp)() = NULL; +void (*monitorExecuteSQLFp)(char *sql) = NULL; static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; @@ -252,11 +253,15 @@ bool taosCfgDynamicOptions(char *msg) { if (monitorStartSystemFp) { (*monitorStartSystemFp)(); uInfo("monitor is enabled"); + } else { + uError("monitor can't be updated, for monitor not initialized"); } } else { if (monitorStopSystemFp) { (*monitorStopSystemFp)(); uInfo("monitor is disabled"); + } else { + uError("monitor can't be updated, for monitor not initialized"); } } return true; @@ -276,7 +281,12 @@ bool taosCfgDynamicOptions(char *msg) { } if (strncasecmp(option, "resetQueryCache", 15) == 0) { - uError("reset query cache can't be executed, for monitor not initialized"); + if (monitorExecuteSQLFp) { + (*monitorExecuteSQLFp)("resetQueryCache"); + uInfo("resetquerycache is executed"); + } else { + uError("resetquerycache can't be executed, for monitor not started"); + } } return false; @@ -1300,3 +1310,32 @@ int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { return 0; } + +/* + * alter dnode 1 balance "vnode:1-dnode:2" + */ + +bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeIndex, int32_t *dnodeIndex) { + int len = strlen(option); + if (strncasecmp(option, "vnode:", 6) != 0) { + return false; + } + + int pos = 0; + for (; pos < len; ++pos) { + if (option[pos] == '-') break; + } + + if (++pos >= len) return false; + if (strncasecmp(option + pos, "dnode:", 6) != 0) { + return false; + } + + *vnodeIndex = strtol(option + 6, NULL, 10); + *dnodeIndex = strtol(option + pos + 6, NULL, 10); + if (*vnodeIndex <= 1 || *dnodeIndex <= 0) { + return false; + } + + return true; +} \ No newline at end of file diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 3503e39d31..59b2c0220b 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -227,6 +227,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 0x080B, "grant cpu // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, 0, 0x0900, "sync invalid configuration") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "sync module not enabled") // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "wal app error") diff --git a/src/inc/tbalance.h b/src/inc/tbalance.h index c52f5afaaa..9ee8d73189 100644 --- a/src/inc/tbalance.h +++ b/src/inc/tbalance.h @@ -29,6 +29,7 @@ void balanceAsyncNotify(); void balanceSyncNotify(); void balanceReset(); int32_t balanceAllocVnodes(struct SVgObj *pVgroup); +int32_t balanceCfgDnode(struct SDnodeObj *pDnode, const char *option); int32_t balanceDropDnode(struct SDnodeObj *pDnode); #ifdef __cplusplus diff --git a/src/mnode/src/mnodeBalance.c b/src/mnode/src/mnodeBalance.c index 23fc10d0bd..d2ec6dd36e 100644 --- a/src/mnode/src/mnodeBalance.c +++ b/src/mnode/src/mnodeBalance.c @@ -28,6 +28,7 @@ void balanceCleanUp() {} void balanceAsyncNotify() {} void balanceSyncNotify() {} void balanceReset() {} +int32_t balanceCfgDnode(struct SDnodeObj *pDnode, const char *option) { return TSDB_CODE_SYN_NOT_ENABLED; } int32_t balanceAllocVnodes(SVgObj *pVgroup) { void * pIter = NULL; diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 7edba8662e..26c4b7a3ea 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -277,45 +277,45 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { SCMCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont; if (pCmCfgDnode->ep[0] == 0) { tstrncpy(pCmCfgDnode->ep, tsLocalEp, TSDB_EP_LEN); - } - - int32_t dnodeId = 0; - char* pos = strchr(pCmCfgDnode->ep, ':'); - if (NULL == pos) { - dnodeId = strtol(pCmCfgDnode->ep, NULL, 10); - if (dnodeId <= 0 || dnodeId > 65536) { - mError("failed to cfg dnode, invalid dnodeId:%s", pCmCfgDnode->ep); - return TSDB_CODE_MND_DNODE_NOT_EXIST; - } } - SRpcEpSet epSet = mnodeGetEpSetFromIp(pCmCfgDnode->ep); - if (dnodeId != 0) { - SDnodeObj *pDnode = mnodeGetDnode(dnodeId); + SDnodeObj *pDnode = mnodeGetDnodeByEp(pCmCfgDnode->ep); + if (pDnode == NULL) { + int32_t dnodeId = strtol(pCmCfgDnode->ep, NULL, 10); + if (dnodeId <= 0 || dnodeId > 65536) { + mError("failed to cfg dnode, invalid dnodeEp:%s", pCmCfgDnode->ep); + return TSDB_CODE_MND_DNODE_NOT_EXIST; + } + + pDnode = mnodeGetDnode(dnodeId); if (pDnode == NULL) { mError("failed to cfg dnode, invalid dnodeId:%d", dnodeId); return TSDB_CODE_MND_DNODE_NOT_EXIST; } - epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); - mnodeDecDnodeRef(pDnode); } - SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg)); - strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep); - strcpy(pMdCfgDnode->config, pCmCfgDnode->config); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); + mnodeDecDnodeRef(pDnode); - SRpcMsg rpcMdCfgDnodeMsg = { - .ahandle = 0, - .code = 0, - .msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE, - .pCont = pMdCfgDnode, - .contLen = sizeof(SMDCfgDnodeMsg) - }; + if (strncasecmp(pCmCfgDnode->config, "balance", 7) == 0) { + return balanceCfgDnode(pDnode, pCmCfgDnode->config + 8); + } else { + SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg)); + strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep); + strcpy(pMdCfgDnode->config, pCmCfgDnode->config); - mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); - dnodeSendMsgToDnode(&epSet, &rpcMdCfgDnodeMsg); + SRpcMsg rpcMdCfgDnodeMsg = { + .ahandle = 0, + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE, + .pCont = pMdCfgDnode, + .contLen = sizeof(SMDCfgDnodeMsg) + }; - return TSDB_CODE_SUCCESS; + mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); + dnodeSendMsgToDnode(&epSet, &rpcMdCfgDnodeMsg); + return TSDB_CODE_SUCCESS; + } } static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index e6e8cf982b..0cc28bb82c 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -27,7 +27,6 @@ #include "dnode.h" #include "monitor.h" - #define monitorFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }} #define monitorError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }} #define monitorWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }} @@ -78,6 +77,7 @@ static void monitorStartTimer(); static void monitorSaveSystemInfo(); extern int32_t (*monitorStartSystemFp)(); extern void (*monitorStopSystemFp)(); +extern void (*monitorExecuteSQLFp)(char *sql); static void monitorCheckDiskUsage(void *para, void *unused) { taosGetDisk(); @@ -207,6 +207,7 @@ static void monitorInitDatabase() { taos_query_a(tsMonitorConn.conn, tsMonitorConn.sql, monitorInitDatabaseCb, NULL); } else { tsMonitorConn.state = MONITOR_STATE_INITIALIZED; + monitorExecuteSQLFp = monitorExecuteSQL; monitorInfo("monitor service init success"); monitorStartTimer(); @@ -230,6 +231,7 @@ static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) { void monitorStopSystem() { monitorInfo("monitor module is stopped"); + monitorExecuteSQLFp = NULL; tsMonitorConn.state = MONITOR_STATE_STOPPED; if (tsMonitorConn.initTimer != NULL) { taosTmrStopA(&(tsMonitorConn.initTimer)); @@ -248,33 +250,13 @@ static void monitorStartTimer() { taosTmrReset(monitorSaveSystemInfo, tsMonitorInterval * 1000, NULL, tscTmr, &tsMonitorConn.timer); } -static void dnodeMontiorInsertAcctCallback(void *param, TAOS_RES *result, int32_t code) { +static void dnodeMontiorLogCallback(void *param, TAOS_RES *result, int32_t code) { if (code < 0) { - monitorError("monitor:%p, save account info failed, code:%s", tsMonitorConn.conn, tstrerror(code)); + monitorError("monitor:%p, save %s failed, reason:%s", tsMonitorConn.conn, (char *)param, tstrerror(code)); } else if (code == 0) { - monitorError("monitor:%p, save account info failed, affect rows:%d", tsMonitorConn.conn, code); + monitorError("monitor:%p, save %s failed, affect rows:%d", tsMonitorConn.conn, (char *)param, code); } else { - monitorDebug("monitor:%p, save account info success, code:%s", tsMonitorConn.conn, tstrerror(code)); - } -} - -static void dnodeMontiorInsertSysCallback(void *param, TAOS_RES *result, int32_t code) { - if (code < 0) { - monitorError("monitor:%p, save system info failed, code:%s %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); - } else if (code == 0) { - monitorError("monitor:%p, save system info failed, affect rows:%d %s", tsMonitorConn.conn, code, tsMonitorConn.sql); - } else { - monitorDebug("monitor:%p, save system info success, code:%s %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); - } -} - -static void dnodeMontiorInsertLogCallback(void *param, TAOS_RES *result, int32_t code) { - if (code < 0) { - monitorError("monitor:%p, save log failed, code:%s", tsMonitorConn.conn, tstrerror(code)); - } else if (code == 0) { - monitorError("monitor:%p, save log failed, affect rows:%d", tsMonitorConn.conn, code); - } else { - monitorDebug("monitor:%p, save log info success, code:%s", tsMonitorConn.conn, tstrerror(code)); + monitorDebug("monitor:%p, save %s info success, reason:%s", tsMonitorConn.conn, (char *)param, tstrerror(code)); } } @@ -359,7 +341,7 @@ static void monitorSaveSystemInfo() { pos += monitorBuildReqSql(sql + pos); monitorDebug("monitor:%p, save system info, sql:%s", tsMonitorConn.conn, sql); - taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertSysCallback, "log"); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorLogCallback, "sys"); if (tsMonitorConn.timer != NULL && tsMonitorConn.state != MONITOR_STATE_STOPPED) { monitorStartTimer(); @@ -397,7 +379,7 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) { pMon->accessState); monitorDebug("monitor:%p, save account info, sql %s", tsMonitorConn.conn, sql); - taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertAcctCallback, "account"); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorLogCallback, "account"); } void monitorSaveLog(int32_t level, const char *const format, ...) { @@ -421,14 +403,11 @@ void monitorSaveLog(int32_t level, const char *const format, ...) { sql[len++] = 0; monitorDebug("monitor:%p, save log, sql: %s", tsMonitorConn.conn, sql); - taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertLogCallback, "log"); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorLogCallback, "log"); } void monitorExecuteSQL(char *sql) { if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) return; - monitorDebug("monitor:%p, execute sql: %s", tsMonitorConn.conn, sql); - - // bug while insert binary - // taos_query_a(tsMonitorConn.conn, sql, NULL, NULL); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorLogCallback, "sql"); } From 8b1451b3eabd0c9ba2022e1811460fc652007f8e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 22 Jul 2020 10:01:54 +0000 Subject: [PATCH 37/63] scripts --- tests/script/general/alter/dnode.sim | 71 ++++++++++++++++++++++++++++ tests/script/jenkins/basic.txt | 1 + 2 files changed, 72 insertions(+) create mode 100644 tests/script/general/alter/dnode.sim diff --git a/tests/script/general/alter/dnode.sim b/tests/script/general/alter/dnode.sim new file mode 100644 index 0000000000..20ce879979 --- /dev/null +++ b/tests/script/general/alter/dnode.sim @@ -0,0 +1,71 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print ======== step1 +sql alter dnode 1 resetlog +sql alter dnode 1 monitor 1 + +sleep 5000 +sql select * from log.dn +if $rows <= 0 then + return -1 +endi + +print ======== step2 + +sql alter dnode 1 resetquerycache +sql alter dnode 1 debugFlag 135 +sql alter dnode 1 debugFlag 131 +sql alter dnode 1 monitor 0 +sql alter dnode 1 debugFlag 135 +sql alter dnode 1 monitorDebugFlag 135 +sql alter dnode 1 vDebugFlag 135 +sql alter dnode 1 mDebugFlag 135 +sql alter dnode 1 cDebugFlag 135 +sql alter dnode 1 httpDebugFlag 135 +sql alter dnode 1 qDebugflag 135 +sql alter dnode 1 sdbDebugFlag 135 +sql alter dnode 1 uDebugFlag 135 +sql alter dnode 1 tsdbDebugFlag 135 +sql alter dnode 1 sDebugflag 135 +sql alter dnode 1 rpcDebugFlag 135 +sql alter dnode 1 dDebugFlag 135 +sql alter dnode 1 mqttDebugFlag 135 +sql alter dnode 1 wDebugFlag 135 +sql alter dnode 1 tmrDebugFlag 135 +sql_error alter dnode 2 wDebugFlag 135 +sql_error alter dnode 2 tmrDebugFlag 135 + +print ======== step3 +sql_error alter $hostname1 debugFlag 135 +sql_error alter $hostname1 monitorDebugFlag 135 +sql_error alter $hostname1 vDebugFlag 135 +sql_error alter $hostname1 mDebugFlag 135 +sql_error alter dnode $hostname2 debugFlag 135 +sql_error alter dnode $hostname2 monitorDebugFlag 135 +sql_error alter dnode $hostname2 vDebugFlag 135 +sql_error alter dnode $hostname2 mDebugFlag 135 +sql alter dnode $hostname1 debugFlag 135 +sql alter dnode $hostname1 monitorDebugFlag 135 +sql alter dnode $hostname1 vDebugFlag 135 +sql alter dnode $hostname1 tmrDebugFlag 131 + +print ======== step4 +sql_error sql alter dnode 1 balance 0 +sql_error sql alter dnode 1 balance vnode:1-dnode:1 +sql_error sql alter dnode 1 balance "vnode:1" +sql_error sql alter dnode 1 balance "vnode:1-dnode:1" +sql_error sql alter dnode 1 balance "dnode:1-vnode:1" +sql_error sql alter dnode 1 balance "dnode:1-" +sql_error sql alter dnode 1 balance "vnode:2-dnod" +sql alter dnode 1 balance "vnode:2-dnode:1" -x step4 +step4: + +print ======= over +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 3b9806558e..2fd5db0165 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -3,6 +3,7 @@ cd ../../../debug; make ./test.sh -f general/alter/cached_schema_after_alter.sim ./test.sh -f general/alter/count.sim +./test.sh -f general/alter/dnode.sim ./test.sh -f general/alter/import.sim ./test.sh -f general/alter/insert1.sim ./test.sh -f general/alter/insert2.sim From 057e808930c97d9f16ec8f51ef88f40f2a3e5126 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jul 2020 18:17:38 +0800 Subject: [PATCH 38/63] [td-225] for expriments. --- src/vnode/src/vnodeRead.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 973df7c5a1..83986de682 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -122,7 +122,8 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { // current connect is broken if (code == TSDB_CODE_SUCCESS) { - handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t) pQInfo); +// handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t) pQInfo); + handle = &pQInfo; if (handle == NULL) { // failed to register qhandle vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo, tstrerror(pRsp->code)); @@ -133,11 +134,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRsp->qhandle = htobe64((uint64_t) pQInfo); } - pQInfo = NULL; +// pQInfo = NULL; if (handle != NULL && vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); +// qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); return pRsp->code; } } else { @@ -148,12 +149,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, register qhandle and return to app", vgId, *handle); vnodePutItemIntoReadQueue(pVnode, *handle); - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); +// qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } } else { assert(pCont != NULL); - handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont); + void* p = (void*) pCont; + handle = &p; +// handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont); if (handle == NULL) { vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle); code = TSDB_CODE_QRY_INVALID_QHANDLE; @@ -162,7 +165,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { code = TSDB_CODE_VND_ACTION_IN_PROGRESS; qTableQuery(*handle); // do execute query } - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); +// qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } return code; @@ -181,7 +184,11 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { memset(pRet, 0, sizeof(SRspRet)); int32_t code = TSDB_CODE_SUCCESS; - void** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle); + void** handle = NULL; + void* p1 = (void*) pRetrieve->qhandle; + handle = &p1; + +// void** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle); if (handle == NULL || (*handle) != (void*) pRetrieve->qhandle) { code = TSDB_CODE_QRY_INVALID_QHANDLE; vDebug("vgId:%d, invalid qhandle in fetch result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle); @@ -201,7 +208,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (pRetrieve->free == 1) { vDebug("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle); qKillQuery(*handle); - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); +// qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); pRet->len = sizeof(SRetrieveTableRsp); @@ -234,7 +241,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } } - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); + UNUSED(freeHandle); +// qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); return code; } From a6d92173d937d1cf5e276d5a014b5e9f02b36979 Mon Sep 17 00:00:00 2001 From: Liu Tao Date: Wed, 22 Jul 2020 18:53:49 +0800 Subject: [PATCH 39/63] performance tune --- src/client/src/tscUtil.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 27f2535bdc..1ef5d3feaf 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -351,14 +351,14 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { int32_t cmd = pCmd->command; if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscRemoveFromSqlList(pSql); + //tscRemoveFromSqlList(pSql); } // pSql->sqlstr will be used by tscBuildQueryStreamDesc if (pObj->signature == pObj) { - pthread_mutex_lock(&pObj->mutex); + //pthread_mutex_lock(&pObj->mutex); tfree(pSql->sqlstr); - pthread_mutex_unlock(&pObj->mutex); + //pthread_mutex_unlock(&pObj->mutex); } tscFreeSqlResult(pSql); @@ -1885,7 +1885,7 @@ void tscDoQuery(SSqlObj* pSql) { } if (pCmd->command == TSDB_SQL_SELECT) { - tscAddIntoSqlList(pSql); + //tscAddIntoSqlList(pSql); } if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) { From 63709430bfdbd8fdf488c340fc32612d445d3fc2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 22 Jul 2020 16:34:43 +0000 Subject: [PATCH 40/63] TD-884: fixbug --- src/client/src/tscServer.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9a1fa77629..6c1faf6f9d 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -106,13 +106,14 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; taosCorBeginWrite(&pVgroupInfo->version); - //TODO(dengyihao), dont care vgid + tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse); pVgroupInfo->inUse = pEpSet->inUse; pVgroupInfo->numOfEps = pEpSet->numOfEps; - for (int32_t i = 0; pVgroupInfo->numOfEps; i++) { + for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { strncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); pVgroupInfo->epAddr[i].port = pEpSet->port[i]; } + tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); taosCorEndWrite(&pVgroupInfo->version); } void tscPrintMgmtEp() { @@ -283,9 +284,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { } if (pEpSet) { - //SRpcEpSet dump; - tscEpSetHtons(pEpSet); - if (tscEpSetIsEqual(&pSql->epSet, pEpSet)) { + if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { if(pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pEpSet); } else { From 8e637b4f7024dbb1d79148f051941b1549f9c6e7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 23 Jul 2020 10:13:12 +0800 Subject: [PATCH 41/63] TD-982 --- src/tsdb/src/tsdbMemTable.c | 5 ++--- src/tsdb/src/tsdbRWHelper.c | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 5325eb5b0c..7708646a61 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -264,13 +264,11 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } do { - if (numOfRows >= maxRowsToRead) break; - SDataRow row = tsdbNextIterRow(pIter); if (row == NULL) break; keyNext = dataRowKey(row); - if (keyNext < 0 || keyNext > maxKey) break; + if (keyNext > maxKey) break; bool keyFiltered = false; if (nFilterKeys != 0) { @@ -289,6 +287,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } if (!keyFiltered) { + if (numOfRows >= maxRowsToRead) break; if (pCols) { if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index f8792469cb..3063f22a28 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1477,6 +1477,11 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1; } +#ifndef NDEBUG + TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter); + ASSERT(keyNext < 0 || keyNext > pIdx->maxKey); +#endif + return 0; } @@ -1561,11 +1566,12 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; tblkIdx++; } + ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || + tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { ASSERT(keyFirst <= blkKeyLast); int16_t colId = 0; if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; - ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows); slIter = *(pCommitIter->pIter); int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows); @@ -1574,9 +1580,10 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (rows2 == 0) { // all filtered out *(pCommitIter->pIter) = slIter; + ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || + tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2; - ASSERT(rows3 >= rows2); if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) { int rows = (rows1 >= rows3) ? rows3 : rows2; @@ -1588,6 +1595,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; + ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || + tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; int round = 0; @@ -1608,6 +1617,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, round++; tblkIdx++; } + ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || + tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } } } From fd7b5e6e8655e231d9fecbddf886db3cffdb500c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jul 2020 13:29:14 +0800 Subject: [PATCH 42/63] scripts --- src/mnode/src/mnodeVgroup.c | 34 +++++-------------- tests/script/jenkins/basic.txt | 3 +- tests/script/sh/deploy.sh | 2 +- .../arbitrator/dn3_mn1_r2_vnode_delDir.sim | 12 +++---- .../arbitrator/dn3_mn1_r3_vnode_delDir.sim | 16 ++++----- .../arbitrator/dn3_mn1_replica_change.sim | 16 ++++----- .../arbitrator/dn3_mn1_vnode_change.sim | 12 +++---- .../dn3_mn1_vnode_corruptFile_offline.sim | 16 ++++----- .../dn3_mn1_vnode_corruptFile_online.sim | 12 +++---- .../dn3_mn1_vnode_createErrData_online.sim | 12 +++---- .../arbitrator/dn3_mn1_vnode_delDir.sim | 16 ++++----- .../dn3_mn1_vnode_noCorruptFile_offline.sim | 16 ++++----- .../arbitrator/dn3_mn1_vnode_nomaster.sim | 8 ++--- .../offline_replica2_alterTable_online.sim | 4 +-- .../offline_replica2_alterTag_online.sim | 4 +-- .../offline_replica2_createTable_online.sim | 4 +-- .../offline_replica2_dropDb_online.sim | 4 +-- .../offline_replica2_dropTable_online.sim | 4 +-- .../offline_replica3_alterTable_online.sim | 4 +-- .../offline_replica3_alterTag_online.sim | 4 +-- .../offline_replica3_createTable_online.sim | 4 +-- .../offline_replica3_dropDb_online.sim | 4 +-- .../offline_replica3_dropTable_online.sim | 4 +-- .../sync_replica2_alterTable_add.sim | 4 +-- .../sync_replica2_alterTable_drop.sim | 4 +-- .../arbitrator/sync_replica2_dropDb.sim | 4 +-- .../arbitrator/sync_replica2_dropTable.sim | 4 +-- .../sync_replica3_alterTable_add.sim | 4 +-- .../sync_replica3_alterTable_drop.sim | 4 +-- .../arbitrator/sync_replica3_createTable.sim | 4 +-- ...ca3_dnodeChang_DropAddAlterTableDropDb.sim | 28 +++++++-------- .../arbitrator/sync_replica3_dropDb.sim | 4 +-- .../arbitrator/sync_replica3_dropTable.sim | 4 +-- .../migrate/mn2_vn2_repl2_rmMnodeDir.sim | 12 +++---- .../migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim | 12 +++---- ..._repl2_rmMnodeVnodeDir_stopAll_starAll.sim | 12 +++---- .../migrate/mn2_vn2_repl2_rmVnodeDir.sim | 12 +++---- 37 files changed, 154 insertions(+), 173 deletions(-) diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 46255a6c8f..089168c31e 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -592,7 +592,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; + pShow->bytes[cols] = 8 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -619,12 +619,6 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 40 + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end_point"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "vstatus"); @@ -716,27 +710,15 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v *(int16_t *) pWrite = pVgroup->vnodeGid[i].dnodeId; cols++; - SDnodeObj *pDnode = pVgroup->vnodeGid[i].pDnode; - + SDnodeObj * pDnode = pVgroup->vnodeGid[i].pDnode; + const char *role = "NULL"; if (pDnode != NULL) { - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols]); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char *role = mnodeGetMnodeRoleStr(pVgroup->vnodeGid[i].role); - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); - cols++; - } else { - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - const char *src = "NULL"; - STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); - cols++; + role = mnodeGetMnodeRoleStr(pVgroup->vnodeGid[i].role); } + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); + cols++; } mnodeDecVgroupRef(pVgroup); diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 2fd5db0165..828b907b9c 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -327,10 +327,9 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim #./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim -./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim ./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim ./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim ./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index adbb8207be..ac65237319 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -125,7 +125,7 @@ echo "mqttDebugFlag 135" >> $TAOS_CFG echo "qdebugFlag 135" >> $TAOS_CFG echo "rpcDebugFlag 135" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 143" >> $TAOS_CFG +echo "udebugFlag 135" >> $TAOS_CFG echo "sdebugFlag 135" >> $TAOS_CFG echo "wdebugFlag 135" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG diff --git a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim index 3df21a1c5d..b625619678 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim @@ -137,8 +137,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_2 -$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data5_2 +$dnode2Vtatus = $data7_2 if $dnode3Vtatus != offline then sleep 2000 @@ -204,8 +204,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_2 -$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data5_2 +$dnode2Vtatus = $data7_2 print dnode2Vtatus: $dnode3Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -319,8 +319,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_2 -$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data5_2 +$dnode2Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim index 97b77aa66f..c647a35a5c 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim @@ -139,8 +139,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 @@ -206,8 +206,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -325,8 +325,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -386,8 +386,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim index 95fb21a017..23ace47e3c 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim @@ -358,10 +358,10 @@ print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $dat print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 $data10_4 #print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 #print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$thirdDnode_2 = $data10_1 -$thirdDnode_3 = $data10_2 -$thirdDnode_4 = $data10_3 -$thirdDnode_5 = $data10_4 +$thirdDnode_2 = $data8_1 +$thirdDnode_3 = $data8_2 +$thirdDnode_4 = $data8_3 +$thirdDnode_5 = $data8_4 if $thirdDnode_2 != null then sleep 2000 @@ -405,10 +405,10 @@ print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $dat print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 #print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 #print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$sencodDnode_2 = $data7_1 -$sencodDnode_3 = $data7_2 -$sencodDnode_4 = $data7_3 -$sencodDnode_5 = $data7_4 +$sencodDnode_2 = $data6_1 +$sencodDnode_3 = $data6_2 +$sencodDnode_4 = $data6_3 +$sencodDnode_5 = $data6_4 if $sencodDnode_2 != null then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim index ee974ad6db..b18edf0670 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim @@ -138,8 +138,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 @@ -213,8 +213,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -287,8 +287,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim index 6a8d69c271..3b208e6f1a 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim @@ -144,8 +144,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_2 -$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data5_2 +$dnode2Vtatus = $data7_2 if $dnode3Vtatus != offline then sleep 2000 @@ -234,8 +234,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode2Vtatus != master then sleep 2000 @@ -313,8 +313,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -393,8 +393,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus == offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim index 1642e8a322..4f737b9c40 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim @@ -196,8 +196,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -269,8 +269,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -325,8 +325,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != slave then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim index 59a62aa63d..f50081ea70 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim @@ -158,8 +158,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -231,8 +231,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -287,8 +287,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != slave then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim index a0629abab8..3582a3b7b2 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim @@ -141,8 +141,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 @@ -201,8 +201,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -301,8 +301,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -417,8 +417,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim index e2fa1fbe2f..4040318801 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim @@ -144,8 +144,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_2 -$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data5_2 +$dnode2Vtatus = $data7_2 if $dnode3Vtatus != offline then sleep 2000 @@ -234,8 +234,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != master then sleep 2000 @@ -313,8 +313,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -392,8 +392,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus == offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim index a425bc3e92..53209b0d5a 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim @@ -151,8 +151,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 @@ -259,8 +259,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_3 -$dnode2Vtatus = $data9_3 +$dnode3Vtatus = $data5_3 +$dnode2Vtatus = $data7_3 if $dnode3Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim index 12d0a19f73..c11bfa62f9 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim @@ -139,8 +139,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim index be032dfa76..bed5cefa76 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim index d79ba724f7..9fdb70142b 100644 --- a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim index 20f4552fe3..6752e0345d 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim index 7131df5d24..1280cfe7a0 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim index 48eb9512c8..27c2ba4328 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim index 13d88f0f38..86fb51cfa9 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim index c96d7f5b6e..4dcc5977f3 100644 --- a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim @@ -144,8 +144,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim index 1392e6c8a7..4f4cf6a263 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim index 729aff56c0..98946c4b7c 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim index 25da315be4..dc8b1d015d 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim index a26c1b04ed..d2c82f98ff 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim index 1556a03b67..96aec2dcf8 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim index 669b41becb..a1d7d18c94 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim index 00c42cb7e3..ba973cbe06 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim index 1d774e195e..9b918acc8e 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_createTable.sim b/tests/script/unique/arbitrator/sync_replica3_createTable.sim index db29eaf838..3bf274a3ea 100644 --- a/tests/script/unique/arbitrator/sync_replica3_createTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_createTable.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim index 5b751e96f8..c8fe96008b 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim @@ -139,8 +139,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 @@ -207,8 +207,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != slave then sleep 2000 @@ -240,8 +240,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != master then sleep 2000 @@ -314,8 +314,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != slave then sleep 2000 @@ -347,8 +347,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != master then sleep 2000 @@ -437,8 +437,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != slave then sleep 2000 @@ -470,8 +470,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != master then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim index 83f1d16856..4c7bb3d26c 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim index e874afeaed..7ac424666a 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim index 7dd911a63e..12f0013191 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim @@ -194,13 +194,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data6_4 -$d2v3status = $data6_2 -$d2v4status = $data6_3 +$d2v2status = $data5_4 +$d2v3status = $data5_2 +$d2v4status = $data5_3 -$d1v2status = $data9_4 -$d1v3status = $data9_2 -$d1v4status = $data9_3 +$d1v2status = $data7_4 +$d1v3status = $data7_2 +$d1v4status = $data7_3 if $d2v2status != master then sleep 2000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim index bcbf388946..29da1fd773 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim @@ -196,13 +196,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data6_4 -$d2v3status = $data6_2 -$d2v4status = $data6_3 +$d2v2status = $data5_4 +$d2v3status = $data5_2 +$d2v4status = $data5_3 -$d1v2status = $data9_4 -$d1v3status = $data9_2 -$d1v4status = $data9_3 +$d1v2status = $data7_4 +$d1v3status = $data7_2 +$d1v4status = $data7_3 if $d2v2status != master then sleep 2000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim index ab76affafc..691b2cb568 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim @@ -161,13 +161,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data6_4 -$d2v3status = $data6_2 -$d2v4status = $data6_3 +$d2v2status = $data5_4 +$d2v3status = $data5_2 +$d2v4status = $data5_3 -$d1v2status = $data9_4 -$d1v3status = $data9_2 -$d1v4status = $data9_3 +$d1v2status = $data7_4 +$d1v3status = $data7_2 +$d1v4status = $data7_3 if $d2v2status != master then sleep 2000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim index f6a60d6b08..8a1132de2e 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim @@ -194,13 +194,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data6_4 -$d2v3status = $data6_2 -$d2v4status = $data6_3 +$d2v2status = $data5_4 +$d2v3status = $data5_2 +$d2v4status = $data5_3 -$d1v2status = $data9_4 -$d1v3status = $data9_2 -$d1v4status = $data9_3 +$d1v2status = $data7_4 +$d1v3status = $data7_2 +$d1v4status = $data7_3 if $d2v2status != master then sleep 2000 From 8090b713f71c113410760a3f9c543ee6b8030690 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jul 2020 06:05:19 +0000 Subject: [PATCH 43/63] scripts --- tests/script/general/parser/stream_on_sys.sim | 2 +- tests/script/jenkins/basic.txt | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/script/general/parser/stream_on_sys.sim b/tests/script/general/parser/stream_on_sys.sim index 5405e09256..5507b4db48 100644 --- a/tests/script/general/parser/stream_on_sys.sim +++ b/tests/script/general/parser/stream_on_sys.sim @@ -28,7 +28,7 @@ sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), a sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) sliding(2s) sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) sliding(2s) sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) sliding(2s) -sleep 20000 +sleep 120000 sql select * from cpustrm if $rows <= 0 then return -1 diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 828b907b9c..e953d337e4 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -309,13 +309,10 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica3_repeat.sim ./test.sh -f unique/vnode/replica3_vgroup.sim -# stream still has bugs -#./test.sh -f general/parser/stream_on_sys.sim -#./test.sh -f general/parser/repeatStream.sim - -#./test.sh -f general/stream/metrics_del.sim +./test.sh -f general/parser/stream_on_sys.sim +./test.sh -f general/stream/metrics_del.sim ./test.sh -f general/stream/metrics_n.sim -#./test.sh -f general/stream/metrics_replica1_vnoden.sim +./test.sh -f general/stream/metrics_replica1_vnoden.sim ./test.sh -f general/stream/restart_stream.sim ./test.sh -f general/stream/stream_3.sim ./test.sh -f general/stream/stream_restart.sim @@ -332,7 +329,7 @@ cd ../../../debug; make #./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim -./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +#./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim # lower the priority while file corruption #./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim #./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim From 74a5a231ad4a372c7da16d3ef643bb6892aa9710 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jul 2020 14:31:44 +0800 Subject: [PATCH 44/63] [td-225]add some logs. --- src/tsdb/src/tsdbRead.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 63ad8701ff..ccfcff3599 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -622,7 +622,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo .uid = (_checkInfo)->tableId.uid}) -static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { +static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { STsdbRepo *pRepo = pQueryHandle->pTsdb; bool blockLoaded = false; int64_t st = taosGetTimestampUs(); @@ -657,8 +657,8 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo int64_t elapsedTime = (taosGetTimestampUs() - st); pQueryHandle->cost.blockLoadTime += elapsedTime; - tsdbDebug("%p load file block into buffer, brange:%"PRId64"-%"PRId64" , rows:%d, elapsed time:%"PRId64 " us", - pQueryHandle, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime); + tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64" , rows:%d, elapsed time:%"PRId64 " us, %p", + pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qinfo); return blockLoaded; } @@ -681,8 +681,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* // do not load file block into buffer int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, - pQueryHandle->outputCapacity, &cur->win, pQueryHandle); + cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); pQueryHandle->realNumOfRows = cur->rows; // update the last key value @@ -696,7 +695,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* return; } - doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); + doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { /* @@ -723,7 +722,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock if (ASCENDING_TRAVERSE(pQueryHandle->order)) { // query ended in/started from current block if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { - if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { + if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) { return false; } @@ -744,7 +743,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock } } else { //desc order, query ended in current block if (pQueryHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { - if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { + if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) { return false; } @@ -1859,8 +1858,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int } int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d", pQueryHandle, - elapsedTime, numOfRows, numOfCols); + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, + elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); return numOfRows; } @@ -1975,7 +1974,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { return pHandle->pColumns; } else { // only load the file block SCompBlock* pBlock = pBlockInfo->compBlock; - doLoadFileDataBlock(pHandle, pBlock, pCheckInfo); + doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot); // todo refactor int32_t numOfRows = copyDataFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1); From b472a553830a89735b5383d277e074fe9119847c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jul 2020 14:50:25 +0800 Subject: [PATCH 45/63] [td-225] destory qhandle. --- src/vnode/src/vnodeRead.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 83986de682..2586a2db99 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -236,6 +236,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { freeHandle = false; } else { qKillQuery(*handle); + qDestroyQueryInfo(*handle); freeHandle = true; } } From 4f3fd4e034f30b584f150355624b17d4818c977d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 23 Jul 2020 15:24:28 +0800 Subject: [PATCH 46/63] add cassandra test script. --- .../cassandraTestWriteLoop.sh | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 tests/perftest-scripts/cassandraTestWriteLoop.sh diff --git a/tests/perftest-scripts/cassandraTestWriteLoop.sh b/tests/perftest-scripts/cassandraTestWriteLoop.sh new file mode 100755 index 0000000000..4e42c4b3df --- /dev/null +++ b/tests/perftest-scripts/cassandraTestWriteLoop.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +DATA_DIR=/mnt/root/testdata +NUM_LOOP=1 +NUM_OF_FILES=100 +OUT_FILE=cassandraWrite.out + +rowsPerRequest=(1 10 50 100 500 1000 2000) + +function printTo { + if $verbose ; then + echo $1 + fi +} + +function runTest { + for c in `seq 1 $clients`; do + avgRPR[$c]=0 + done + + printf "R/R, " + for c in `seq 1 $clients`; do + if [ "$c" == "1" ]; then + printf "$c client, " + else + printf "$c clients, " + fi + done + printf "\n" + + for r in ${rowsPerRequest[@]}; do + printf "$r, " + for c in `seq 1 $clients`; do + totalRPR=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -datadir $DATA_DIR \ + -numofFiles $NUM_OF_FILES \ + -rowsperrequest $r \ + -writeclients $c \ + -conf $CAS_TEST_DIR/application.conf" + java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -datadir $DATA_DIR \ + -numofFiles $NUM_OF_FILES \ + -rowsperrequest $r \ + -writeclients $c \ + -conf $CAS_TEST_DIR/application.conf \ + 2>&1 > $OUT_FILE + RPR=`cat $OUT_FILE | grep "insertation speed:" | awk '{print $(NF-1)}'` + totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` + printTo "rows:$r, clients:$c, i:$i RPR:$RPR" + done + avgRPR[$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc` + done + for c in `seq 1 $clients`; do + printf "${avgRPR[$c]}, " + done + printf "\n" + done +} + +################ Main ################ + +verbose=false +clients=1 + +while : ; do + case $1 in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + *) + break ;; + esac +done + +printTo "Cassandra Test begin.." + +WORK_DIR=/mnt/root/TDengine +CAS_TEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra + +runTest + +printTo "Cassandra Test done!" From f606318248f50de0270fbcff0f13245fa4457731 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 23 Jul 2020 15:57:06 +0800 Subject: [PATCH 47/63] TD-987 --- src/inc/tsdb.h | 2 - src/kit/taosmigrate/taosmigrateVnodeCfg.c | 26 ++++--- src/tsdb/inc/tsdbMain.h | 8 ++- src/tsdb/src/tsdbFile.c | 2 +- src/tsdb/src/tsdbMain.c | 79 ++++++--------------- src/tsdb/src/tsdbMemTable.c | 84 +++++++++++++++++------ src/tsdb/src/tsdbMeta.c | 43 +++++++++--- src/tsdb/src/tsdbRead.c | 21 +++--- src/vnode/src/vnodeMain.c | 30 ++++---- 9 files changed, 162 insertions(+), 133 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index a2258dcfd8..bab17322f0 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -53,14 +53,12 @@ typedef struct { int32_t tsdbId; int32_t cacheBlockSize; int32_t totalBlocks; - int32_t maxTables; // maximum number of tables this repository can have int32_t daysPerFile; // day per file sharding policy int32_t keep; // day of data to keep int32_t keep1; int32_t keep2; int32_t minRowsPerFileBlock; // minimum rows per file block int32_t maxRowsPerFileBlock; // maximum rows per file block - int32_t commitTime; int8_t precision; int8_t compression; } STsdbCfg; diff --git a/src/kit/taosmigrate/taosmigrateVnodeCfg.c b/src/kit/taosmigrate/taosmigrateVnodeCfg.c index e830ef78f5..e80e687f02 100644 --- a/src/kit/taosmigrate/taosmigrateVnodeCfg.c +++ b/src/kit/taosmigrate/taosmigrateVnodeCfg.c @@ -37,14 +37,12 @@ static int32_t saveVnodeCfg(SVnodeObj *pVnode, char* cfgFile) len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnode->cfgVersion); len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnode->tsdbCfg.cacheBlockSize); len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnode->tsdbCfg.totalBlocks); - len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnode->tsdbCfg.maxTables); len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnode->tsdbCfg.daysPerFile); len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnode->tsdbCfg.keep); len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnode->tsdbCfg.keep1); len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnode->tsdbCfg.keep2); len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnode->tsdbCfg.minRowsPerFileBlock); len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnode->tsdbCfg.maxRowsPerFileBlock); - len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnode->tsdbCfg.commitTime); len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnode->tsdbCfg.precision); len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnode->tsdbCfg.compression); len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pVnode->walCfg.walLevel); @@ -136,12 +134,12 @@ static int32_t readVnodeCfg(SVnodeObj *pVnode, char* cfgFile) } pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint; - cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); - if (!maxTables || maxTables->type != cJSON_Number) { - printf("vgId:%d, failed to read vnode cfg, maxTables not found\n", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.maxTables = maxTables->valueint; + // cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); + // if (!maxTables || maxTables->type != cJSON_Number) { + // printf("vgId:%d, failed to read vnode cfg, maxTables not found\n", pVnode->vgId); + // goto PARSE_OVER; + // } + // pVnode->tsdbCfg.maxTables = maxTables->valueint; cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { @@ -185,12 +183,12 @@ static int32_t readVnodeCfg(SVnodeObj *pVnode, char* cfgFile) } pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; - cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); - if (!commitTime || commitTime->type != cJSON_Number) { - printf("vgId:%d, failed to read vnode cfg, commitTime not found\n", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; + // cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); + // if (!commitTime || commitTime->type != cJSON_Number) { + // printf("vgId:%d, failed to read vnode cfg, commitTime not found\n", pVnode->vgId); + // goto PARSE_OVER; + // } + // pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; cJSON *precision = cJSON_GetObjectItem(root, "precision"); if (!precision || precision->type != cJSON_Number) { diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 762d2253e2..3e92c01765 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -70,6 +70,7 @@ typedef struct { pthread_rwlock_t rwLock; int32_t nTables; + int32_t maxTables; STable** tables; SList* superList; SHashObj* uidMap; @@ -111,9 +112,11 @@ typedef struct { typedef struct { T_REF_DECLARE(); + SRWLatch latch; TSKEY keyFirst; TSKEY keyLast; int64_t numOfRows; + int32_t maxTables; STableData** tData; SList* actList; SList* bufBlockList; @@ -304,6 +307,7 @@ typedef struct { // Operations // ------------------ tsdbMeta.c +#define TSDB_INIT_NTABLES 1024 #define TABLE_TYPE(t) (t)->type #define TABLE_NAME(t) (t)->name #define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data @@ -395,6 +399,7 @@ int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); +void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem); void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, @@ -429,7 +434,7 @@ STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg); void tsdbFreeFileH(STsdbFileH* pFileH); int tsdbOpenFileH(STsdbRepo* pRepo); void tsdbCloseFileH(STsdbRepo* pRepo); -SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid, int maxTables); +SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid); void tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction); void tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid); SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter); @@ -511,6 +516,7 @@ void tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type, char* fname int tsdbLockRepo(STsdbRepo* pRepo); int tsdbUnlockRepo(STsdbRepo* pRepo); char* tsdbGetDataDirName(char* rootDir); +int tsdbGetNextMaxTables(int tid); STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo); STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 767fbc8252..f698878698 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -149,7 +149,7 @@ void tsdbCloseFileH(STsdbRepo *pRepo) { } } -SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int maxTables) { +SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) { STsdbFileH *pFileH = pRepo->tsdbFileH; if (pFileH->nFGroups >= pFileH->maxFGroups) return NULL; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 772bcf48d6..11a84cd552 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -62,7 +62,6 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo); static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression); static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep); -static int tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables); static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); static int keyFGroupCompFunc(const void *key, const void *fgroup); static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg); @@ -85,10 +84,10 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { if (tsdbSetRepoEnv(rootDir, pCfg) < 0) return -1; tsdbDebug( - "vgId:%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d maxTables %d daysPerFile %d keep " + "vgId:%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d daysPerFile %d keep " "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d", - pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep, - pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); + pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock, + pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); return 0; } @@ -307,13 +306,6 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); configChanged = true; } - if (pRCfg->maxTables != pCfg->maxTables) { - if (tsdbAlterMaxTables(pRepo, pCfg->maxTables) < 0) { - tsdbError("vgId:%d failed to configure repo when alter maxTables since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - configChanged = true; - } if (configChanged) { if (tsdbSaveConfig(pRepo->rootDir, &pRepo->config) < 0) { @@ -385,6 +377,18 @@ char *tsdbGetDataDirName(char *rootDir) { return fname; } +int tsdbGetNextMaxTables(int tid) { + ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES); + int maxTables = TSDB_INIT_NTABLES; + while (true) { + maxTables = MIN(maxTables, TSDB_MAX_TABLES); + if (tid <= maxTables + 1) break; + maxTables *= 2; + } + + return maxTables + 1; +} + STsdbMeta * tsdbGetMeta(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbMeta; } STsdbFileH * tsdbGetFile(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbFileH; } STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { return NULL; } @@ -417,17 +421,6 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { goto _err; } - // Check maxTables - if (pCfg->maxTables == -1) { - pCfg->maxTables = TSDB_DEFAULT_TABLES+1; - } else { - if (pCfg->maxTables - 1 < TSDB_MIN_TABLES || pCfg->maxTables - 1 > TSDB_MAX_TABLES) { - tsdbError("vgId:%d invalid maxTables configuration! maxTables %d TSDB_MIN_TABLES %d TSDB_MAX_TABLES %d", - pCfg->tsdbId, pCfg->maxTables - 1, TSDB_MIN_TABLES, TSDB_MAX_TABLES); - goto _err; - } - } - // Check daysPerFile if (pCfg->daysPerFile == -1) { pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; @@ -713,6 +706,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY STsdbMeta *pMeta = pRepo->tsdbMeta; int64_t points = 0; + ASSERT(pBlock->tid < pMeta->maxTables); STable *pTable = pMeta->tables[pBlock->tid]; ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid); @@ -779,7 +773,6 @@ static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { } static int tsdbRestoreInfo(STsdbRepo *pRepo) { - // TODO STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; SFileGroup *pFGroup = NULL; @@ -792,7 +785,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_DESC); while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) { if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; - for (int i = 1; i < pRepo->config.maxTables; i++) { + for (int i = 1; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; tsdbSetHelperTable(&rhelper, pTable, pRepo); @@ -868,36 +861,6 @@ static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { return 0; } -static int tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { - // TODO - int oldMaxTables = pRepo->config.maxTables; - if (oldMaxTables < pRepo->config.maxTables) { - terrno = TSDB_CODE_TDB_INVALID_ACTION; - return -1; - } - - STsdbMeta *pMeta = pRepo->tsdbMeta; - - pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *)); - memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables - oldMaxTables)); - pRepo->config.maxTables = maxTables; - - if (pRepo->mem) { - pRepo->mem->tData = realloc(pRepo->mem->tData, maxTables * sizeof(STableData *)); - memset(POINTER_SHIFT(pRepo->mem->tData, sizeof(STableData *) * oldMaxTables), 0, - sizeof(STableData *) * (maxTables - oldMaxTables)); - } - - if (pRepo->imem) { - pRepo->imem->tData = realloc(pRepo->imem->tData, maxTables * sizeof(STableData *)); - memset(POINTER_SHIFT(pRepo->imem->tData, sizeof(STableData *) * oldMaxTables), 0, - sizeof(STableData *) * (maxTables - oldMaxTables)); - } - - tsdbDebug("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); - return 0; -} - static int keyFGroupCompFunc(const void *key, const void *fgroup) { int fid = *(int *)key; SFileGroup *pFGroup = (SFileGroup *)fgroup; @@ -914,7 +877,6 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { tlen += taosEncodeVariantI32(buf, pCfg->tsdbId); tlen += taosEncodeFixedI32(buf, pCfg->cacheBlockSize); tlen += taosEncodeVariantI32(buf, pCfg->totalBlocks); - tlen += taosEncodeVariantI32(buf, pCfg->maxTables); tlen += taosEncodeVariantI32(buf, pCfg->daysPerFile); tlen += taosEncodeVariantI32(buf, pCfg->keep); tlen += taosEncodeVariantI32(buf, pCfg->keep1); @@ -931,7 +893,6 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { buf = taosDecodeVariantI32(buf, &(pCfg->tsdbId)); buf = taosDecodeFixedI32(buf, &(pCfg->cacheBlockSize)); buf = taosDecodeVariantI32(buf, &(pCfg->totalBlocks)); - buf = taosDecodeVariantI32(buf, &(pCfg->maxTables)); buf = taosDecodeVariantI32(buf, &(pCfg->daysPerFile)); buf = taosDecodeVariantI32(buf, &(pCfg->keep)); buf = taosDecodeVariantI32(buf, &(pCfg->keep1)); @@ -1037,7 +998,7 @@ static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) { pBlock->schemaLen = htonl(pBlock->schemaLen); pBlock->numOfRows = htons(pBlock->numOfRows); - if (pBlock->tid <= 0 || pBlock->tid >= pRepo->config.maxTables) { + if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid, pBlock->tid); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; @@ -1120,7 +1081,7 @@ TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) { static void tsdbStartStream(STsdbRepo *pRepo) { STsdbMeta *pMeta = pRepo->tsdbMeta; - for (int i = 0; i < pRepo->config.maxTables; i++) { + for (int i = 0; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->type == TSDB_STREAM_TABLE) { pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql, @@ -1133,7 +1094,7 @@ static void tsdbStartStream(STsdbRepo *pRepo) { static void tsdbStopStream(STsdbRepo *pRepo) { STsdbMeta *pMeta = pRepo->tsdbMeta; - for (int i = 0; i < pRepo->config.maxTables; i++) { + for (int i = 0; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->type == TSDB_STREAM_TABLE) { (*pRepo->appH.cqDropFunc)(pTable->cqhandle); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 7708646a61..9ca28d06cb 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -21,7 +21,7 @@ static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo); static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes); -static SMemTable * tsdbNewMemTable(STsdbCfg *pCfg); +static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); static void tsdbFreeMemTable(SMemTable *pMemTable); static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); static void tsdbFreeTableData(STableData *pTableData); @@ -30,13 +30,15 @@ static void * tsdbCommitData(void *arg); static int tsdbCommitMeta(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo); static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey); -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols); +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols); static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo); static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables); +static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); // ---------------- INTERNAL FUNCTIONS ---------------- int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { STsdbCfg * pCfg = &pRepo->config; + STsdbMeta * pMeta = pRepo->tsdbMeta; int32_t level = 0; int32_t headSize = 0; TSKEY key = dataRowKey(row); @@ -45,7 +47,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { SSkipList * pSList = NULL; int bytes = 0; - if (pMemTable != NULL && pMemTable->tData[TABLE_TID(pTable)] != NULL && + if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL && pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) { pTableData = pMemTable->tData[TABLE_TID(pTable)]; pSList = pTableData->pData; @@ -66,13 +68,20 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { // Operations above may change pRepo->mem, retake those values ASSERT(pRepo->mem != NULL); pMemTable = pRepo->mem; + + if (TABLE_TID(pTable) >= pMemTable->maxTables) { + if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) return -1;; + } pTableData = pMemTable->tData[TABLE_TID(pTable)]; if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) { if (pTableData != NULL) { // destroy the table skiplist (may have race condition problem) + taosWLockLatch(&(pMemTable->latch)); pMemTable->tData[TABLE_TID(pTable)] = NULL; tsdbFreeTableData(pTableData); + taosWUnLockLatch(&(pMemTable->latch)); } + pTableData = tsdbNewTableData(pCfg, pTable); if (pTableData == NULL) { tsdbError("vgId:%d failed to insert row with key %" PRId64 @@ -122,7 +131,6 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { int ref = T_REF_DEC(pMemTable); tsdbDebug("vgId:%d unref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref); if (ref == 0) { - STsdbCfg * pCfg = &pRepo->config; STsdbBufPool *pBufPool = pRepo->pPool; SListNode *pNode = NULL; @@ -139,7 +147,7 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { } if (tsdbUnlockRepo(pRepo) < 0) return -1; - for (int i = 0; i < pCfg->maxTables; i++) { + for (int i = 0; i < pMemTable->maxTables; i++) { if (pMemTable->tData[i] != NULL) { tsdbFreeTableData(pMemTable->tData[i]); } @@ -161,11 +169,24 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) { tsdbRefMemTable(pRepo, *pIMem); if (tsdbUnlockRepo(pRepo) < 0) return -1; - tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), *pMem, *pIMem); + if (*pMem != NULL) taosRLockLatch(&((*pMem)->latch)); + + tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), *pMem, *pIMem); return 0; } +void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem) { + if (pMem != NULL) { + taosRUnLockLatch(&(pMem->latch)); + tsdbUnRefMemTable(pRepo, pMem); + } + + if (pIMem != NULL) { + tsdbUnRefMemTable(pRepo, pIMem); + } +} + void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { STsdbCfg * pCfg = &pRepo->config; STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); @@ -182,7 +203,7 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { } if (pRepo->mem == NULL) { - SMemTable *pMemTable = tsdbNewMemTable(&pRepo->config); + SMemTable *pMemTable = tsdbNewMemTable(pRepo); if (pMemTable == NULL) return NULL; if (tsdbLockRepo(pRepo) < 0) { @@ -329,7 +350,9 @@ static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes) { listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain); } -static SMemTable* tsdbNewMemTable(STsdbCfg* pCfg) { +static SMemTable* tsdbNewMemTable(STsdbRepo *pRepo) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + SMemTable *pMemTable = (SMemTable *)calloc(1, sizeof(*pMemTable)); if (pMemTable == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -340,7 +363,8 @@ static SMemTable* tsdbNewMemTable(STsdbCfg* pCfg) { pMemTable->keyLast = 0; pMemTable->numOfRows = 0; - pMemTable->tData = (STableData**)calloc(pCfg->maxTables, sizeof(STableData*)); + pMemTable->maxTables = pMeta->maxTables; + pMemTable->tData = (STableData **)calloc(pMemTable->maxTables, sizeof(STableData *)); if (pMemTable->tData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -398,9 +422,6 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { goto _err; } - // TODO: operation here should not be here, remove it - pTableData->pData->level = 1; - return pTableData; _err: @@ -473,7 +494,7 @@ static void *tsdbCommitData(void *arg) { _exit: tdFreeDataCols(pDataCols); - tsdbDestroyCommitIters(iters, pCfg->maxTables); + tsdbDestroyCommitIters(iters, pMem->maxTables); tsdbDestroyHelper(&whelper); tsdbEndCommit(pRepo); tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId); @@ -552,12 +573,13 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe STsdbCfg * pCfg = &pRepo->config; STsdbFileH *pFileH = pRepo->tsdbFileH; SFileGroup *pGroup = NULL; + SMemTable * pMem = pRepo->imem; TSKEY minKey = 0, maxKey = 0; tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); // Check if there are data to commit to this file - int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); + int hasDataToCommit = tsdbHasDataToCommit(iters, pMem->maxTables, minKey, maxKey); if (!hasDataToCommit) { tsdbDebug("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid); return 0; @@ -570,7 +592,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe return -1; } - if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid, pCfg->maxTables)) == NULL) { + if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid)) == NULL) { tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); goto _err; } @@ -582,7 +604,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe } // Loop to commit data in each table - for (int tid = 1; tid < pCfg->maxTables; tid++) { + for (int tid = 1; tid < pMem->maxTables; tid++) { SCommitIter *pIter = iters + tid; if (pIter->pTable == NULL) continue; @@ -643,11 +665,10 @@ _err: } static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) { - STsdbCfg * pCfg = &(pRepo->config); SMemTable *pMem = pRepo->imem; STsdbMeta *pMeta = pRepo->tsdbMeta; - SCommitIter *iters = (SCommitIter *)calloc(pCfg->maxTables, sizeof(SCommitIter)); + SCommitIter *iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter)); if (iters == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; @@ -656,7 +677,7 @@ static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) { if (tsdbRLockRepoMeta(pRepo) < 0) goto _err; // reference all tables - for (int i = 0; i < pCfg->maxTables; i++) { + for (int i = 0; i < pMem->maxTables; i++) { if (pMeta->tables[i] != NULL) { tsdbRefTable(pMeta->tables[i]); iters[i].pTable = pMeta->tables[i]; @@ -665,7 +686,7 @@ static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) { if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err; - for (int i = 0; i < pCfg->maxTables; i++) { + for (int i = 0; i < pMem->maxTables; i++) { if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) { if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -679,7 +700,7 @@ static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) { return iters; _err: - tsdbDestroyCommitIters(iters, pCfg->maxTables); + tsdbDestroyCommitIters(iters, pMem->maxTables); return NULL; } @@ -694,4 +715,25 @@ static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) { } free(iters); +} + +static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { + ASSERT(pMemTable->maxTables < maxTables); + + STableData **pTableData = (STableData **)calloc(maxTables, sizeof(STableData *)); + if (pTableData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + STableData **tData = pMemTable->tData; + + taosWLockLatch(&(pMemTable->latch)); + pMemTable->maxTables = maxTables; + pMemTable->tData = pTableData; + taosWUnLockLatch(&(pMemTable->latch)); + + tfree(tData); + + return 0; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index b25e734694..8d237ab673 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -49,6 +49,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable); static void * tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STable *pTable); static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable); static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable); +static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid); // ------------------ OUTER FUNCTIONS ------------------ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { @@ -60,13 +61,13 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { int tid = pCfg->tableId.tid; STable * pTable = NULL; - if (tid < 0 || tid >= pRepo->config.maxTables) { + if (tid < 1 || tid > TSDB_MAX_TABLES) { tsdbError("vgId:%d failed to create table since invalid tid %d", REPO_ID(pRepo), tid); terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO; goto _err; } - if (pMeta->tables[tid] != NULL) { + if (tid < pMeta->maxTables && pMeta->tables[tid] != NULL) { if (TABLE_UID(pMeta->tables[tid]) == pCfg->tableId.uid) { tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); @@ -422,7 +423,8 @@ STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { goto _err; } - pMeta->tables = (STable **)calloc(pCfg->maxTables, sizeof(STable *)); + pMeta->maxTables = TSDB_INIT_NTABLES + 1; + pMeta->tables = (STable **)calloc(pMeta->maxTables, sizeof(STable *)); if (pMeta->tables == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -434,7 +436,7 @@ STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { goto _err; } - pMeta->uidMap = taosHashInit(pCfg->maxTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + pMeta->uidMap = taosHashInit(TSDB_INIT_NTABLES * 1.1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); if (pMeta->uidMap == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -484,14 +486,13 @@ _err: } int tsdbCloseMeta(STsdbRepo *pRepo) { - STsdbCfg * pCfg = &pRepo->config; STsdbMeta *pMeta = pRepo->tsdbMeta; SListNode *pNode = NULL; STable * pTable = NULL; if (pMeta == NULL) return 0; tdCloseKVStore(pMeta->pStore); - for (int i = 1; i < pCfg->maxTables; i++) { + for (int i = 1; i < pMeta->maxTables; i++) { tsdbFreeTable(pMeta->tables[i]); } @@ -624,9 +625,8 @@ static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { static void tsdbOrgMeta(void *pHandle) { STsdbRepo *pRepo = (STsdbRepo *)pHandle; STsdbMeta *pMeta = pRepo->tsdbMeta; - STsdbCfg * pCfg = &pRepo->config; - for (int i = 1; i < pCfg->maxTables; i++) { + for (int i = 1; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable != NULL && pTable->type == TSDB_CHILD_TABLE) { tsdbAddTableIntoIndex(pMeta, pTable, true); @@ -781,6 +781,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo goto _err; } } else { + if (tsdbAdjustMetaTables(pRepo, TABLE_TID(pTable)) < 0) goto _err; if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index if (tsdbAddTableIntoIndex(pMeta, pTable, true) < 0) { tsdbDebug("vgId:%d failed to add table %s to meta while add table to index since %s", REPO_ID(pRepo), @@ -827,7 +828,6 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro SListIter lIter = {0}; SListNode *pNode = NULL; STable * tTable = NULL; - STsdbCfg * pCfg = &(pRepo->config); STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); int maxCols = schemaNCols(pSchema); @@ -860,7 +860,7 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro if (maxCols == pMeta->maxCols || maxRowBytes == pMeta->maxRowBytes) { maxCols = 0; maxRowBytes = 0; - for (int i = 0; i < pCfg->maxTables; i++) { + for (int i = 0; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable != NULL) { pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); @@ -1266,5 +1266,28 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) { tsdbRemoveTableFromMeta(pRepo, pTable, true, true); } + return 0; +} + +static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + if (pMeta->maxTables >= tid) return 0; + + int maxTables = tsdbGetNextMaxTables(tid); + + STable **tables = (STable **)calloc(maxTables, sizeof(STable *)); + if (tables == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + memcpy((void *)tables, (void *)pMeta->tables, sizeof(STable *) * pMeta->maxTables); + pMeta->maxTables = maxTables; + + STable **tTables = pMeta->tables; + pMeta->tables = tables; + tfree(tTables); + tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables); + return 0; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c82d1f905a..577a0965d8 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -317,17 +317,20 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh } assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); - - if (pHandle->mem && pHandle->mem->tData[pCheckInfo->tableId.tid] != NULL) { + + // TODO: add uid check + if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables && + pHandle->mem->tData[pCheckInfo->tableId.tid] != NULL) { pCheckInfo->iter = tSkipListCreateIterFromVal(pHandle->mem->tData[pCheckInfo->tableId.tid]->pData, - (const char*) &pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } - - if (pHandle->imem && pHandle->imem->tData[pCheckInfo->tableId.tid] != NULL) { + + if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables && + pHandle->imem->tData[pCheckInfo->tableId.tid] != NULL) { pCheckInfo->iiter = tSkipListCreateIterFromVal(pHandle->imem->tData[pCheckInfo->tableId.tid]->pData, - (const char*) &pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } - + // both iterators are NULL, no data in buffer right now if (pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL) { return false; @@ -1529,7 +1532,6 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - assert(numOfTables <= ((STsdbRepo*)pQueryHandle->pTsdb)->config.maxTables); while (pQueryHandle->activeIndex < numOfTables) { if (hasMoreDataInCache(pQueryHandle)) { @@ -2418,8 +2420,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tfree(pQueryHandle->statis); // todo check error - tsdbUnRefMemTable(pQueryHandle->pTsdb, pQueryHandle->mem); - tsdbUnRefMemTable(pQueryHandle->pTsdb, pQueryHandle->imem); + tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pQueryHandle->mem, pQueryHandle->imem); tsdbDestroyHelper(&pQueryHandle->rhelper); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 249fb428e7..dd5ec18439 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -123,7 +123,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize; tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks; - tsdbCfg.maxTables = pVnodeCfg->cfg.maxTables; + // tsdbCfg.maxTables = pVnodeCfg->cfg.maxTables; tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; @@ -630,14 +630,14 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion); len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize); len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks); - len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables); + // len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables); len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnodeCfg->cfg.daysPerFile); len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnodeCfg->cfg.daysToKeep1); len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnodeCfg->cfg.daysToKeep2); len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.minRowsPerFileBlock); len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.maxRowsPerFileBlock); - len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime); + // len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime); len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pVnodeCfg->cfg.walLevel); @@ -729,12 +729,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint; - cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); - if (!maxTables || maxTables->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, maxTables not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.maxTables = maxTables->valueint; + // cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); + // if (!maxTables || maxTables->type != cJSON_Number) { + // vError("vgId:%d, failed to read vnode cfg, maxTables not found", pVnode->vgId); + // goto PARSE_OVER; + // } + // pVnode->tsdbCfg.maxTables = maxTables->valueint; cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { @@ -778,12 +778,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; - cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); - if (!commitTime || commitTime->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, commitTime not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; + // cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); + // if (!commitTime || commitTime->type != cJSON_Number) { + // vError("vgId:%d, failed to read vnode cfg, commitTime not found", pVnode->vgId); + // goto PARSE_OVER; + // } + // pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; cJSON *precision = cJSON_GetObjectItem(root, "precision"); if (!precision || precision->type != cJSON_Number) { From 218e83f4ee1054959aed97cc0444865403b55638 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 23 Jul 2020 16:02:28 +0800 Subject: [PATCH 48/63] fix TSDB test file --- src/tsdb/tests/tsdbTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index 88bd524c20..5e678a8ee5 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -98,7 +98,7 @@ static void tsdbSetCfg(STsdbCfg *pCfg, int32_t tsdbId, int32_t cacheBlockSize, i pCfg->tsdbId = tsdbId; pCfg->cacheBlockSize = cacheBlockSize; pCfg->totalBlocks = totalBlocks; - pCfg->maxTables = maxTables; + // pCfg->maxTables = maxTables; pCfg->daysPerFile = daysPerFile; pCfg->keep = keep; pCfg->minRowsPerFileBlock = minRows; From da88d370db238662dfeed1d3788efb1bd2153c60 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jul 2020 17:51:06 +0800 Subject: [PATCH 49/63] [td-225] update test scripts. --- src/query/inc/qExtbuffer.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h index b57c48933f..aa30de4d0f 100644 --- a/src/query/inc/qExtbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -19,7 +19,6 @@ extern "C" { #endif - #include "os.h" #include "taosmsg.h" @@ -28,9 +27,9 @@ extern "C" { #include "tdataformat.h" #include "talgo.h" -#define DEFAULT_PAGE_SIZE (1024L*4) // 16k larger than the SHistoInfo #define MAX_TMPFILE_PATH_LENGTH PATH_MAX -#define INITIAL_ALLOCATION_BUFFER_SIZE 64 +#define INITIAL_ALLOCATION_BUFFER_SIZE 1L +#define DEFAULT_PAGE_SIZE (4096L*(INITIAL_ALLOCATION_BUFFER_SIZE)) // 16k larger than the SHistoInfo typedef enum EXT_BUFFER_FLUSH_MODEL { /* From dc8e7ebc8257755ff7149bcff2be3098e93295c2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 23 Jul 2020 17:56:35 +0800 Subject: [PATCH 50/63] fix invalid write --- src/tsdb/src/tsdbMain.c | 2 +- src/tsdb/src/tsdbMeta.c | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 11a84cd552..89a19c15af 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -382,7 +382,7 @@ int tsdbGetNextMaxTables(int tid) { int maxTables = TSDB_INIT_NTABLES; while (true) { maxTables = MIN(maxTables, TSDB_MAX_TABLES); - if (tid <= maxTables + 1) break; + if (tid <= maxTables) break; maxTables *= 2; } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 8d237ab673..684c87462c 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -781,7 +781,9 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo goto _err; } } else { - if (tsdbAdjustMetaTables(pRepo, TABLE_TID(pTable)) < 0) goto _err; + if (TABLE_TID(pTable) >= pMeta->maxTables) { + if (tsdbAdjustMetaTables(pRepo, TABLE_TID(pTable)) < 0) goto _err; + } if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index if (tsdbAddTableIntoIndex(pMeta, pTable, true) < 0) { tsdbDebug("vgId:%d failed to add table %s to meta while add table to index since %s", REPO_ID(pRepo), @@ -789,6 +791,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo goto _err; } } + ASSERT(TABLE_TID(pTable) < pMeta->maxTables); pMeta->tables[TABLE_TID(pTable)] = pTable; pMeta->nTables++; } @@ -1271,7 +1274,7 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) { static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { STsdbMeta *pMeta = pRepo->tsdbMeta; - if (pMeta->maxTables >= tid) return 0; + ASSERT(tid >= pMeta->maxTables); int maxTables = tsdbGetNextMaxTables(tid); From 42288d325f0f125d1044f97e4c7d85efa361c708 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 23 Jul 2020 18:25:18 +0800 Subject: [PATCH 51/63] add case to test minTablesPerVnode.py case. [TD-988] --- tests/pytest/fulltest.sh | 3 +- tests/pytest/test.sh | 23 +++++ tests/pytest/testMinTablesPerVnode.py | 131 ++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100755 tests/pytest/test.sh create mode 100644 tests/pytest/testMinTablesPerVnode.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 904e712fa4..cbe82b2c8c 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -162,6 +162,7 @@ python3 ./test.py -f client/client.py # Misc python3 testCompress.py python3 testNoCompress.py +python3 testMinTablesPerVnode.py # functions python3 ./test.py -f functions/function_avg.py @@ -180,4 +181,4 @@ python3 ./test.py -f functions/function_spread.py python3 ./test.py -f functions/function_stddev.py python3 ./test.py -f functions/function_sum.py python3 ./test.py -f functions/function_top.py -python3 ./test.py -f functions/function_twa.py \ No newline at end of file +python3 ./test.py -f functions/function_twa.py diff --git a/tests/pytest/test.sh b/tests/pytest/test.sh new file mode 100755 index 0000000000..fbb9ba9879 --- /dev/null +++ b/tests/pytest/test.sh @@ -0,0 +1,23 @@ +EXEC_DIR=`dirname "$0"` +if [[ $EXEC_DIR != "." ]] +then + echo "ERROR: Please execute `basename "$0"` in its own directory (for now anyway, pardon the dust)" + exit -1 +fi +CURR_DIR=`pwd` +IN_TDINTERNAL="community" +if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + TAOS_DIR=$CURR_DIR/../../.. +else + TAOS_DIR=$CURR_DIR/../.. +fi +TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1` +LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib +export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3 +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR + +if [[ "$1" == *"test.py"* ]]; then + python3 ./test.py $@ +else + python3 $1 $@ +fi diff --git a/tests/pytest/testMinTablesPerVnode.py b/tests/pytest/testMinTablesPerVnode.py new file mode 100644 index 0000000000..91cea833e7 --- /dev/null +++ b/tests/pytest/testMinTablesPerVnode.py @@ -0,0 +1,131 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/linux/python2/ + +# -*- coding: utf-8 -*- +import sys +import getopt +import subprocess +from distutils.log import warn as printf + +from util.log import * +from util.dnodes import * +from util.cases import * +from util.sql import * + +import taos + + +if __name__ == "__main__": + fileName = "all" + deployPath = "" + testCluster = False + valgrind = 0 + logSql = True + stop = 0 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'l:sgh', [ + 'logSql', 'stop', 'valgrind', 'help']) + for key, value in opts: + if key in ['-h', '--help']: + tdLog.printNoPrefix( + 'A collection of test cases written using Python') + tdLog.printNoPrefix('-l logSql Flag') + tdLog.printNoPrefix('-s stop All dnodes') + tdLog.printNoPrefix('-g valgrind Test Flag') + sys.exit(0) + + if key in ['-l', '--logSql']: + if (value.upper() == "TRUE"): + logSql = True + elif (value.upper() == "FALSE"): + logSql = False + else: + tdLog.printNoPrefix("logSql value %s is invalid" % logSql) + sys.exit(0) + + if key in ['-g', '--valgrind']: + valgrind = 1 + + if key in ['-s', '--stop']: + stop = 1 + + if (stop != 0): + if (valgrind == 0): + toBeKilled = "taosd" + else: + toBeKilled = "valgrind.bin" + + killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -HUP > /dev/null 2>&1" % toBeKilled + + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output(psCmd, shell=True) + + while(processID): + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output(psCmd, shell=True) + + for port in range(6030, 6041): + usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port + processID = subprocess.check_output(usePortPID, shell=True) + + if processID: + killCmd = "kill -TERM %s" % processID + os.system(killCmd) + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if valgrind: + time.sleep(2) + + tdLog.info('stop All dnodes') + sys.exit(0) + + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + + tdDnodes.stopAll() + tdDnodes.addSimExtraCfg("minTablesPerVnode", "100") + tdDnodes.deploy(1) + tdDnodes.start(1) + + host = '127.0.0.1' + + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + + tdCases.logSql(logSql) + + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + + tdSql.init(conn.cursor(), True) + + tdSql.execute("DROP DATABASE IF EXISTS db") + tdSql.execute("CREATE DATABASE IF NOT EXISTS db") + tdSql.execute("USE db") + + for i in range(0, 100): + tdSql.execute( + "CREATE TABLE IF NOT EXISTS tb%d (ts TIMESTAMP, temperature INT, humidity FLOAT)" % i) + + for i in range(1, 6): + tdSql.execute("INSERT INTO tb99 values (now + %da, %d, %f)" % (i, i, i * 1.0)) + + tdSql.execute("DROP TABLE tb99") + tdSql.execute( + "CREATE TABLE IF NOT EXISTS tb99 (ts TIMESTAMP, temperature INT, humidity FLOAT)") + tdSql.query("SELECT * FROM tb99") + tdSql.checkRows(0) + + conn.close() From c5adbd69897ff7efad29e9a36652527130ecb7de Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 23 Jul 2020 18:25:56 +0800 Subject: [PATCH 52/63] fix bug --- src/tsdb/src/tsdbMemTable.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 9ca28d06cb..a65d956eb0 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -725,6 +725,7 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } + memcpy((void *)pTableData, (void *)pMemTable->tData, sizeof(STableData *) * pMemTable->maxTables); STableData **tData = pMemTable->tData; From 26bbbe221d0dcacda58b0d6d314278efecbcca49 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 24 Jul 2020 03:02:18 +0000 Subject: [PATCH 53/63] add query not ready msg --- src/dnode/src/dnodeVRead.c | 2 +- src/inc/taoserror.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index bbea1a5e0b..ebbad525dd 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -206,7 +206,7 @@ static void *dnodeProcessReadQueue(void *param) { taosMsg[pReadMsg->rpcMsg.msgType], type); int32_t code = vnodeProcessRead(pVnode, pReadMsg); - if (type == TAOS_QTYPE_RPC) { + if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { dnodeSendRpcReadRsp(pVnode, pReadMsg, code); } else { dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 59b2c0220b..5a49f24804 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -210,6 +210,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, 0, 0x0703, "query out TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, 0, 0x0704, "query app error") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, 0, 0x0705, "query duplicated join key") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "query tag conditon too many") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "query not ready") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "grant expired") From 64acc4e4f3eab7b6f867153c71716e61064cfd0f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 24 Jul 2020 11:46:11 +0800 Subject: [PATCH 54/63] update a offset --- src/tsdb/src/tsdbRWHelper.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 3063f22a28..27f95f31de 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1231,8 +1231,8 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl if (tsdbCheckAndDecodeColumnData(pDataCol, pHelper->pBuffer, pCompCol->len, pCompBlock->algorithm, pCompBlock->numOfRows, pHelper->pRepo->config.maxRowsPerFileBlock, pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0) { - tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname, pCompCol->colId, - (int64_t)pCompCol->offset); + tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname, + pCompCol->colId, offset); return -1; } From 1a9fce812657ef47ac7323330b338bb1d086f013 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 24 Jul 2020 14:38:57 +0800 Subject: [PATCH 55/63] [td-225] avoid the retrieved thread being blocked. --- src/dnode/src/dnodeVRead.c | 6 ++- src/inc/query.h | 11 +++-- src/inc/taoserror.h | 1 + src/query/inc/qExecutor.h | 14 ++++-- src/query/src/qExecutor.c | 95 ++++++++++++++++++++++++++------------ src/vnode/src/vnodeRead.c | 50 ++++++++++++++------ 6 files changed, 127 insertions(+), 50 deletions(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index ebbad525dd..40b29cab79 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -209,7 +209,11 @@ static void *dnodeProcessReadQueue(void *param) { if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { dnodeSendRpcReadRsp(pVnode, pReadMsg, code); } else { - dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); + if (code == TSDB_CODE_QRY_HAS_RSP) { + dnodeSendRpcReadRsp(pVnode, pReadMsg, code); + } else { + dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); + } } taosFreeQitem(pReadMsg); diff --git a/src/inc/query.h b/src/inc/query.h index d201b649f9..a9a21bc911 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -28,7 +28,7 @@ typedef void* qinfo_t; * @param qinfo * @return */ -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, void* param, qinfo_t* qinfo); +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo); /** @@ -38,7 +38,10 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMs * @param qinfo * @return */ -void qTableQuery(qinfo_t qinfo); +bool qTableQuery(qinfo_t qinfo); + +void* pGetRspMsg(qinfo_t qinfo); + /** * Retrieve the produced results information, if current query is not paused or completed, @@ -48,7 +51,7 @@ void qTableQuery(qinfo_t qinfo); * @param qinfo * @return */ -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo); +int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext); /** * @@ -60,7 +63,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo); * @param contLen payload length * @return */ -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen); +int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); /** * Decide if more results will be produced or not, NOTE: this function will increase the ref count of QInfo, diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 5a49f24804..697ce79715 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -211,6 +211,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, 0, 0x0704, "query app TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, 0, 0x0705, "query duplicated join key") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "query tag conditon too many") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "query not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "query should response") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "grant expired") diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 92fe078c3f..b6f7b3600b 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -177,13 +177,18 @@ typedef struct SQueryRuntimeEnv { SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file } SQueryRuntimeEnv; +enum { + QUERY_RESULT_NOT_READY = 1, + QUERY_RESULT_READY = 2, +}; + typedef struct SQInfo { void* signature; int32_t pointsInterpo; int32_t code; // error code to returned to client - sem_t dataReady; +// sem_t dataReady; + void* tsdb; - void* param; int32_t vgId; STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure @@ -200,8 +205,11 @@ typedef struct SQInfo { */ int32_t tableIndex; int32_t numOfGroupResultPages; - void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; + void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; + pthread_mutex_t lock; // used to synchronize the rsp/query threads + int32_t dataReady; // denote if query result is ready or not + void* rspContext; // response context } SQInfo; #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 532bdeb43b..9e8a5839dd 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5894,16 +5894,11 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, } pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo)); + pQInfo->dataReady = QUERY_RESULT_NOT_READY; + pthread_mutex_init(&pQInfo->lock, NULL); pQuery->pos = -1; pQuery->window = pQueryMsg->window; - - if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { - int32_t code = TAOS_SYSTEM_ERROR(errno); - qError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, tstrerror(code)); - goto _cleanup; - } - colIdCheck(pQuery); qDebug("qmsg:%p QInfo:%p created", pQueryMsg, pQInfo); @@ -5943,7 +5938,7 @@ static bool isValidQInfo(void *param) { return (sig == (uint64_t)pQInfo); } -static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable, void* param) { +static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -5966,8 +5961,6 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ return TSDB_CODE_SUCCESS; } - pQInfo->param = param; - if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo); setQueryStatus(pQuery, QUERY_COMPLETED); @@ -6012,7 +6005,6 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQuery->sdata[col]); } - sem_destroy(&(pQInfo->dataReady)); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { @@ -6164,7 +6156,7 @@ typedef struct SQueryMgmt { pthread_mutex_t lock; } SQueryMgmt; -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, void* param, qinfo_t* pQInfo) { +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo) { assert(pQueryMsg != NULL && tsdb != NULL); int32_t code = TSDB_CODE_SUCCESS; @@ -6260,7 +6252,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo goto _over; } - code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery, param); + code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery); _over: free(tagCond); @@ -6300,26 +6292,32 @@ void qDestroyQueryInfo(qinfo_t qHandle) { freeQInfo(pQInfo); } -void qTableQuery(qinfo_t qinfo) { +static void setQueryResultReady(SQInfo* pQInfo) { + pthread_mutex_lock(&pQInfo->lock); + pQInfo->dataReady = QUERY_RESULT_READY; + pthread_mutex_unlock(&pQInfo->lock); +} + +bool qTableQuery(qinfo_t qinfo) { SQInfo *pQInfo = (SQInfo *)qinfo; if (pQInfo == NULL || pQInfo->signature != pQInfo) { qDebug("QInfo:%p has been freed, no need to execute", pQInfo); - return; + return false; } if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); - sem_post(&pQInfo->dataReady); - return; + setQueryResultReady(pQInfo); + return false; } if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED); + setQueryResultReady(pQInfo); qDebug("QInfo:%p no table exists for query, abort", pQInfo); - sem_post(&pQInfo->dataReady); - return; + return false; } // error occurs, record the error code and return to client @@ -6327,8 +6325,9 @@ void qTableQuery(qinfo_t qinfo) { if (ret != TSDB_CODE_SUCCESS) { pQInfo->code = ret; qDebug("QInfo:%p query abort due to error/cancel occurs, code:%s", pQInfo, tstrerror(pQInfo->code)); - sem_post(&pQInfo->dataReady); - return; + + setQueryResultReady(pQInfo); + return false; } qDebug("QInfo:%p query task is launched", pQInfo); @@ -6353,10 +6352,23 @@ void qTableQuery(qinfo_t qinfo) { pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - sem_post(&pQInfo->dataReady); + taosMsleep(1000); + +// pQInfo->dataReady = QUERY_RESULT_READY; + bool buildRes = false; + pthread_mutex_lock(&pQInfo->lock); + pQInfo->dataReady = QUERY_RESULT_READY; + + if (pQInfo->rspContext != NULL) { + buildRes = true; + } + pthread_mutex_unlock(&pQInfo->lock); + + return buildRes; +// sem_post(&pQInfo->dataReady); } -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { +int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) { SQInfo *pQInfo = (SQInfo *)qinfo; if (pQInfo == NULL || !isValidQInfo(pQInfo)) { @@ -6369,10 +6381,20 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { return pQInfo->code; } - sem_wait(&pQInfo->dataReady); - qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, - pQInfo->code); + *buildRes = false; + pthread_mutex_lock(&pQInfo->lock); + if (pQInfo->dataReady == QUERY_RESULT_READY) { + *buildRes = true; + qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, + pQInfo->code); + } else { + pQInfo->rspContext = pRspContext; + } + + pthread_mutex_unlock(&pQInfo->lock); + +// sem_wait(&pQInfo->dataReady); return pQInfo->code; } @@ -6385,6 +6407,7 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { } SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + bool ret = false; if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { ret = false; @@ -6403,7 +6426,7 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { return ret; } -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen) { +int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) { SQInfo *pQInfo = (SQInfo *)qinfo; if (pQInfo == NULL || !isValidQInfo(pQInfo)) { @@ -6413,8 +6436,10 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; size_t size = getResultSize(pQInfo, &pQuery->rec.rows); + size += sizeof(int32_t); size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo); + *contLen = size + sizeof(SRetrieveTableRsp); // todo proper handle failed to allocate memory, @@ -6423,6 +6448,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co if (*pRsp == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } + (*pRsp)->numOfRows = htonl(pQuery->rec.rows); int32_t code = pQInfo->code; @@ -6430,8 +6456,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co (*pRsp)->offset = htobe64(pQuery->limit.offset); (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } else { - (*pRsp)->offset = 0; (*pRsp)->useconds = 0; + (*pRsp)->offset = 0; } (*pRsp)->precision = htons(pQuery->precision); @@ -6442,10 +6468,21 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co code = pQInfo->code; } + pQInfo->rspContext = NULL; + pQInfo->dataReady = QUERY_RESULT_NOT_READY; + if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { (*pRsp)->completed = 1; // notify no more result to client } + if (qHasMoreResultsToRetrieve(pQInfo)) { + *continueExec = true; + } else { // failed to dump result, free qhandle immediately + *continueExec = false; + qKillQuery(pQInfo); + qDestroyQueryInfo(pQInfo); + } + return code; } @@ -6456,7 +6493,7 @@ int32_t qKillQuery(qinfo_t qinfo) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - sem_post(&pQInfo->dataReady); +// sem_post(&pQInfo->dataReady); setQueryKilled(pQInfo); return TSDB_CODE_SUCCESS; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 2586a2db99..542c132e03 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -66,11 +66,12 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); } -static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void *qhandle) { +static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void *qhandle, void* handle) { SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; pRead->pCont = qhandle; pRead->contLen = 0; + pRead->rpcMsg.handle = handle; atomic_add_fetch_32(&pVnode->refCount, 1); taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead); @@ -110,7 +111,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (contLen != 0) { qinfo_t pQInfo = NULL; - code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, pVnode, &pQInfo); + code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->code = code; @@ -148,7 +149,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (handle != NULL) { vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, register qhandle and return to app", vgId, *handle); - vnodePutItemIntoReadQueue(pVnode, *handle); + vnodePutItemIntoReadQueue(pVnode, *handle, pReadMsg->rpcMsg.handle); // qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } @@ -163,7 +164,23 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } else { vDebug("vgId:%d, QInfo:%p, dnode continue exec query", pVnode->vgId, (void*) pCont); code = TSDB_CODE_VND_ACTION_IN_PROGRESS; - qTableQuery(*handle); // do execute query + bool buildRes = qTableQuery(*handle); // do execute query + + if (buildRes) { // build result rsp + pRet = &pReadMsg->rspRet; + + bool continueExec = false; + if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { + if (continueExec) { + vnodePutItemIntoReadQueue(pVnode, *handle, pReadMsg->rpcMsg.handle); + pRet->qhandle = *handle; + + } + } else { // todo handle error + } + + code = TSDB_CODE_QRY_HAS_RSP; + } } // qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } @@ -223,22 +240,29 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } bool freeHandle = true; - code = qRetrieveQueryResultInfo(*handle); + bool buildRes = false; + + code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg); if (code != TSDB_CODE_SUCCESS) { //TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); - } else { // if failed to dump result, free qhandle immediately - if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len)) == TSDB_CODE_SUCCESS) { - if (qHasMoreResultsToRetrieve(*handle)) { - vnodePutItemIntoReadQueue(pVnode, *handle); + } else { + // result is not ready, return immediately + if (!buildRes) { + return TSDB_CODE_QRY_NOT_READY; + } + + bool continueExec = false; + if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { + if (continueExec) { + vnodePutItemIntoReadQueue(pVnode, *handle, pReadMsg->rpcMsg.handle); pRet->qhandle = *handle; freeHandle = false; - } else { - qKillQuery(*handle); - qDestroyQueryInfo(*handle); - freeHandle = true; } + } else { + pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); + memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); } } From d9a04b397ffe8eada57a3cc87781b971896a4232 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 24 Jul 2020 15:05:44 +0800 Subject: [PATCH 56/63] [td-225] fix bugs in non-blocking processing. --- src/dnode/src/dnodeVRead.c | 2 +- src/query/src/qExecutor.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 40b29cab79..f8a31d568e 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -210,7 +210,7 @@ static void *dnodeProcessReadQueue(void *param) { dnodeSendRpcReadRsp(pVnode, pReadMsg, code); } else { if (code == TSDB_CODE_QRY_HAS_RSP) { - dnodeSendRpcReadRsp(pVnode, pReadMsg, code); + dnodeSendRpcReadRsp(pVnode, pReadMsg, TSDB_CODE_SUCCESS); } else { dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 9e8a5839dd..c4142f5d93 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6352,8 +6352,6 @@ bool qTableQuery(qinfo_t qinfo) { pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - taosMsleep(1000); - // pQInfo->dataReady = QUERY_RESULT_READY; bool buildRes = false; pthread_mutex_lock(&pQInfo->lock); From 699f28284132596e0288ebb2c5d6b0059995e48b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 25 Jul 2020 00:22:46 +0800 Subject: [PATCH 57/63] [td-225] fix bugs in non-blocking processing. --- src/dnode/src/dnodeVRead.c | 2 +- src/query/src/qExecutor.c | 7 +++---- src/vnode/src/vnodeRead.c | 7 +++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index f8a31d568e..85ed194976 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -49,7 +49,7 @@ static taos_qset readQset; int32_t dnodeInitVnodeRead() { readQset = taosOpenQset(); - readPool.min = 2; + readPool.min = 4; readPool.max = tsNumOfCores * tsNumOfThreadsPerCore; if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min; readPool.readWorker = (SReadWorker *)calloc(sizeof(SReadWorker), readPool.max); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c4142f5d93..6de300400e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6352,7 +6352,6 @@ bool qTableQuery(qinfo_t qinfo) { pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } -// pQInfo->dataReady = QUERY_RESULT_READY; bool buildRes = false; pthread_mutex_lock(&pQInfo->lock); pQInfo->dataReady = QUERY_RESULT_READY; @@ -6360,8 +6359,9 @@ bool qTableQuery(qinfo_t qinfo) { if (pQInfo->rspContext != NULL) { buildRes = true; } - pthread_mutex_unlock(&pQInfo->lock); + + pthread_mutex_unlock(&pQInfo->lock); return buildRes; // sem_post(&pQInfo->dataReady); } @@ -6387,12 +6387,11 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, pQInfo->code); } else { + qDebug("QInfo:%p retrieve req set query return result after paused", pQInfo); pQInfo->rspContext = pRspContext; } pthread_mutex_unlock(&pQInfo->lock); - -// sem_wait(&pQInfo->dataReady); return pQInfo->code; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 542c132e03..5edfcf597c 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -163,18 +163,21 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { code = TSDB_CODE_QRY_INVALID_QHANDLE; } else { vDebug("vgId:%d, QInfo:%p, dnode continue exec query", pVnode->vgId, (void*) pCont); - code = TSDB_CODE_VND_ACTION_IN_PROGRESS; bool buildRes = qTableQuery(*handle); // do execute query if (buildRes) { // build result rsp + vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused", pVnode->vgId, *handle); + pRet = &pReadMsg->rspRet; bool continueExec = false; + code = TSDB_CODE_QRY_HAS_RSP; if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { + if (continueExec) { vnodePutItemIntoReadQueue(pVnode, *handle, pReadMsg->rpcMsg.handle); pRet->qhandle = *handle; - + code = TSDB_CODE_SUCCESS; } } else { // todo handle error } From 4ce8f84e7e9f8a0ed7a1a255df6a24bd67b57118 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 25 Jul 2020 10:27:27 +0800 Subject: [PATCH 58/63] [td-225] fix bugs in non-blocking processing. --- src/query/src/qExecutor.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6de300400e..28d1c138f6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6380,6 +6380,8 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex } *buildRes = false; + int32_t code = TSDB_CODE_SUCCESS; + pthread_mutex_lock(&pQInfo->lock); if (pQInfo->dataReady == QUERY_RESULT_READY) { *buildRes = true; @@ -6391,8 +6393,9 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex pQInfo->rspContext = pRspContext; } + code = pQInfo->code; pthread_mutex_unlock(&pQInfo->lock); - return pQInfo->code; + return code; } bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { From 956eafa03165ab59c8f6b84fd416fd981f143320 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jul 2020 06:45:58 +0000 Subject: [PATCH 59/63] update cache expire time --- src/mnode/src/mnodeProfile.c | 4 ++-- src/plugins/http/src/httpContext.c | 2 +- src/util/src/tcache.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 9121f31131..30a292f522 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -68,7 +68,7 @@ int32_t mnodeInitProfile() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_CONN, mnodeProcessKillConnectionMsg); - tsMnodeConnCache = taosCacheInit(TSDB_DATA_TYPE_INT, CONN_CHECK_TIME, false, mnodeFreeConn, "conn"); + tsMnodeConnCache = taosCacheInit(TSDB_DATA_TYPE_INT, CONN_CHECK_TIME, true, mnodeFreeConn, "conn"); return 0; } @@ -119,7 +119,7 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po return NULL; } - if (pConn->ip != ip || pConn->port != port /* || strcmp(pConn->user, user) != 0 */) { + if (/* pConn->ip != ip || */ pConn->port != port /* || strcmp(pConn->user, user) != 0 */) { mError("connId:%d, incoming conn user:%s ip:%s:%u, not match exist conn user:%s ip:%s:%u", connId, user, taosIpStr(ip), port, pConn->user, taosIpStr(pConn->ip), pConn->port); taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false); diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 225977abae..ca65f65608 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -58,7 +58,7 @@ static void httpDestroyContext(void *data) { } bool httpInitContexts() { - tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, false, httpDestroyContext, "restc"); + tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpDestroyContext, "restc"); if (tsHttpServer.contextCache == NULL) { httpError("failed to init context cache"); return false; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 7c2a6b3219..3a3acdc05b 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -343,7 +343,7 @@ void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t ke SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); if (ptNode != NULL) { T_REF_INC(*ptNode); - (*ptNode)->expireTime = taosGetTimestampMs() + (*ptNode)->lifespan; + (*ptNode)->expireTime = expireTime; // taosGetTimestampMs() + (*ptNode)->lifespan; } __cache_unlock(pCacheObj); From d9622d9d8cb52de2343fb0f7137a6ab44d088ea1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Jul 2020 16:17:34 +0800 Subject: [PATCH 60/63] [td-225] fix bugs in rsp data using wrong connection handle --- src/inc/query.h | 9 +-------- src/query/src/qExecutor.c | 8 ++++++++ src/vnode/src/vnodeRead.c | 8 ++++++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/inc/query.h b/src/inc/query.h index a9a21bc911..ccff05bc1b 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -65,14 +65,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex */ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); -/** - * Decide if more results will be produced or not, NOTE: this function will increase the ref count of QInfo, - * so it can be only called once for each retrieve - * - * @param qinfo - * @return - */ -bool qHasMoreResultsToRetrieve(qinfo_t qinfo); +void* qGetResultRetrieveMsg(qinfo_t qinfo); /** * kill current ongoing query and free query handle automatically diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 28d1c138f6..5b904936f2 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6640,6 +6640,13 @@ static void buildTagQueryResult(SQInfo* pQInfo) { setQueryStatus(pQuery, QUERY_COMPLETED); } +void* qGetResultRetrieveMsg(qinfo_t qinfo) { + SQInfo* pQInfo = (SQInfo*) qinfo; + assert(pQInfo != NULL); + + return pQInfo->rspContext; +} + void freeqinfoFn(void *qhandle) { void** handle = qhandle; if (handle == NULL || *handle == NULL) { @@ -6762,3 +6769,4 @@ void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree) { return 0; } + diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 5edfcf597c..3659ef1430 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -166,12 +166,16 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { bool buildRes = qTableQuery(*handle); // do execute query if (buildRes) { // build result rsp - vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused", pVnode->vgId, *handle); + + SReadMsg* pRetrieveMsg = qGetResultRetrieveMsg(*handle); + assert(pRetrieveMsg != NULL); + vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pRetrieveMsg->rpcMsg.handle); + pReadMsg->rpcMsg.handle = pRetrieveMsg->rpcMsg.handle; // update the connection info according to the retrieve connection pRet = &pReadMsg->rspRet; + code = TSDB_CODE_QRY_HAS_RSP; bool continueExec = false; - code = TSDB_CODE_QRY_HAS_RSP; if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { if (continueExec) { From f32a87041316315664aa87a07a70bc44c1b23793 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Jul 2020 16:40:34 +0800 Subject: [PATCH 61/63] [td-225] fix bugs in rsp data using wrong connection handle --- src/vnode/src/vnodeRead.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 3659ef1430..bb7b7c5a80 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -168,7 +168,8 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (buildRes) { // build result rsp SReadMsg* pRetrieveMsg = qGetResultRetrieveMsg(*handle); - assert(pRetrieveMsg != NULL); + assert(pRetrieveMsg != NULL && pRetrieveMsg->rpcMsg.handle != NULL); + vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pRetrieveMsg->rpcMsg.handle); pReadMsg->rpcMsg.handle = pRetrieveMsg->rpcMsg.handle; // update the connection info according to the retrieve connection @@ -203,7 +204,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRetrieve->qhandle = htobe64(pRetrieve->qhandle); pRetrieve->free = htons(pRetrieve->free); - vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, (void*) pRetrieve->qhandle); + vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void*) pRetrieve->qhandle, pRetrieve->free, pReadMsg->rpcMsg.handle); memset(pRet, 0, sizeof(SRspRet)); From d32eb69fdb4f600304555a70fd720519cb2d2656 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Jul 2020 16:45:18 +0800 Subject: [PATCH 62/63] [td-225] fix bugs in rsp data using wrong connection handle --- src/vnode/src/vnodeRead.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index bb7b7c5a80..49c10dca3f 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -167,11 +167,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (buildRes) { // build result rsp - SReadMsg* pRetrieveMsg = qGetResultRetrieveMsg(*handle); - assert(pRetrieveMsg != NULL && pRetrieveMsg->rpcMsg.handle != NULL); + void* retrieveHandle = qGetResultRetrieveMsg(*handle); + assert(retrieveHandle != NULL); - vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pRetrieveMsg->rpcMsg.handle); - pReadMsg->rpcMsg.handle = pRetrieveMsg->rpcMsg.handle; // update the connection info according to the retrieve connection + vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, retrieveHandle); + pReadMsg->rpcMsg.handle = retrieveHandle; // update the connection info according to the retrieve connection pRet = &pReadMsg->rspRet; code = TSDB_CODE_QRY_HAS_RSP; @@ -250,7 +250,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { bool freeHandle = true; bool buildRes = false; - code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg); + code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle); if (code != TSDB_CODE_SUCCESS) { //TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); From ed4fb4ac0822356d601e4c764b4c8ab317333187 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Jul 2020 17:02:21 +0800 Subject: [PATCH 63/63] [td-225] --- src/client/src/tscUtil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 1ef5d3feaf..e7fa2a84a9 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -351,7 +351,7 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { int32_t cmd = pCmd->command; if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - //tscRemoveFromSqlList(pSql); + tscRemoveFromSqlList(pSql); } // pSql->sqlstr will be used by tscBuildQueryStreamDesc @@ -1885,7 +1885,7 @@ void tscDoQuery(SSqlObj* pSql) { } if (pCmd->command == TSDB_SQL_SELECT) { - //tscAddIntoSqlList(pSql); + tscAddIntoSqlList(pSql); } if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) {