From 48f5309e0d2792cad815a4b4103b3e2ef8a7f384 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 3 Sep 2020 12:35:46 +0800 Subject: [PATCH 01/11] remove system-files interface. --- snap/snapcraft.yaml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 7a0e1c3b80..3f5b5f1de0 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -18,14 +18,12 @@ apps: - network - network-bind - system-observe - - systemfiles taos: command: taoswrapper.sh plugs: - network - system-observe - - systemfiles - historyfile taosdemo: @@ -41,18 +39,6 @@ plugs: write: - $HOME/.taos_history - systemfiles: - interface: system-files - read: - - /etc/taos - - /var/lib/taos - - /var/log/taos - - /tmp - write: - - /var/log/taos - - /var/lib/taos - - /tmp - parts: script: plugin: dump @@ -119,4 +105,4 @@ layout: hooks: install: - plugs: [systemfiles, historyfile] + plugs: [historyfile] From 91d7c7748f29af9049d2acce352cdc88ab15e4af Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 3 Sep 2020 17:21:23 +0800 Subject: [PATCH 02/11] refactor part of code --- src/tsdb/inc/tsdbMain.h | 8 +- src/tsdb/src/tsdbFile.c | 73 +++++++++------- src/tsdb/src/tsdbMain.c | 4 +- src/tsdb/src/tsdbRWHelper.c | 168 ++++++++++++++++++++++-------------- 4 files changed, 153 insertions(+), 100 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index e7a86798ee..ede0b33d01 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -45,6 +45,8 @@ extern int tsdbDebugFlag; #define TSDB_FILE_DELIMITER 0xF00AFA0F #define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF +#define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax))) + // NOTE: Any file format change must increase this version number by 1 // Also, implement the convert function #define TSDB_FILE_VERSION ((uint32_t)0) @@ -475,6 +477,7 @@ int tsdbUpdateFileHeader(SFile* pFile); int tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo); void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); +int tsdbLoadFileHeader(SFile* pFile, uint32_t* version); void tsdbGetFileInfoImpl(char* fname, uint32_t* magic, int64_t* size); void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey); @@ -513,7 +516,10 @@ int tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols int tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper); int tsdbWriteCompInfo(SRWHelper* pHelper); int tsdbWriteCompIdx(SRWHelper* pHelper); +int tsdbLoadCompIdxImpl(SFile* pFile, uint32_t offset, uint32_t len, void* buffer); +int tsdbDecodeSCompIdxImpl(void* buffer, uint32_t len, SCompIdx** ppCompIdx, int* numOfIdx); int tsdbLoadCompIdx(SRWHelper* pHelper, void* target); +int tsdbLoadCompInfoImpl(SFile* pFile, SCompIdx* pIdx, SCompInfo** ppCompInfo); int tsdbLoadCompInfo(SRWHelper* pHelper, void* target); int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target); void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols); @@ -537,7 +543,7 @@ static FORCE_INLINE int compTSKEY(const void* key1, const void* key2) { #define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) char* tsdbGetMetaFileName(char* rootDir); -void tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type, char* fname); +void tsdbGetDataFileName(char* rootDir, int vid, int fid, int type, char* fname); int tsdbLockRepo(STsdbRepo* pRepo); int tsdbUnlockRepo(STsdbRepo* pRepo); char* tsdbGetDataDirName(char* rootDir); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 29e46a88af..1009a61e86 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -302,7 +302,7 @@ int tsdbCreateFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { memset((void *)pFile, 0, sizeof(SFile)); pFile->fd = -1; - tsdbGetDataFileName(pRepo, fid, type, pFile->fname); + tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), fid, type, pFile->fname); if (access(pFile->fname, F_OK) == 0) { tsdbError("vgId:%d file %s already exists", REPO_ID(pRepo), pFile->fname); @@ -424,33 +424,57 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { } } -void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int64_t *size) { - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - uint32_t version = 0; - STsdbFileInfo info = {0}; +int tsdbLoadFileHeader(SFile *pFile, uint32_t *version) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - int fd = open(fname, O_RDONLY); - if (fd < 0) goto _err; + if (lseek(pFile->fd, 0, SEEK_SET) < 0) { + tsdbError("failed to lseek file %s to start since %s", pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } - if (taosTRead(fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) goto _err; + if (taosTRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + tsdbError("failed to read file %s header part with %d bytes, reason:%s", pFile->fname, TSDB_FILE_HEAD_SIZE, + strerror(errno)); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } - if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) goto _err; + if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { + tsdbError("file %s header part is corrupted with failed checksum", pFile->fname); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } void *pBuf = (void *)buf; - pBuf = taosDecodeFixedU32(pBuf, &version); - pBuf = tsdbDecodeSFileInfo(pBuf, &info); + pBuf = taosDecodeFixedU32(pBuf, version); + pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); - off_t offset = lseek(fd, 0, SEEK_END); + return 0; +} + +void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int64_t *size) { + uint32_t version = 0; + SFile file = {0}; + SFile * pFile = &file; + + strncpy(pFile->fname, fname, TSDB_FILENAME_LEN); + pFile->fd = -1; + + if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err; + if (tsdbLoadFileHeader(pFile, &version) < 0) goto _err; + + off_t offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) goto _err; - close(fd); + tsdbCloseFile(pFile); - *magic = info.magic; + *magic = pFile->info.magic; *size = offset; return; _err: - if (fd >= 0) close(fd); + tsdbCloseFile(pFile); *magic = TSDB_FILE_INIT_MAGIC; *size = 0; } @@ -458,34 +482,23 @@ _err: // ---------------- LOCAL FUNCTIONS ---------------- static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { uint32_t version; - char buf[512] = "\0"; - tsdbGetDataFileName(pRepo, fid, type, pFile->fname); + tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), fid, type, pFile->fname); pFile->fd = -1; if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err; - if (taosTRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILE_HEAD_SIZE, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + if (tsdbLoadFileHeader(pFile, &version) < 0) { + tsdbError("vgId:%d failed to load file %s header part since %s", REPO_ID(pRepo), pFile->fname, tstrerror(terrno)); goto _err; } - if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { - tsdbError("vgId:%d file %s head part is corrupted", REPO_ID(pRepo), pFile->fname); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - goto _err; - } - - void *pBuf = buf; - pBuf = taosDecodeFixedU32(pBuf, &version); - pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); if (pFile->info.size == TSDB_FILE_HEAD_SIZE) { pFile->info.size = lseek(pFile->fd, 0, SEEK_END); } if (version != TSDB_FILE_VERSION) { + // TODO: deal with error tsdbError("vgId:%d file %s version %u is not the same as program version %u which may cause problem", REPO_ID(pRepo), pFile->fname, version, TSDB_FILE_VERSION); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index ea912ad1f4..294df10edb 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -354,8 +354,8 @@ char *tsdbGetMetaFileName(char *rootDir) { return fname; } -void tsdbGetDataFileName(STsdbRepo *pRepo, int fid, int type, char *fname) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/%s/v%df%d%s", pRepo->rootDir, TSDB_DATA_DIR_NAME, REPO_ID(pRepo), fid, tsdbFileSuffix[type]); +void tsdbGetDataFileName(char *rootDir, int vid, int fid, int type, char *fname) { + snprintf(fname, TSDB_FILENAME_LEN, "%s/%s/v%df%d%s", rootDir, TSDB_DATA_DIR_NAME, vid, fid, tsdbFileSuffix[type]); } int tsdbLockRepo(STsdbRepo *pRepo) { diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 84f22918ec..db24eae148 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -102,7 +102,8 @@ void tsdbResetHelper(SRWHelper *pHelper) { int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { ASSERT(pHelper != NULL && pGroup != NULL); - SFile *pFile = NULL; + SFile * pFile = NULL; + STsdbRepo *pRepo = pHelper->pRepo; // Clear the helper object tsdbResetHelper(pHelper); @@ -112,8 +113,10 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Set the files pHelper->files.fGroup = *pGroup; if (helperType(pHelper) == TSDB_WRITE_HELPER) { - tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, helperNewHeadF(pHelper)->fname); - tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, helperNewLastF(pHelper)->fname); + tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), pGroup->fileId, TSDB_FILE_TYPE_NHEAD, + helperNewHeadF(pHelper)->fname); + tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), pGroup->fileId, TSDB_FILE_TYPE_NLAST, + helperNewLastF(pHelper)->fname); } // Open the files @@ -443,10 +446,64 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { return 0; } +int tsdbLoadCompIdxImpl(SFile *pFile, uint32_t offset, uint32_t len, void *buffer) { + const char *prefixMsg = "failed to load SCompIdx part"; + if (lseek(pFile->fd, offset, SEEK_SET) < 0) { + tsdbError("%s: seek to file %s offset %u failed since %s", prefixMsg, pFile->fname, offset, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (taosTRead(pFile->fd, buffer, len) < len) { + tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, offset, len, + strerror(errno)); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)buffer, len)) { + tsdbError("%s: file %s corrupted, offset %u len %u", prefixMsg, pFile->fname, offset, len); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + + return 0; +} + +int tsdbDecodeSCompIdxImpl(void *buffer, uint32_t len, SCompIdx **ppCompIdx, int *numOfIdx) { + int nIdx = 0; + void *pPtr = buffer; + + while (POINTER_DISTANCE(pPtr, buffer) < (int)(len - sizeof(TSCKSUM))) { + size_t tlen = taosTSizeof(*ppCompIdx); + if (tlen < sizeof(SCompIdx) * (nIdx + 1)) { + *ppCompIdx = (SCompIdx *)taosTRealloc(*ppCompIdx, (tlen == 0) ? 1024 : tlen * 2); + if (*ppCompIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } + + pPtr = tsdbDecodeSCompIdx(pPtr, &((*ppCompIdx)[nIdx])); + if (pPtr == NULL) { + tsdbError("failed to decode SCompIdx part, idx:%d", nIdx); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + + nIdx++; + + ASSERT(nIdx == 1 || (*ppCompIdx)[nIdx - 1].tid > (*ppCompIdx)[nIdx - 2].tid); + ASSERT(POINTER_DISTANCE(pPtr, buffer) <= (int)(len - sizeof(TSCKSUM))); + } + + *numOfIdx = nIdx; + return 0; +} + int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); SFile *pFile = helperHeadF(pHelper); - int fd = pFile->fd; if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object @@ -456,54 +513,18 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { return -1; } - if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + // Load SCompIdx binary from file + if (tsdbLoadCompIdxImpl(pFile, pFile->info.offset, pFile->info.len, (void *)(pHelper->pBuffer)) < 0) { return -1; } - if (taosTRead(fd, (void *)(pHelper->pBuffer), pFile->info.len) < (int)pFile->info.len) { - tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + // Decode the SCompIdx part + if (tsdbDecodeSCompIdxImpl(pHelper->pBuffer, pFile->info.len, &(pHelper->idxH.pIdxArray), + &(pHelper->idxH.numOfIdx)) < 0) { + tsdbError("vgId:%d failed to decode SCompIdx part from file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, + tstrerror(errno)); return -1; } - - if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pBuffer), pFile->info.len)) { - tsdbError("vgId:%d file %s SCompIdx part is corrupted. len %u", REPO_ID(pHelper->pRepo), pFile->fname, - pFile->info.len); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - // Decode it - pHelper->idxH.numOfIdx = 0; - void *ptr = pHelper->pBuffer; - while (POINTER_DISTANCE(ptr, pHelper->pBuffer) < (int)(pFile->info.len - sizeof(TSCKSUM))) { - size_t tlen = taosTSizeof(pHelper->idxH.pIdxArray); - pHelper->idxH.numOfIdx++; - - if (tlen < pHelper->idxH.numOfIdx * sizeof(SCompIdx)) { - pHelper->idxH.pIdxArray = (SCompIdx *)taosTRealloc(pHelper->idxH.pIdxArray, (tlen == 0) ? 1024 : tlen * 2); - if (pHelper->idxH.pIdxArray == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - - ptr = tsdbDecodeSCompIdx(ptr, &(pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 1])); - if (ptr == NULL) { - tsdbError("vgId:%d file %s SCompIdx part is corrupted. len %u", REPO_ID(pHelper->pRepo), pFile->fname, - pFile->info.len); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - ASSERT(pHelper->idxH.numOfIdx == 1 || pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 1].tid > - pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 2].tid); - - ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= (int)(pFile->info.len - sizeof(TSCKSUM))); - } } } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); @@ -515,36 +536,49 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { return 0; } +int tsdbLoadCompInfoImpl(SFile *pFile, SCompIdx *pIdx, SCompInfo **ppCompInfo) { + const char *prefixMsg = "failed to load SCompInfo/SCompBlock part"; + + if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) { + tsdbError("%s: seek to file %s offset %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + *ppCompInfo = taosTRealloc((void *)(*ppCompInfo), pIdx->len); + if (*ppCompInfo == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (taosTRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) { + tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, pIdx->len, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)(*ppCompInfo), pIdx->len)) { + tsdbError("%s: file %s corrupted, offset %u len %u", prefixMsg, pFile->fname, pIdx->offset, pIdx->len); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + + return 0; +} + int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); SCompIdx *pIdx = &(pHelper->curCompIdx); - int fd = helperHeadF(pHelper)->fd; + SFile *pFile = helperHeadF(pHelper); if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { if (pIdx->offset > 0) { ASSERT(pIdx->uid == pHelper->tableInfo.uid); - if (lseek(fd, pIdx->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperHeadF(pHelper)->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pHelper->pCompInfo = taosTRealloc((void *)pHelper->pCompInfo, pIdx->len); - if (taosTRead(fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) { - tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - helperHeadF(pHelper)->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompInfo, pIdx->len)) { - tsdbError("vgId:%d file %s SCompInfo part is corrupted, tid %d uid %" PRIu64, REPO_ID(pHelper->pRepo), - helperHeadF(pHelper)->fname, pHelper->tableInfo.tid, pHelper->tableInfo.uid); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } + if (tsdbLoadCompInfoImpl(pFile, pIdx, &(pHelper->pCompInfo)) < 0) return -1; ASSERT(pIdx->uid == pHelper->pCompInfo->uid && pIdx->tid == pHelper->pCompInfo->tid); } From 139dabf1515dabcf760868a7152b80548025f06e Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 3 Sep 2020 18:32:06 +0800 Subject: [PATCH 03/11] td-804: fix infinite SQL parse retry --- src/client/inc/tsclient.h | 1 + src/client/src/tscAsync.c | 1 + src/client/src/tscParseInsert.c | 12 ++++-------- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 78544b9b99..5f4a46ddad 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -348,6 +348,7 @@ typedef struct SSqlObj { void * pStream; void * pSubscription; char * sqlstr; + char parseRetry; char retry; char maxRetry; SRpcEpSet epSet; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 650f101645..d07089539a 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -43,6 +43,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->signature = pSql; pSql->param = param; pSql->pTscObj = pObj; + pSql->parseRetry= 0; pSql->maxRetry = TSDB_MAX_REPLICA; pSql->fp = fp; pSql->fetchFp = fp; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 7f8fd7f4fe..09eb8f167e 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1335,13 +1335,13 @@ int tsParseSql(SSqlObj *pSql, bool initial) { // make a backup as tsParseInsertSql may modify the string char* sqlstr = strdup(pSql->sqlstr); ret = tsParseInsertSql(pSql); - if (sqlstr == NULL || pSql->retry >= 1 || ret != TSDB_CODE_TSC_INVALID_SQL) { + if (sqlstr == NULL || pSql->parseRetry >= 1 || ret != TSDB_CODE_TSC_INVALID_SQL) { free(sqlstr); } else { tscResetSqlCmdObj(pCmd, true); free(pSql->sqlstr); pSql->sqlstr = sqlstr; - pSql->retry++; + pSql->parseRetry++; if ((ret = tsInsertInitialCheck(pSql)) == TSDB_CODE_SUCCESS) { ret = tsParseInsertSql(pSql); } @@ -1349,18 +1349,14 @@ int tsParseSql(SSqlObj *pSql, bool initial) { } else { SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr); ret = tscToSQLCmd(pSql, &SQLInfo); - if (ret == TSDB_CODE_TSC_INVALID_SQL && pSql->retry == 0 && SQLInfo.type == TSDB_SQL_NULL) { + if (ret == TSDB_CODE_TSC_INVALID_SQL && pSql->parseRetry == 0 && SQLInfo.type == TSDB_SQL_NULL) { tscResetSqlCmdObj(pCmd, true); - pSql->retry++; + pSql->parseRetry++; ret = tscToSQLCmd(pSql, &SQLInfo); } SQLInfoDestroy(&SQLInfo); } - if (ret == TSDB_CODE_SUCCESS) { - pSql->retry = 0; - } - /* * the pRes->code may be modified or released by another thread in tscTableMetaCallBack function, * so do NOT use pRes->code to determine if the getTableMeta function From b7584337ab543ffcc1e143a0d7037975b8010bc2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 4 Sep 2020 13:57:04 +0800 Subject: [PATCH 04/11] add more functionality file --- src/tsdb/src/tsdbCompact.c | 14 ++++++++++++++ src/tsdb/src/tsdbRecover.c | 14 ++++++++++++++ src/tsdb/src/tsdbScan.c | 14 ++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 src/tsdb/src/tsdbCompact.c create mode 100644 src/tsdb/src/tsdbRecover.c create mode 100644 src/tsdb/src/tsdbScan.c diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/src/tsdb/src/tsdbCompact.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbRecover.c b/src/tsdb/src/tsdbRecover.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/src/tsdb/src/tsdbRecover.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbScan.c b/src/tsdb/src/tsdbScan.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/src/tsdb/src/tsdbScan.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file From f21200d9f13a79a2ffaa5b0a7ab324f8039d5e9c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 4 Sep 2020 14:35:14 +0800 Subject: [PATCH 05/11] remove personalfiles interface. --- snap/snapcraft.yaml | 14 -------------- src/kit/shell/src/shellLinux.c | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 3f5b5f1de0..0ac0764217 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -24,21 +24,12 @@ apps: plugs: - network - system-observe - - historyfile taosdemo: command: usr/bin/taosdemo plugs: - network -plugs: - historyfile: - interface: personal-files - read: - - $HOME/.taos_history - write: - - $HOME/.taos_history - parts: script: plugin: dump @@ -101,8 +92,3 @@ layout: bind: $SNAP_DATA/var/log/taos /etc/taos: bind: $SNAP_DATA/etc/taos - - -hooks: - install: - plugs: [historyfile] diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 6c09d5c9d0..69bab44985 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -409,7 +409,7 @@ void set_terminal_mode() { } } -void get_history_path(char *history) { sprintf(history, "%s/%s", getpwuid(getuid())->pw_dir, HISTORY_FILE); } +void get_history_path(char *history) { sprintf(history, "%s/%s", getenv("HOME"), HISTORY_FILE); } void clearScreen(int ecmd_pos, int cursor_pos) { struct winsize w; From 7f56ba6056a936dfd74c6c55ef868b9744c79d4b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 4 Sep 2020 15:09:04 +0800 Subject: [PATCH 06/11] TD-1027 --- CMakeLists.txt | 1 + cmake/define.inc | 4 ++++ src/tsdb/inc/tsdbMain.h | 20 ++++++++++++++++++++ src/tsdb/src/tsdbScan.c | 24 +++++++++++++++++++++++- 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fee2e548a..946ceb95ab 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,7 @@ ENDIF () SET(TD_ACCOUNT FALSE) SET(TD_ADMIN FALSE) SET(TD_GRANT FALSE) +SET(TD_TSDB_PLUGINS FALSE) SET(TD_COVER FALSE) SET(TD_MEM_CHECK FALSE) diff --git a/cmake/define.inc b/cmake/define.inc index 0a25dd9ee7..6dd604057c 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -13,6 +13,10 @@ IF (TD_GRANT) ADD_DEFINITIONS(-D_GRANT) ENDIF () +IF (TD_TSDB_PLUGINS) + ADD_DEFINITIONS(-D_TSDB_PLUGINS) +ENDIF () + IF (TD_GODLL) ADD_DEFINITIONS(-D_TD_GO_DLL_) ENDIF () diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index ede0b33d01..256b8189f8 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -320,6 +320,16 @@ typedef struct { void* compBuffer; // Buffer for temperary compress/decompress purpose } SRWHelper; +// ------------------ tsdbScan.c +typedef struct { + SFileGroup fGroup; + int numOfIdx; + SCompIdx* pCompIdx; + SCompInfo* pCompInfo; + void* pBuf; + FILE* tLogStream; +} STsdbScanHandle; + // Operations // ------------------ tsdbMeta.c #define TSDB_INIT_NTABLES 1024 @@ -552,6 +562,16 @@ STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo); STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo); int tsdbCheckCommit(STsdbRepo* pRepo); +// ------------------ tsdbScan.c +int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid); +STsdbScanHandle* tsdbNewScanHandle(); +void tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream); +int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid); +int tsdbScanSCompIdx(STsdbScanHandle* pScanHandle); +int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx); +int tsdbCloseScanFile(STsdbScanHandle* pScanHandle); +void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle); + #ifdef __cplusplus } #endif diff --git a/src/tsdb/src/tsdbScan.c b/src/tsdb/src/tsdbScan.c index 6dea4a4e57..98fe1fd444 100644 --- a/src/tsdb/src/tsdbScan.c +++ b/src/tsdb/src/tsdbScan.c @@ -11,4 +11,26 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "tsdbMain.h" + +#ifndef _TSDB_PLUGINS + +int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } + +STsdbScanHandle* tsdbNewScanHandle() { return NULL; } + +void tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream) {} + +int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0 } + +int tsdbScanSCompIdx(STsdbScanHandle* pScanHandle) { return 0; } + +int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx) { return 0; } + +int tsdbCloseScanFile(STsdbScanHandle* pScanHandle) { return 0; } + +void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle) {} + +#endif \ No newline at end of file From 10621dafd8f657f5c72bb7b340f12135eaa79a84 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 4 Sep 2020 16:06:28 +0800 Subject: [PATCH 07/11] TD-1027 --- src/inc/tsdb.h | 5 +++++ src/tsdb/src/tsdbFile.c | 1 + src/tsdb/src/tsdbMain.c | 7 ++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index be73d8d383..85f9b3bdc7 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -38,6 +38,10 @@ extern "C" { #define TSDB_STATUS_COMMIT_START 1 #define TSDB_STATUS_COMMIT_OVER 2 +// TSDB STATE DEFINITION +#define TSDB_STATE_OK 0x0 +#define TSDB_STATE_BAD_FILE 0x1 + // --------- TSDB APPLICATION HANDLE DEFINITION typedef struct { void *appH; @@ -80,6 +84,7 @@ int32_t tsdbDropRepo(char *rootDir); TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH); void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit); int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg); +int tsdbGetState(TSDB_REPO_T *repo); // --------- TSDB TABLE DEFINITION typedef struct { diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 1009a61e86..930c64c030 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -542,6 +542,7 @@ static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo) { memset(&pFGroup->files[type].info, 0, sizeof(STsdbFileInfo)); pFGroup->files[type].info.magic = TSDB_FILE_INIT_MAGIC; pFGroup->state = 1; + pRepo->state = TSDB_STATE_BAD_FILE; terrno = TSDB_CODE_TDB_FILE_CORRUPTED; } } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 294df10edb..a1e6376304 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -142,7 +142,6 @@ TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { } tsdbStartStream(pRepo); - // pRepo->state = TSDB_REPO_STATE_ACTIVE; tsdbDebug("vgId:%d open tsdb repository succeed!", REPO_ID(pRepo)); @@ -341,6 +340,10 @@ void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int *compStorage = pRepo->stat.compStorage; } +int tsdbGetState(TSDB_REPO_T *repo) { + return ((STsdbRepo *)repo)->state; +} + // ----------------- INTERNAL FUNCTIONS ----------------- char *tsdbGetMetaFileName(char *rootDir) { int tlen = (int)(strlen(rootDir) + strlen(TSDB_META_FILE_NAME) + 2); @@ -661,6 +664,8 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { goto _err; } + pRepo->state = TSDB_STATE_OK; + int code = pthread_mutex_init(&pRepo->mutex, NULL); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); From 3bdde80638965e9a99522c35ec354fd434ab2923 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 4 Sep 2020 16:35:32 +0800 Subject: [PATCH 08/11] fix compile error --- src/tsdb/src/tsdbScan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbScan.c b/src/tsdb/src/tsdbScan.c index 98fe1fd444..91f6787874 100644 --- a/src/tsdb/src/tsdbScan.c +++ b/src/tsdb/src/tsdbScan.c @@ -23,7 +23,7 @@ STsdbScanHandle* tsdbNewScanHandle() { return NULL; } void tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream) {} -int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0 } +int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } int tsdbScanSCompIdx(STsdbScanHandle* pScanHandle) { return 0; } From 0a7eca2cdf7e9f37e8014277169f954a38202c02 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 4 Sep 2020 16:37:41 +0800 Subject: [PATCH 09/11] [TD-1266]update the nodejs test case --- tests/examples/nodejs/node-example-raw.js | 2 +- tests/examples/nodejs/node-example.js | 44 ++++++++++++++++++----- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/tests/examples/nodejs/node-example-raw.js b/tests/examples/nodejs/node-example-raw.js index a644051ce1..058a50c4c3 100644 --- a/tests/examples/nodejs/node-example-raw.js +++ b/tests/examples/nodejs/node-example-raw.js @@ -26,7 +26,7 @@ var c1 = conn.cursor(); // c1.execute(query) will execute the query // Let's create a database named db try { - c1.execute('create database db;'); + c1.execute('create database if not exists db;'); } catch(err) { conn.close(); diff --git a/tests/examples/nodejs/node-example.js b/tests/examples/nodejs/node-example.js index ce77af9d7c..bfdd9e49a0 100644 --- a/tests/examples/nodejs/node-example.js +++ b/tests/examples/nodejs/node-example.js @@ -22,8 +22,9 @@ var c1 = conn.cursor(); // c1.query(query) will return a TaosQuery object, of which then we can execute. The execute function then returns a promise // Let's create a database named db try { - var query = c1.query('create database db;'); - query.execute(); + c1.execute('create database if not exists db;'); + //var query = c1.query('create database if not exists db;'); + //query.execute(); } catch(err) { conn.close(); @@ -71,6 +72,28 @@ catch (err) { throw err; } + +Date.prototype.Format = function(fmt){ + var o = { + 'M+': this.getMonth() + 1, + 'd+': this.getDate(), + 'H+': this.getHours(), + 'm+': this.getMinutes(), + 's+': this.getSeconds(), + 'S+': this.getMilliseconds() + }; + if (/(y+)/.test(fmt)) { + fmt = fmt.replace(RegExp.$1, (this.getFullYear() + '').substr(4 - RegExp.$1.length)); + } + for (var k in o) { + if (new RegExp('(' + k + ')').test(fmt)) { + fmt = fmt.replace(RegExp.$1, (RegExp.$1.length == 1) ? (o[k]) : (('00' + o[k]).substr(String(o[k]).length))); + } + } + return fmt; +} + + // Let's try to insert some random generated data to test with // We will use the bind function of the TaosQuery object to easily bind values to question marks in the query // For Timestamps, a normal Datetime object or TaosTimestamp or milliseconds can be passed in through the bind function @@ -79,17 +102,21 @@ let interval = 1000; try { for (let i = 0; i < 1000; i++) { stime.setMilliseconds(stime.getMilliseconds() + interval); + + //console.log(stime.Format('yyyy-MM-dd HH:mm:ss.SSS')); + let insertData = [stime, parseInt(Math.random()*100), parseInt(Math.random()*300), parseFloat(Math.random()*10 + 30), - "\"random note!\""]; + "Note"]; //c1.execute('insert into db.weather values(' + insertData.join(',') + ' );'); - var query = c1.query('insert into db.weather values(?, ?, ?, ?, ?);').bind(insertData); - query.execute(); + + //var query = c1.query('insert into db.weather values(?, ?, ?, ?, ?);').bind(insertData); + //query.execute(); + c1.execute('insert into db.weather values(\"'+stime.Format('yyyy-MM-dd HH:mm:ss.SSS')+'\",'+parseInt(Math.random() * 100)+','+parseInt(Math.random() * 300)+','+parseFloat(Math.random()*10 + 30)+',"Note");'); } -} -catch (err) { +}catch (err) { conn.close(); throw err; } @@ -98,7 +125,8 @@ catch (err) { var retrievedData; try { c1.query('select * from db.weather limit 5 offset 100;', true).then(function(result){ - result.pretty(); + //result.pretty(); + console.log('=========>'+JSON.stringify(result)); // Neat! }); From 9bf2f5c29a57648b5ecf773f5835669afbaff288 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 4 Sep 2020 17:48:55 +0800 Subject: [PATCH 10/11] fix compile problem --- src/tsdb/src/tsdbFile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 930c64c030..51c3a625e8 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -455,7 +455,7 @@ int tsdbLoadFileHeader(SFile *pFile, uint32_t *version) { void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int64_t *size) { uint32_t version = 0; - SFile file = {0}; + SFile file; SFile * pFile = &file; strncpy(pFile->fname, fname, TSDB_FILENAME_LEN); From 49f9d2787721dbe74e59516b2f6c6d3d20bd648b Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 4 Sep 2020 17:52:38 +0800 Subject: [PATCH 11/11] TD-1331: update md for alter database --- documentation20/webdocs/markdowndocs/TAOS SQL-ch.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index b0f8ed276d..29f6e5ee57 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -82,13 +82,23 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ``` 删除数据库。所包含的全部数据表将被删除，谨慎使用 +- **修改数据库参数** + ```mysql + ALTER DATABASE db_name COMP 2; + ``` + 修改数据库文件压缩标志位，有效数字为0，1，2. 0表示不压缩，1表示一阶段压缩，2表示两阶段压缩。修改后可以使用show databases命令查看是否修改成功 + + ```mysql + ALTER DATABASE db_name REPLICA 2; + ``` + 修改数据库副本数，有效副本数为1到3。在集群中使用，副本数必须小于dnode的数目。修改后可以使用show databases命令查看是否修改成功 + - **显示系统所有数据库** ```mysql SHOW DATABASES; ``` - ## 表管理 - **创建数据表**