diff --git a/include/common/tmsgtype.h b/include/common/tmsgtype.h index 8e7ad87a0a..ebbf99b942 100644 --- a/include/common/tmsgtype.h +++ b/include/common/tmsgtype.h @@ -40,8 +40,9 @@ enum { // the SQL below is for mgmt node TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DB, "create-db" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_STABLE, "create-stable" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_TABLE, "create-table" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_FUNCTION, "create-function" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_FUNCTION, "create-function" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DB, "drop-db" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_TABLE, "drop-table" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_FUNCTION, "drop-function" ) diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index 041adbb582..b326ac032c 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -27,7 +27,7 @@ extern "C" { #include "tname.h" #include "tvariant.h" -/* +/** * The first field of a node of any type is guaranteed to be the int16_t. * Hence the type of any node can be gotten by casting it to SQueryNode. */ @@ -157,7 +157,7 @@ typedef struct SVgDataBlocks { typedef struct SInsertStmtInfo { int16_t nodeType; SArray* pDataBlocks; // data block for each vgroup, SArray. - int8_t schemaAttache; // denote if submit block is built with table schema or not + int8_t schemaAttache; // denote if submit block is built with table schema or not uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert uint32_t insertType; // insert data from [file|sql statement| bound statement] const char* sql; // current sql statement position diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 050e763919..b0b2c57ee4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -202,7 +202,7 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQuery, SQueryDag** pDag) { } int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { - if (TSDB_SQL_INSERT == pRequest->type) { + if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { return scheduleExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob, &pRequest->affectedRows); } return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob); diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 048c9e804e..6e1256d857 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -108,8 +108,17 @@ void iterateValueDestroy(IterateValue* iv, bool destroy); extern void* indexQhandle; +typedef struct TFileCacheKey { + uint64_t suid; + uint8_t colType; + char* colName; + int32_t nColName; +} ICacheKey; + int indexFlushCacheTFile(SIndex* sIdx, void*); +int32_t indexSerialCacheKey(ICacheKey* key, char* buf); + #define indexFatal(...) \ do { \ if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); } \ diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 12b66bca2c..805137ccaf 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -42,6 +42,7 @@ typedef struct IndexCache { int32_t version; int32_t nTerm; int8_t type; + uint64_t suid; pthread_mutex_t mtx; } IndexCache; @@ -58,7 +59,7 @@ typedef struct CacheTerm { } CacheTerm; // -IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type); +IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type); void indexCacheDestroy(void* cache); diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index 4928e01a63..4618a39197 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -49,13 +49,6 @@ typedef struct TFileValue { int32_t offset; } TFileValue; -typedef struct TFileCacheKey { - uint64_t suid; - uint8_t colType; - char* colName; - int32_t nColName; -} TFileCacheKey; - // table cache // refactor to LRU cache later typedef struct TFileCache { @@ -103,10 +96,10 @@ typedef struct TFileReaderOpt { // tfile cache, manage tindex reader TFileCache* tfileCacheCreate(const char* path); void tfileCacheDestroy(TFileCache* tcache); -TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key); -void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader); +TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key); +void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader); -TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName); +TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName); TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName); TFileReader* tfileReaderCreate(WriterCtx* ctx); @@ -124,6 +117,7 @@ int tfileWriterFinish(TFileWriter* tw); // IndexTFile* indexTFileCreate(const char* path); +void indexTFileDestroy(IndexTFile* tfile); int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid); int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result); diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index 21c5ca155b..adeb52bb8c 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -34,7 +34,7 @@ extern "C" { #define SERIALIZE_VAR_TO_BUF(buf, var, type) \ do { \ type c = var; \ - assert(sizeof(var) == sizeof(type)); \ + assert(sizeof(type) == sizeof(c)); \ memcpy((void*)buf, (void*)&c, sizeof(c)); \ buf += sizeof(c); \ } while (0) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 5167196031..9c7320b301 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -17,6 +17,7 @@ #include "indexInt.h" #include "index_cache.h" #include "index_tfile.h" +#include "index_util.h" #include "tdef.h" #include "tsched.h" @@ -102,6 +103,7 @@ void indexClose(SIndex* sIdx) { } taosHashCleanup(sIdx->colObj); pthread_mutex_destroy(&sIdx->mtx); + indexTFileDestroy(sIdx->tindex); #endif free(sIdx->path); free(sIdx); @@ -130,18 +132,28 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { // TODO(yihao): reduce the lock range pthread_mutex_lock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm* p = taosArrayGetP(fVals, i); - IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName); + SIndexTerm* p = taosArrayGetP(fVals, i); + + char buf[128] = {0}; + ICacheKey key = {.suid = p->suid, .colName = p->colName}; + int32_t sz = indexSerialCacheKey(&key, buf); + + IndexCache** cache = taosHashGet(index->colObj, buf, sz); if (cache == NULL) { - IndexCache* pCache = indexCacheCreate(index, p->colName, p->colType); - taosHashPut(index->colObj, p->colName, p->nColName, &pCache, sizeof(void*)); + IndexCache* pCache = indexCacheCreate(index, p->suid, p->colName, p->colType); + taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*)); } } pthread_mutex_unlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm* p = taosArrayGetP(fVals, i); - IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName); + SIndexTerm* p = taosArrayGetP(fVals, i); + + char buf[128] = {0}; + ICacheKey key = {.suid = p->suid, .colName = p->colName}; + int32_t sz = indexSerialCacheKey(&key, buf); + + IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); int ret = indexCachePut(*cache, p, uid); if (ret != 0) { return ret; } @@ -296,7 +308,12 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result // Get col info IndexCache* cache = NULL; pthread_mutex_lock(&sIdx->mtx); - IndexCache** pCache = taosHashGet(sIdx->colObj, colName, nColName); + + char buf[128] = {0}; + ICacheKey key = {.suid = term->suid, .colName = term->colName}; + int32_t sz = indexSerialCacheKey(&key, buf); + + IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz); if (pCache == NULL) { pthread_mutex_unlock(&sIdx->mtx); return -1; @@ -360,6 +377,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) { if (sz > 0) { // TODO(yihao): remove duplicate tableid TFileValue* lv = taosArrayGetP(result, sz - 1); + // indexError("merge colVal: %s", lv->colVal); if (strcmp(lv->colVal, tv->colVal) == 0) { taosArrayAddAll(lv->tableId, tv->tableId); tfileValueDestroy(tv); @@ -368,6 +386,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) { } } else { taosArrayPush(result, &tv); + // indexError("merge colVal: %s", tv->colVal); } } static void indexDestroyTempResult(SArray* result) { @@ -383,10 +402,12 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); IndexCache* pCache = (IndexCache*)cache; - TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->colName); + TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); + if (pReader == NULL) { indexWarn("empty pReader found"); } // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); Iterate* tfileIter = tfileIteratorCreate(pReader); + if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); } SArray* result = taosArrayInit(1024, sizeof(void*)); @@ -459,14 +480,14 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { } else { if (value->val != NULL) { taosArrayClear(value->val); } } - // free(value->colVal); + free(value->colVal); value->colVal = NULL; } static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { int32_t version = CACHE_VERSION(cache); uint8_t colType = cache->type; - TFileWriter* tw = tfileWriterOpen(sIdx->path, sIdx->suid, version, cache->colName, colType); + TFileWriter* tw = tfileWriterOpen(sIdx->path, cache->suid, version, cache->colName, colType); if (tw == NULL) { indexError("failed to open file to write"); return -1; @@ -479,14 +500,13 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { } tfileWriterClose(tw); - TFileReader* reader = tfileReaderOpen(sIdx->path, sIdx->suid, version, cache->colName); + TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName); + + char buf[128] = {0}; + TFileHeader* header = &reader->header; + ICacheKey key = { + .suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; - char buf[128] = {0}; - TFileHeader* header = &reader->header; - TFileCacheKey key = {.suid = header->suid, - .colName = header->colName, - .nColName = strlen(header->colName), - .colType = header->colType}; pthread_mutex_lock(&sIdx->mtx); IndexTFile* ifile = (IndexTFile*)sIdx->tindex; @@ -497,3 +517,13 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { END: tfileWriterClose(tw); } + +int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { + char* p = buf; + SERIALIZE_MEM_TO_BUF(buf, key, suid); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + // SERIALIZE_MEM_TO_BUF(buf, key, colType); + // SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); + return buf - p; +} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 0e46445a00..b4c533e998 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,7 +20,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define MEM_TERM_LIMIT 10000 * 10 +#define MEM_TERM_LIMIT 10 * 10000 // ref index_cache.h:22 //#define CACHE_KEY_LEN(p) \ // (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + @@ -40,7 +40,7 @@ static bool indexCacheIteratorNext(Iterate* itera); static IterateValue* indexCacheIteratorGetValue(Iterate* iter); -IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) { +IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { IndexCache* cache = calloc(1, sizeof(IndexCache)); if (cache == NULL) { indexError("failed to create index cache"); @@ -53,7 +53,7 @@ IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) { cache->type = type; cache->index = idx; cache->version = 0; - + cache->suid = suid; pthread_mutex_init(&cache->mtx, NULL); indexCacheRef(cache); return cache; @@ -150,6 +150,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { MemTable* tbl = cache->imm; iiter->val.val = taosArrayInit(1, sizeof(uint64_t)); + iiter->val.colVal = NULL; iiter->iter = tbl != NULL ? tSkipListCreateIter(tbl->mem) : NULL; iiter->next = indexCacheIteratorNext; iiter->getValue = indexCacheIteratorGetValue; @@ -353,6 +354,9 @@ static bool indexCacheIteratorNext(Iterate* itera) { SSkipListIterator* iter = itera->iter; if (iter == NULL) { return false; } IterateValue* iv = &itera->val; + if (iv->colVal != NULL && iv->val != NULL) { + // indexError("value in cache: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val)); + } iterateValueDestroy(iv, false); bool next = tSkipListIterNext(iter); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 04a08dafd2..bfaeeaaa33 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -319,7 +319,7 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) { assert(s->state == OneTransNext || s->state == OneTrans); uint8_t val; - COMMON_INDEX(inp, 0x111111, val); + COMMON_INDEX(inp, 0b111111, val); s->val = (s->val & fstStateDict[s->state].val) | val; } @@ -369,7 +369,7 @@ uint8_t fstStateInput(FstState* s, FstNode* node) { bool null = false; uint8_t inp = fstStateCommInput(s, &null); uint8_t* data = fstSliceData(slice, NULL); - return null == false ? inp : data[-1]; + return null == false ? inp : data[node->start - 1]; } uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); @@ -1062,6 +1062,7 @@ Output fstEmptyFinalOutput(Fst* fst, bool* null) { } else { *null = true; } + fstNodeDestroy(node); return res; } @@ -1286,6 +1287,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); free(buf); fstSliceDestroy(&slice); + taosArrayDestroy(nodes); return result; } free(buf); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index fc4f8593a1..95c713fb0a 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -51,7 +51,6 @@ static void tfileDestroyFileName(void* elem); static int tfileCompare(const void* a, const void* b); static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version); static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version); -static void tfileSerialCacheKey(TFileCacheKey* key, char* buf); TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = calloc(1, sizeof(TFileCache)); @@ -80,18 +79,18 @@ TFileCache* tfileCacheCreate(const char* path) { goto End; } - char buf[128] = {0}; - TFileReader* reader = tfileReaderCreate(wc); - TFileHeader* header = &reader->header; - TFileCacheKey key = {.suid = header->suid, - .colName = header->colName, - .nColName = strlen(header->colName), - .colType = header->colType}; - tfileSerialCacheKey(&key, buf); + char buf[128] = {0}; + TFileReader* reader = tfileReaderCreate(wc); + TFileHeader* header = &reader->header; + ICacheKey key = {.suid = header->suid, + .colName = header->colName, + .nColName = strlen(header->colName), + .colType = header->colType}; + int32_t sz = indexSerialCacheKey(&key, buf); + assert(sz < sizeof(buf)); + taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); tfileReaderRef(reader); - // indexTable - taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); } taosArrayDestroyEx(files, tfileDestroyFileName); return tcache; @@ -117,30 +116,30 @@ void tfileCacheDestroy(TFileCache* tcache) { free(tcache); } -TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) { - char buf[128] = {0}; - tfileSerialCacheKey(key, buf); - - TFileReader** reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); +TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { + char buf[128] = {0}; + int32_t sz = indexSerialCacheKey(key, buf); + assert(sz < sizeof(buf)); + TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz); if (reader == NULL) { return NULL; } tfileReaderRef(*reader); return *reader; } -void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) { - char buf[128] = {0}; - tfileSerialCacheKey(key, buf); +void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { + char buf[128] = {0}; + int32_t sz = indexSerialCacheKey(key, buf); // remove last version index reader - TFileReader** p = taosHashGet(tcache->tableCache, buf, strlen(buf)); + TFileReader** p = taosHashGet(tcache->tableCache, buf, sz); if (p != NULL) { TFileReader* oldReader = *p; - taosHashRemove(tcache->tableCache, buf, strlen(buf)); + taosHashRemove(tcache->tableCache, buf, sz); oldReader->remove = true; tfileReaderUnRef(oldReader); } + taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); tfileReaderRef(reader); - taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); return; } TFileReader* tfileReaderCreate(WriterCtx* ctx) { @@ -230,8 +229,6 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c TFileReader* reader = tfileReaderCreate(wc); return reader; - - // tfileSerialCacheKey(&key, buf); } TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { // char pathBuf[128] = {0}; @@ -325,15 +322,19 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { tfileWriterClose(tw); return -1; } - // write fst + + // write data indexError("--------Begin----------------"); for (size_t i = 0; i < sz; i++) { // TODO, fst batch write later TFileValue* v = taosArrayGetP((SArray*)data, i); - if (tfileWriteData(tw, v) == 0) { - // + if (tfileWriteData(tw, v) != 0) { + indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset, + (int)taosArrayGetSize(v->tableId)); + } else { + indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset, + (int)taosArrayGetSize(v->tableId)); } - indexError("data: %s, offset: %d len: %d", v->colVal, v->offset, (int)taosArrayGetSize(v->tableId)); } indexError("--------End----------------"); fstBuilderFinish(tw->fb); @@ -359,7 +360,7 @@ IndexTFile* indexTFileCreate(const char* path) { tfile->cache = tfileCacheCreate(path); return tfile; } -void IndexTFileDestroy(IndexTFile* tfile) { +void indexTFileDestroy(IndexTFile* tfile) { tfileCacheDestroy(tfile->cache); free(tfile); } @@ -369,9 +370,8 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { if (tfile == NULL) { return ret; } IndexTFile* pTfile = (IndexTFile*)tfile; - SIndexTerm* term = query->term; - TFileCacheKey key = { - .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; + SIndexTerm* term = query->term; + ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileReader* reader = tfileCacheGet(pTfile->cache, &key); if (reader == NULL) { return 0; } @@ -385,8 +385,10 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { } static bool tfileIteratorNext(Iterate* iiter) { IterateValue* iv = &iiter->val; + if (iv->colVal != NULL && iv->val != NULL) { + // indexError("value in fst: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val)); + } iterateValueDestroy(iv, false); - // SArray* tblIds = iv->val; char* colVal = NULL; uint64_t offset = 0; @@ -406,14 +408,14 @@ static bool tfileIteratorNext(Iterate* iiter) { if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } iv->colVal = colVal; - + return true; // std::string key(ch, sz); } static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { - TFileFstIter* tIter = calloc(1, sizeof(Iterate)); + TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter)); if (tIter == NULL) { return NULL; } tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); @@ -435,6 +437,7 @@ Iterate* tfileIteratorCreate(TFileReader* reader) { iter->next = tfileIteratorNext; iter->getValue = tifileIterateGetValue; iter->val.val = taosArrayInit(1, sizeof(uint64_t)); + iter->val.colVal = NULL; return iter; } void tfileIteratorDestroy(Iterate* iter) { @@ -447,13 +450,14 @@ void tfileIteratorDestroy(Iterate* iter) { streamWithStateDestroy(tIter->st); fstStreamBuilderDestroy(tIter->fb); automCtxDestroy(tIter->ctx); + free(tIter); free(iter); } -TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) { +TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) { if (tf == NULL) { return NULL; } - TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; + ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; return tfileCacheGet(tf->cache, &key); } @@ -480,7 +484,7 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { TFileValue* tfileValueCreate(char* val) { TFileValue* tf = calloc(1, sizeof(TFileValue)); if (tf == NULL) { return NULL; } - tf->colVal = val; + tf->colVal = tstrdup(val); tf->tableId = taosArrayInit(32, sizeof(uint64_t)); return tf; } @@ -491,6 +495,7 @@ int tfileValuePush(TFileValue* tf, uint64_t val) { } void tfileValueDestroy(TFileValue* tf) { taosArrayDestroy(tf->tableId); + free(tf->colVal); free(tf); } static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { @@ -648,10 +653,3 @@ static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, } return -1; } -static void tfileSerialCacheKey(TFileCacheKey* key, char* buf) { - // SERIALIZE_MEM_TO_BUF(buf, key, suid); - // SERIALIZE_VAR_TO_BUF(buf, '_', char); - // SERIALIZE_MEM_TO_BUF(buf, key, colType); - // SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); -} diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index da974ce6c4..3d978c05a5 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -24,8 +24,13 @@ class FstWriter { _b = fstBuilderCreate(_wc, 0); } bool Put(const std::string& key, uint64_t val) { + // char buf[128] = {0}; + // int len = 0; + // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len); + // FstSlice skey = fstSliceCreate((uint8_t*)buf, len); FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); bool ok = fstBuilderInsert(_b, skey, val); + fstSliceDestroy(&skey); return ok; } @@ -61,6 +66,11 @@ class FstReadMemory { return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { + // char buf[128] = {0}; + // int len = 0; + // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len); + // FstSlice skey = fstSliceCreate((uint8_t*)buf, len); + FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); bool ok = fstGet(_fst, &skey, val); fstSliceDestroy(&skey); @@ -135,15 +145,109 @@ int Performance_fstWriteRecords(FstWriter* b) { } return L * M * N; } +void Performance_fstReadRecords(FstReadMemory* m) { + std::string str("aa"); + for (int i = 0; i < M; i++) { + str[0] = 'a' + i; + str.resize(2); + for (int j = 0; j < N; j++) { + str[1] = 'a' + j; + str.resize(2); + for (int k = 0; k < L; k++) { + str.push_back('a'); + uint64_t val, cost; + if (m->GetWithTimeCostUs(str, &val, &cost)) { + printf("succes to get kv(%s, %" PRId64 "), cost: %" PRId64 "\n", str.c_str(), val, cost); + } else { + printf("failed to get key: %s\n", str.c_str()); + } + } + } + } +} + +void checkMillonWriteAndReadOfFst() { + tfInit(); + FstWriter* fw = new FstWriter; + Performance_fstWriteRecords(fw); + delete fw; + FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024); + + if (fr->init()) { printf("success to init fst read"); } + + Performance_fstReadRecords(fr); + tfCleanup(); + delete fr; +} +void checkFstLongTerm() { + tfInit(); + FstWriter* fw = new FstWriter; + // Performance_fstWriteRecords(fw); + + fw->Put("A B", 1); + fw->Put("C", 2); + fw->Put("a", 3); + delete fw; + + FstReadMemory* m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; + delete m; + return; + } + { + uint64_t val = 0; + if (m->Get("A B", &val)) { + std::cout << "success to Get: " << val << std::endl; + } else { + std::cout << "failed to Get:" << val << std::endl; + } + } + { + uint64_t val = 0; + if (m->Get("C", &val)) { + std::cout << "success to Get: " << val << std::endl; + } else { + std::cout << "failed to Get:" << val << std::endl; + } + } + { + uint64_t val = 0; + if (m->Get("a", &val)) { + std::cout << "success to Get: " << val << std::endl; + } else { + std::cout << "failed to Get:" << val << std::endl; + } + } + + // prefix search + // std::vector result; + + // AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS); + // m->Search(ctx, result); + // std::cout << "size: " << result.size() << std::endl; + // assert(result.size() == count); + // for (int i = 0; i < result.size(); i++) { + // assert(result[i] == i); // check result + //} + tfCleanup(); + // free(ctx); + // delete m; +} void checkFstCheckIterator() { tfInit(); FstWriter* fw = new FstWriter; int64_t s = taosGetTimestampUs(); int count = 2; - Performance_fstWriteRecords(fw); + // Performance_fstWriteRecords(fw); int64_t e = taosGetTimestampUs(); std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl; + + fw->Put("Hello world", 1); + fw->Put("hello world", 2); + fw->Put("hello worle", 3); + fw->Put("hello worlf", 4); delete fw; FstReadMemory* m = new FstReadMemory(1024 * 64); @@ -171,7 +275,7 @@ void checkFstCheckIterator() { void fst_get(Fst* fst) { for (int i = 0; i < 10000; i++) { - std::string term = "Hello"; + std::string term = "Hello World"; FstSlice key = fstSliceCreate((uint8_t*)term.c_str(), term.size()); uint64_t offset = 0; bool ret = fstGet(fst, &key, &offset); @@ -189,7 +293,7 @@ void validateTFile(char* arg) { std::thread threads[NUM_OF_THREAD]; // std::vector threads; - TFileReader* reader = tfileReaderOpen(arg, 0, 295868, "tag1"); + TFileReader* reader = tfileReaderOpen(arg, 0, 999992, "tag1"); for (int i = 0; i < NUM_OF_THREAD; i++) { threads[i] = std::thread(fst_get, reader->fst); @@ -203,9 +307,12 @@ void validateTFile(char* arg) { tfCleanup(); } int main(int argc, char* argv[]) { - if (argc > 1) { validateTFile(argv[1]); } + // tool to check all kind of fst test + // if (argc > 1) { validateTFile(argv[1]); } // checkFstCheckIterator(); + // checkFstLongTerm(); // checkFstPrefixSearch(); + checkMillonWriteAndReadOfFst(); return 1; } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 080becccf1..bdfb86ce17 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -457,7 +457,10 @@ TEST_F(IndexTFileEnv, test_tfile_write) { // taosArrayPush(data, &v4); fObj->Put(data); - for (size_t i = 0; i < taosArrayGetSize(data); i++) { destroyTFileValue(taosArrayGetP(data, i)); } + for (size_t i = 0; i < taosArrayGetSize(data); i++) { + // data + destroyTFileValue(taosArrayGetP(data, i)); + } taosArrayDestroy(data); std::string colName("voltage"); @@ -470,6 +473,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { fObj->Get(&query, result); assert(taosArrayGetSize(result) == 200); indexTermDestroy(term); + taosArrayDestroy(result); // tfileWriterDestroy(twrite); } @@ -477,7 +481,7 @@ class CacheObj { public: CacheObj() { // TODO - cache = indexCacheCreate(NULL, "voltage", TSDB_DATA_TYPE_BINARY); + cache = indexCacheCreate(NULL, 0, "voltage", TSDB_DATA_TYPE_BINARY); } int Put(SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) { int ret = indexCachePut(cache, term, uid); @@ -534,6 +538,7 @@ TEST_F(IndexCacheEnv, cache_test) { SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); // indexTermDestry(term); } { @@ -541,24 +546,28 @@ TEST_F(IndexCacheEnv, cache_test) { SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); } { std::string colVal("v2"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); } { std::string colVal("v3"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); } { std::string colVal("v3"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); } coj->Debug(); std::cout << "--------first----------" << std::endl; @@ -567,12 +576,14 @@ TEST_F(IndexCacheEnv, cache_test) { SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); + indexTermDestroy(term); } { std::string colVal("v4"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); + indexTermDestroy(term); } coj->Debug(); std::cout << "--------second----------" << std::endl; @@ -583,6 +594,7 @@ TEST_F(IndexCacheEnv, cache_test) { SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); } } coj->Debug(); @@ -598,6 +610,9 @@ TEST_F(IndexCacheEnv, cache_test) { coj->Get(&query, colId, 10000, ret, &valType); std::cout << "size : " << taosArrayGetSize(ret) << std::endl; assert(taosArrayGetSize(ret) == 4); + taosArrayDestroy(ret); + + indexTermDestroy(term); } { std::string colVal("v2"); @@ -609,6 +624,9 @@ TEST_F(IndexCacheEnv, cache_test) { coj->Get(&query, colId, 10000, ret, &valType); assert(taosArrayGetSize(ret) == 1); + taosArrayDestroy(ret); + + indexTermDestroy(term); } } class IndexObj { @@ -678,13 +696,16 @@ class IndexObj { SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); if (Search(mq, result) == 0) { std::cout << "search one successfully" << std::endl; } - return taosArrayGetSize(result); + int sz = taosArrayGetSize(result); + indexMultiTermQueryDestroy(mq); + taosArrayDestroy(result); + return sz; // assert(taosArrayGetSize(result) == targetSize); } void PutOne(const std::string& colName, const std::string& colVal) { + SIndexMultiTerm* terms = indexMultiTermCreate(); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); Put(terms, 10); indexMultiTermDestroy(terms); @@ -783,18 +804,21 @@ TEST_F(IndexEnv2, testIndexOpen) { index->Search(mq, result); std::cout << "target size: " << taosArrayGetSize(result) << std::endl; assert(taosArrayGetSize(result) == 400); + taosArrayDestroy(result); + indexMultiTermQueryDestroy(mq); } } TEST_F(IndexEnv2, testIndex_TrigeFlush) { - std::string path = "/tmp/test"; + std::string path = "/tmp/test1"; if (index->Init(path) != 0) { // r std::cout << "failed to init" << std::endl; } int numOfTable = 100 * 10000; - index->WriteMillonData("tag1", "Hello", numOfTable); - int target = index->SearchOne("tag1", "Hello"); + index->WriteMillonData("tag1", "Hello Wolrd", numOfTable); + int target = index->SearchOne("tag1", "Hello Wolrd"); + std::cout << "Get Index: " << target << std::endl; assert(numOfTable == target); } @@ -821,14 +845,6 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { threads[i].join(); } } -TEST_F(IndexEnv2, testIndex_multi_thread_write) { - std::string path = "/tmp"; - if (index->Init(path) != 0) {} -} -TEST_F(IndexEnv2, testIndex_multi_thread_read) { - std::string path = "/tmp"; - if (index->Init(path) != 0) {} -} TEST_F(IndexEnv2, testIndex_restart) { std::string path = "/tmp"; diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 4bbe6ab907..346bd0cbe4 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -68,7 +68,9 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQ * @param type * @return */ -int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStmtInfo* pDcl, char* msgBuf, int32_t msgBufLen); +SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen); + +SInsertStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen); /** * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause. diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index ecea48f583..76e26c159b 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -326,8 +326,7 @@ typedef struct SVgroupTablesBatch { SVgroupInfo info; } SVgroupTablesBatch; -int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len, - SEpSet* pEpSet) { +int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) { const char* msg1 = "invalid table name"; const char* msg2 = "tags number not matched"; const char* msg3 = "tag value too long"; @@ -359,7 +358,11 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p size_t numOfInputTag = taosArrayGetSize(pValList); STableMeta* pSuperTableMeta = NULL; - catalogGetTableMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &name, &pSuperTableMeta); + code = catalogGetTableMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &name, &pSuperTableMeta); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + assert(pSuperTableMeta != NULL); // too long tag values will return invalid sql, not be truncated automatically @@ -501,14 +504,6 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p req.ctbCfg.suid = pSuperTableMeta->suid; req.ctbCfg.pTag = row; - // pEpSet->inUse = info.inUse; - // pEpSet->numOfEps = info.numOfEps; - // for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { - // pEpSet->port[i] = info.epAddr[i].port; - // tstrncpy(pEpSet->fqdn[i], info.epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); - // } - // ((SMsgHead*)(*pOutput))->vgId = htonl(info.vgId); - // ((SMsgHead*)(*pOutput))->contLen = htonl(serLen); SVgroupTablesBatch* pTableBatch = taosHashGet(pVgroupHashmap, &info.vgId, sizeof(info.vgId)); if (pTableBatch == NULL) { SVgroupTablesBatch tBatch = {0}; @@ -525,12 +520,12 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p } // TODO: serialize and - SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); - SVgroupTablesBatch** ppTbBatch = NULL; + SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); + + SVgroupTablesBatch* pTbBatch = NULL; do { - ppTbBatch = taosHashIterate(pVgroupHashmap, ppTbBatch); - if (ppTbBatch == NULL) break; - SVgroupTablesBatch* pTbBatch = *ppTbBatch; + pTbBatch = taosHashIterate(pVgroupHashmap, pTbBatch); + if (pTbBatch == NULL) break; int tlen = sizeof(SMsgHead) + tSVCreateTbBatchReqSerialize(NULL, &(pTbBatch->req)); void* buf = malloc(tlen); @@ -544,17 +539,29 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tSVCreateTbBatchReqSerialize(&pBuf, &(pTbBatch->req)); - taosArrayPush(pBufArray, &buf); + SVgDataBlocks* pVgData = calloc(1, sizeof(SVgDataBlocks)); + pVgData->vg = pTbBatch->info; + pVgData->pData = buf; + pVgData->size = tlen; + pVgData->numOfTables = (int32_t) taosArrayGetSize(pTbBatch->req.pArray); + taosArrayPush(pBufArray, &pVgData); } while (true); + SInsertStmtInfo* pStmtInfo = calloc(1, sizeof(SInsertStmtInfo)); + pStmtInfo->nodeType = TSDB_SQL_CREATE_TABLE; + pStmtInfo->pDataBlocks = pBufArray; + *pOutput = pStmtInfo; + *len = sizeof(SInsertStmtInfo); + return TSDB_CODE_SUCCESS; } -int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStmtInfo* pDcl, char* msgBuf, - int32_t msgBufLen) { +SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { int32_t code = 0; + SDclStmtInfo* pDcl = calloc(1, sizeof(SDclStmtInfo)); + SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf* pMsgBuf = &m; @@ -571,21 +578,25 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm SToken* pPwd = &pUser->passwd; if (pName->n >= TSDB_USER_LEN) { - return buildInvalidOperationMsg(pMsgBuf, msg3); + code = buildInvalidOperationMsg(pMsgBuf, msg3); + goto _error; } if (parserValidateIdToken(pName) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg2); + code = buildInvalidOperationMsg(pMsgBuf, msg2); + goto _error; } if (pInfo->type == TSDB_SQL_CREATE_USER) { if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + code = TSDB_CODE_TSC_INVALID_OPERATION; + goto _error; } } else { if (pUser->type == TSDB_ALTER_USER_PASSWD) { if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + code = TSDB_CODE_TSC_INVALID_OPERATION; + goto _error; } } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { assert(pPwd->type == TSDB_DATA_TYPE_NULL); @@ -596,10 +607,12 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm } else if (strncasecmp(pPrivilege->z, "normal", 4) == 0 && pPrivilege->n == 4) { // pCmd->count = 2; } else { - return buildInvalidOperationMsg(pMsgBuf, msg4); + code = buildInvalidOperationMsg(pMsgBuf, msg4); + goto _error; } } else { - return buildInvalidOperationMsg(pMsgBuf, msg1); + code = buildInvalidOperationMsg(pMsgBuf, msg1); + goto _error; } } @@ -618,15 +631,18 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm SToken* pPwd = &pInfo->pMiscInfo->user.passwd; if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + code = TSDB_CODE_TSC_INVALID_OPERATION; + goto _error; } if (pName->n >= TSDB_USER_LEN) { - return buildInvalidOperationMsg(pMsgBuf, msg3); + code = buildInvalidOperationMsg(pMsgBuf, msg3); + goto _error; } if (parserValidateNameToken(pName) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg2); + code = buildInvalidOperationMsg(pMsgBuf, msg2); + goto _error; } SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; @@ -636,7 +652,8 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { } else { - return buildInvalidOperationMsg(pMsgBuf, msg1); + code = buildInvalidOperationMsg(pMsgBuf, msg1); + goto _error; } } @@ -655,6 +672,10 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm case TSDB_SQL_SHOW: { SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; code = setShowInfo(pShowInfo, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet, &pDcl->pExtension, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pDcl->msgType = (pShowInfo->showType == TSDB_MGMT_TABLE_TABLE) ? TDMT_VND_SHOW_TABLES : TDMT_MND_SHOW; break; } @@ -664,13 +685,15 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (parserValidateNameToken(pToken) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg); + code = buildInvalidOperationMsg(pMsgBuf, msg); + goto _error; } SName n = {0}; int32_t ret = tNameSetDbName(&n, pCtx->acctId, pToken->z, pToken->n); if (ret != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg); + code = buildInvalidOperationMsg(pMsgBuf, msg); + goto _error; } SUseDbMsg* pUseDbMsg = (SUseDbMsg*)calloc(1, sizeof(SUseDbMsg)); @@ -689,19 +712,22 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) { - return buildInvalidOperationMsg(pMsgBuf, msg2); + code = buildInvalidOperationMsg(pMsgBuf, msg2); + goto _error; } char buf[TSDB_DB_NAME_LEN] = {0}; SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf)); if (parserValidateNameToken(&token) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + code = buildInvalidOperationMsg(pMsgBuf, msg1); + goto _error; } SCreateDbMsg* pCreateMsg = buildCreateDbMsg(pCreateDB, pCtx, pMsgBuf); if (doCheckDbOptions(pCreateMsg, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + code = TSDB_CODE_TSC_INVALID_OPERATION; + goto _error; } pDcl->pMsg = (char*)pCreateMsg; @@ -719,7 +745,8 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm SName name = {0}; code = tNameSetDbName(&name, pCtx->acctId, dbName->z, dbName->n); if (code != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + code = buildInvalidOperationMsg(pMsgBuf, msg1); + goto _error; } SDropDbMsg* pDropDbMsg = (SDropDbMsg*)calloc(1, sizeof(SDropDbMsg)); @@ -731,7 +758,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm pDcl->msgType = TDMT_MND_DROP_DB; pDcl->msgLen = sizeof(SDropDbMsg); pDcl->pMsg = (char*)pDropDbMsg; - return TSDB_CODE_SUCCESS; + break; } case TSDB_SQL_CREATE_TABLE: { @@ -739,14 +766,16 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm if (pCreateTable->type == TSQL_CREATE_TABLE || pCreateTable->type == TSQL_CREATE_STABLE) { if ((code = doCheckForCreateTable(pInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) { - return code; + terrno = code; + goto _error; } + pDcl->pMsg = (char*)buildCreateTableMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf); pDcl->msgType = (pCreateTable->type == TSQL_CREATE_TABLE) ? TDMT_VND_CREATE_TABLE : TDMT_MND_CREATE_STB; } else if (pCreateTable->type == TSQL_CREATE_CTABLE) { - if ((code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf, &pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet)) != + if ((code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf, &pDcl->pMsg, &pDcl->msgLen)) != TSDB_CODE_SUCCESS) { - return code; + goto _error; } pDcl->msgType = TDMT_VND_CREATE_TABLE; @@ -761,7 +790,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm case TSDB_SQL_DROP_TABLE: { pDcl->pMsg = (char*)buildDropStableMsg(pInfo, &pDcl->msgLen, pCtx, pMsgBuf); if (pDcl->pMsg == NULL) { - code = terrno; + goto _error; } pDcl->msgType = TDMT_MND_DROP_STB; @@ -771,7 +800,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm case TSDB_SQL_CREATE_DNODE: { pDcl->pMsg = (char*)buildCreateDnodeMsg(pInfo, &pDcl->msgLen, pMsgBuf); if (pDcl->pMsg == NULL) { - code = terrno; + goto _error; } pDcl->msgType = TDMT_MND_CREATE_DNODE; @@ -781,7 +810,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm case TSDB_SQL_DROP_DNODE: { pDcl->pMsg = (char*)buildDropDnodeMsg(pInfo, &pDcl->msgLen, pMsgBuf); if (pDcl->pMsg == NULL) { - code = terrno; + goto _error; } pDcl->msgType = TDMT_MND_DROP_DNODE; @@ -792,5 +821,29 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm break; } - return code; + return pDcl; + + _error: + terrno = code; + tfree(pDcl); + return NULL; } + +SInsertStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { + SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; + assert(pCreateTable->type == TSQL_CREATE_CTABLE); + + SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; + SMsgBuf* pMsgBuf = &m; + + SInsertStmtInfo* pInsertStmt = NULL; + + int32_t msgLen = 0; + int32_t code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf, (char**) &pInsertStmt, &msgLen); + if (code != TSDB_CODE_SUCCESS) { + tfree(pInsertStmt); + return NULL; + } + + return pInsertStmt; +} \ No newline at end of file diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 710cf4b5d0..1b4d05808c 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -32,7 +32,7 @@ bool isInsertSql(const char* pStr, size_t length) { } bool qIsDdlQuery(const SQueryNode* pQuery) { - return TSDB_SQL_INSERT != pQuery->type && TSDB_SQL_SELECT != pQuery->type; + return TSDB_SQL_INSERT != pQuery->type && TSDB_SQL_SELECT != pQuery->type && TSDB_SQL_CREATE_TABLE != pQuery->type; } int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { @@ -44,16 +44,29 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { } if (!isDqlSqlStatement(&info)) { - SDclStmtInfo* pDcl = calloc(1, sizeof(SDclStmtInfo)); - if (NULL == pDcl) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. - return terrno; + bool toVnode = false; + if (info.type == TSDB_SQL_CREATE_TABLE) { + SCreateTableSql* pCreateSql = info.pCreateTableInfo; + if (pCreateSql->type == TSQL_CREATE_CTABLE || pCreateSql->type == TSQL_CREATE_TABLE) { + toVnode = true; + } } - pDcl->nodeType = info.type; - int32_t code = qParserValidateDclSqlNode(&info, &pCxt->ctx, pDcl, pCxt->pMsg, pCxt->msgLen); - if (code == TSDB_CODE_SUCCESS) { + if (toVnode) { + SInsertStmtInfo *pInsertInfo = qParserValidateCreateTbSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); + if (pInsertInfo == NULL) { + return terrno; + } + + *pQuery = (SQueryNode*) pInsertInfo; + } else { + SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); + if (pDcl == NULL) { + return terrno; + } + *pQuery = (SQueryNode*)pDcl; + pDcl->nodeType = info.type; } } else { SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index a67a9a8be8..fe430c5f5e 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -714,10 +714,9 @@ TEST(testCase, show_user_Test) { SSqlInfo info1 = doGenerateAST(sql1); ASSERT_EQ(info1.valid, true); - SDclStmtInfo output; SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL}; - int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, msg, buf.len); - ASSERT_EQ(code, 0); + SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len); + ASSERT_NE(output, nullptr); // convert the show command to be the select query // select name, privilege, create_time, account from information_schema.users; @@ -735,10 +734,9 @@ TEST(testCase, create_user_Test) { ASSERT_EQ(info1.valid, true); ASSERT_EQ(isDclSqlStatement(&info1), true); - SDclStmtInfo output; SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc"}; - int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, msg, buf.len); - ASSERT_EQ(code, 0); + SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len); + ASSERT_NE(output, nullptr); destroySqlInfo(&info1); } \ No newline at end of file diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 35c6d59ffe..a68102ea6e 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -40,7 +40,7 @@ extern "C" { #define QNODE_SESSIONWINDOW 12 #define QNODE_STATEWINDOW 13 #define QNODE_FILL 14 -#define QNODE_INSERT 15 +#define QNODE_MODIFY 15 typedef struct SQueryDistPlanNodeInfo { bool stableQuery; // super table query or not diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index 136073aa60..5f95f86d4a 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -37,15 +37,19 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { return 0; } -int32_t createInsertPlan(const SInsertStmtInfo* pInsert, SQueryPlanNode** pQueryPlan) { +static int32_t createInsertPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { + SInsertStmtInfo* pInsert = (SInsertStmtInfo*)pNode; + *pQueryPlan = calloc(1, sizeof(SQueryPlanNode)); SArray* blocks = taosArrayInit(taosArrayGetSize(pInsert->pDataBlocks), POINTER_BYTES); if (NULL == *pQueryPlan || NULL == blocks) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - (*pQueryPlan)->info.type = QNODE_INSERT; + + (*pQueryPlan)->info.type = QNODE_MODIFY; taosArrayAddAll(blocks, pInsert->pDataBlocks); (*pQueryPlan)->pExtInfo = blocks; + return TSDB_CODE_SUCCESS; } @@ -62,13 +66,14 @@ int32_t createQueryPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { case TSDB_SQL_SELECT: { return createSelectPlan((const SQueryStmtInfo*)pNode, pQueryPlan); } + case TSDB_SQL_INSERT: - return createInsertPlan((const SInsertStmtInfo*)pNode, pQueryPlan); + case TSDB_SQL_CREATE_TABLE: + return createInsertPlan(pNode, pQueryPlan); + default: return TSDB_CODE_FAILED; } - - return TSDB_CODE_SUCCESS; } int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 97c9cec7c7..978b1554f3 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -34,7 +34,7 @@ static const char* gOpName[] = { #undef INCLUDE_AS_NAME }; -static void* vailidPointer(void* p) { +static void* validPointer(void* p) { if (NULL == p) { THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); } @@ -76,7 +76,7 @@ int32_t dsinkNameToDsinkType(const char* name) { } static SDataSink* initDataSink(int32_t type, int32_t size) { - SDataSink* sink = (SDataSink*)vailidPointer(calloc(1, size)); + SDataSink* sink = (SDataSink*)validPointer(calloc(1, size)); sink->info.type = type; sink->info.name = dsinkTypeToDsinkName(type); return sink; @@ -121,7 +121,7 @@ static bool cloneExprArray(SArray** dst, SArray* src) { } static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { - SPhyNode* node = (SPhyNode*)vailidPointer(calloc(1, size)); + SPhyNode* node = (SPhyNode*)validPointer(calloc(1, size)); node->info.type = type; node->info.name = opTypeToOpName(type); if (!cloneExprArray(&node->pTargets, pPlanNode->pExpr) || !toDataBlockSchema(pPlanNode, &(node->targetSchema))) { @@ -184,7 +184,7 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable } static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { - SSubplan* subplan = vailidPointer(calloc(1, sizeof(SSubplan))); + SSubplan* subplan = validPointer(calloc(1, sizeof(SSubplan))); subplan->id = pCxt->nextId; ++(pCxt->nextId.subplanId); subplan->type = type; @@ -192,15 +192,15 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { if (NULL != pCxt->pCurrentSubplan) { subplan->level = pCxt->pCurrentSubplan->level + 1; if (NULL == pCxt->pCurrentSubplan->pChildern) { - pCxt->pCurrentSubplan->pChildern = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); + pCxt->pCurrentSubplan->pChildern = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); } taosArrayPush(pCxt->pCurrentSubplan->pChildern, &subplan); - subplan->pParents = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); + subplan->pParents = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); taosArrayPush(subplan->pParents, &pCxt->pCurrentSubplan); } SArray* currentLevel; if (subplan->level >= taosArrayGetSize(pCxt->pDag->pSubplans)) { - currentLevel = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); + currentLevel = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); taosArrayPush(pCxt->pDag->pSubplans, ¤tLevel); } else { currentLevel = taosArrayGetP(pCxt->pDag->pSubplans, subplan->level); @@ -272,7 +272,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; - case QNODE_INSERT: + case QNODE_MODIFY: // Insert is not an operator in a physical plan. break; default: @@ -306,7 +306,7 @@ static void splitInsertSubplan(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { } static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { - if (QNODE_INSERT == pRoot->info.type) { + if (QNODE_MODIFY == pRoot->info.type) { splitInsertSubplan(pCxt, pRoot); } else { SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); @@ -321,12 +321,12 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD TRY(TSDB_MAX_TAG_CONDITIONS) { SPlanContext context = { .pCatalog = pCatalog, - .pDag = vailidPointer(calloc(1, sizeof(SQueryDag))), + .pDag = validPointer(calloc(1, sizeof(SQueryDag))), .pCurrentSubplan = NULL, .nextId = {0} // todo queryid }; *pDag = context.pDag; - context.pDag->pSubplans = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); + context.pDag->pSubplans = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); createSubplanByLevel(&context, pQueryNode); } CATCH(code) { CLEANUP_EXECUTE(); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 29f84ddcd0..b50eb2c92d 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -267,7 +267,7 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { pOut->metaNum = 1; if (pMetaMsg->dbFname[0]) { - snprintf(pOut->tbFname, "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); + snprintf(pOut->tbFname, sizeof(pOut->tbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); } else { memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname)); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3f1799507d..20eb94c2ff 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -664,7 +664,7 @@ int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) { } SSubQueryMsg *pMsg = msg; - + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(job->queryId); pMsg->taskId = htobe64(task->taskId);