diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 5683a4358e..7247200fe7 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG cac24d3 + GIT_TAG 4a4027c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/include/client/taos.h b/include/client/taos.h index 69774b750f..379363c51d 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -149,7 +149,7 @@ DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); -const char *taos_data_type(int type); +DLL_EXPORT const char *taos_data_type(int type); DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); DLL_EXPORT TAOS_STMT *taos_stmt_init_with_reqid(TAOS *taos, int64_t reqid); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 14d58ef58c..3537ba687d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -70,6 +70,11 @@ static inline bool vnodeIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL); } + +static inline bool syncUtilUserCommit(tmsg_t msgType) { + return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; +} + /* ------------------------ OTHER DEFINITIONS ------------------------ */ // IE type #define TSDB_IE_TYPE_SEC 1 diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 88422072b8..34603fdb64 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -58,7 +58,6 @@ typedef struct SParseContext { bool isSuperUser; bool enableSysInfo; bool async; - int8_t schemalessType; const char* svrVer; bool nodeOffline; SArray* pTableMetaPos; // sql table pos => catalog data pos @@ -108,7 +107,11 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void qDestroyBoundColInfo(void* pInfo); SQuery* smlInitHandle(); -int32_t smlBindData(SQuery* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, +int32_t smlBuildRow(STableDataCxt* pTableCxt); +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void *kv, int32_t index); +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); + +int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 900bc88c41..4cf4800472 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -47,6 +47,7 @@ extern "C" { #define SYNC_HEARTBEAT_SLOW_MS 1500 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 +#define SYNC_SNAP_RESEND_MS 1000 * 60 #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 diff --git a/include/os/os.h b/include/os/os.h index 0688eeb9ad..b27fa84406 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -27,6 +27,7 @@ extern "C" { #if !defined(WINDOWS) #include +#include #include #include #include diff --git a/include/os/osDef.h b/include/os/osDef.h index 0bf9c6184e..c18728c9a7 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -120,12 +120,6 @@ void syslog(int unused, const char *format, ...); #define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) #define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) -#ifndef NDEBUG -#define ASSERT(x) assert(x) -#else -#define ASSERT(x) -#endif - #ifndef UNUSED #define UNUSED(x) ((void)(x)) #endif diff --git a/include/os/osDir.h b/include/os/osDir.h index 40012f8141..73871602c5 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -62,6 +62,7 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen); bool taosIsDir(const char *dirname); char *taosDirName(char *dirname); char *taosDirEntryBaseName(char *dirname); +void taosGetCwd(char *buf, int32_t len); TdDirPtr taosOpenDir(const char *dirname); TdDirEntryPtr taosReadDir(TdDirPtr pDir); diff --git a/include/os/osString.h b/include/os/osString.h index 4c3fbe46c3..b609c9d351 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -62,7 +62,7 @@ typedef int32_t TdUcs4; int32_t taosUcs4len(TdUcs4 *ucs4); int64_t taosStr2int64(const char *str); -void taosConvInit(void); +int32_t taosConvInit(void); void taosConvDestroy(); int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len); diff --git a/include/os/osSystem.h b/include/os/osSystem.h index eca984c41a..8f1f5c58d5 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -46,6 +46,29 @@ void taosSetTerminalMode(); int32_t taosGetOldTerminalMode(); void taosResetTerminalMode(); +#if !defined(WINDOWS) +#define taosPrintTrace(flags, level, dflag) \ + { \ + void* array[100]; \ + int32_t size = backtrace(array, 100); \ + char** strings = backtrace_symbols(array, size); \ + if (strings != NULL) { \ + taosPrintLog(flags, level, dflag, "obtained %d stack frames", size); \ + for (int32_t i = 0; i < size; i++) { \ + taosPrintLog(flags, level, dflag, "frame:%d, %s", i, strings[i]); \ + } \ + } \ + \ + taosMemoryFree(strings); \ + } +#else +#define taosPrintTrace(flags, level, dflag) \ + { \ + taosPrintLog(flags, level, dflag, \ + "backtrace not implemented on windows, so detailed stack information cannot be printed"); \ + } +#endif + #ifdef __cplusplus } #endif diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 38eb0d8fc6..b1bd09e123 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -17,6 +17,7 @@ #define _TD_UTIL_CODING_H_ #include "os.h" +#include "tlog.h" #ifdef __cplusplus extern "C" { diff --git a/include/util/tdef.h b/include/util/tdef.h index 15133630f8..2bc8ef2788 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -308,8 +308,9 @@ typedef enum ELogicConditionType { #define TSDB_MIN_DURATION_PER_FILE 60 // unit minute #define TSDB_MAX_DURATION_PER_FILE (3650 * 1440) #define TSDB_DEFAULT_DURATION_PER_FILE (10 * 1440) -#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute -#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. +#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute +#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. +#define TSDB_MAX_KEEP_NS (365 * 292 * 1440) // data in db to be reserved. #define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years #define TSDB_MIN_MINROWS_FBLOCK 10 #define TSDB_MAX_MINROWS_FBLOCK 1000 diff --git a/include/util/tlog.h b/include/util/tlog.h index 68b004cda7..e6ef7f388f 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -38,6 +38,7 @@ typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content); extern bool tsLogEmbedded; extern bool tsAsyncLog; +extern bool tsAssert; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; extern LogFp tsLogFp; @@ -82,6 +83,10 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons #endif ; +bool taosAssert(bool condition, const char *file, int32_t line, const char *format, ...); +#define ASSERTS(condition, ...) taosAssert(condition, __FILE__, __LINE__, __VA_ARGS__) +#define ASSERT(condition) ASSERTS(condition, "assert info not provided") + // clang-format off #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 9034fd85f5..6a82322a12 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -497,27 +497,27 @@ function install_service_on_systemd() { taosd_service_config="${service_config_dir}/${serverName}.service" - ${csudo}bash -c "echo '[Unit]' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'Description=${productName} server service' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'After=network-online.target' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'Wants=network-online.target' >> ${taosd_service_config}" + ${csudo}bash -c "echo [Unit] >> ${taosd_service_config}" + ${csudo}bash -c "echo Description=${productName} server service >> ${taosd_service_config}" + ${csudo}bash -c "echo After=network-online.target >> ${taosd_service_config}" + ${csudo}bash -c "echo Wants=network-online.target >> ${taosd_service_config}" ${csudo}bash -c "echo >> ${taosd_service_config}" - ${csudo}bash -c "echo '[Service]' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'Type=simple' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'ExecStart=/usr/bin/${serverName}' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'ExecStartPre=${installDir}/bin/startPre.sh' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'Restart=always' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}" + ${csudo}bash -c "echo [Service] >> ${taosd_service_config}" + ${csudo}bash -c "echo Type=simple >> ${taosd_service_config}" + ${csudo}bash -c "echo ExecStart=/usr/bin/${serverName} >> ${taosd_service_config}" + ${csudo}bash -c "echo ExecStartPre=${installDir}/bin/startPre.sh >> ${taosd_service_config}" + ${csudo}bash -c "echo TimeoutStopSec=1000000s >> ${taosd_service_config}" + ${csudo}bash -c "echo LimitNOFILE=infinity >> ${taosd_service_config}" + ${csudo}bash -c "echo LimitNPROC=infinity >> ${taosd_service_config}" + ${csudo}bash -c "echo LimitCORE=infinity >> ${taosd_service_config}" + ${csudo}bash -c "echo TimeoutStartSec=0 >> ${taosd_service_config}" + ${csudo}bash -c "echo StandardOutput=null >> ${taosd_service_config}" + ${csudo}bash -c "echo Restart=always >> ${taosd_service_config}" + ${csudo}bash -c "echo StartLimitBurst=3 >> ${taosd_service_config}" + ${csudo}bash -c "echo StartLimitInterval=60s >> ${taosd_service_config}" ${csudo}bash -c "echo >> ${taosd_service_config}" - ${csudo}bash -c "echo '[Install]' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" + ${csudo}bash -c "echo [Install] >> ${taosd_service_config}" + ${csudo}bash -c "echo WantedBy=multi-user.target >> ${taosd_service_config}" ${csudo}systemctl enable ${serverName} } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index ea76f726ea..c3b79d7829 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -149,7 +149,6 @@ typedef struct STscObj { int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; SHashObj* pRequests; - int8_t schemalessType; // todo remove it, this attribute should be move to request } STscObj; typedef struct SResultColumn { diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index 0cb36ff61d..c29f495201 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -30,7 +30,7 @@ extern "C" { #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) -//#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", 0, cDebugFlag, __VA_ARGS__); }} while(0) // clang-format on #ifdef __cplusplus diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 7564d91330..d3d713c643 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -76,13 +76,20 @@ static void deregisterRequest(SRequestObj *pRequest) { "current:%d, app current:%d", pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); + tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 + "us, exec:%" PRId64 "us, stmtType:%d", + duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, + pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - + pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd, + pRequest->stmtType); + if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) { - // tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 - // "us, exec:%" PRId64 "us", - // duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, - // pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - - // pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd); - atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); +// tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 +// "us, exec:%" PRId64 "us", +// duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, +// pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - +// pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd); +// atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { // tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 // "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%" PRIx64, @@ -265,7 +272,6 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - pObj->schemalessType = 1; atomic_add_fetch_64(&pObj->pAppInfo->numOfConns, 1); @@ -407,7 +413,9 @@ void taos_init_imp(void) { initQueryModuleMsgHandle(); - taosConvInit(); + if (taosConvInit() != 0) { + ASSERTS(0, "failed to init conv"); + } rpcInit(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 897931d5b2..299c08745e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -232,7 +232,6 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = pStmtCb, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .svrVer = pTscObj->sVer, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index f2dec7217f..9b2e97139b 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -863,7 +863,6 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = NULL, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .async = true, diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 71da09db4e..1257d74619 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -20,6 +20,19 @@ #include "ttime.h" #include "ttypes.h" +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#ifndef likely +#define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely +#define unlikely(expr) expect((expr) != 0, 0) +#endif + //================================================================================================= #define SPACE ' ' @@ -30,28 +43,29 @@ #define JUMP_SPACE(sql, sqlEnd) \ while (sql < sqlEnd) { \ - if (*sql == SPACE) \ + if (unlikely(*sql == SPACE)) \ sql++; \ else \ break; \ } // comma , -#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) +//#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) // space -#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) +//#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) // equal = -#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) +//#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) // quote " -#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) +//#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) #define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) // SLASH -#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) +//#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) #define IS_SLASH_LETTER(sql) \ - (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || *(sql) == SLASH)) \ +// (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) #define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) @@ -75,6 +89,8 @@ #define VALUE "_value" #define VALUE_LEN 6 +#define JSON_METERS_NAME "__JM" + #define BINARY_ADD_LEN 2 // "binary" 2 means " " #define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " @@ -91,17 +107,99 @@ typedef enum { SCHEMA_ACTION_CHANGE_TAG_SIZE, } ESchemaAction; +/*********************** list start *********************************/ typedef struct { - const char *measure; - const char *tags; - const char *cols; - const char *timestamp; + const void *key; + int32_t keyLen; + void *value; + bool used; +}Node; + +typedef struct NodeList{ + Node data; + struct NodeList* next; +}NodeList; + +typedef int32_t (*_equal_fn_sml)(const void *, const void *); + +static void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn){ + NodeList *tmp = list; + while(tmp){ + if(fn == NULL){ + if(tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return tmp->data.value; + } + }else{ + if(tmp->data.used && fn(tmp->data.key, key) == 0) { + return tmp->data.value; + } + } + + tmp = tmp->next; + } + return NULL; +} + +static int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn){ + NodeList *tmp = *list; + while (tmp){ + if(!tmp->data.used) break; + if(fn == NULL){ + if(tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return -1; + } + }else{ + if(tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { + return -1; + } + } + + tmp = tmp->next; + } + if(tmp){ + tmp->data.key = key; + tmp->data.keyLen = len; + tmp->data.value = value; + tmp->data.used = true; + }else{ + NodeList *newNode = taosMemoryCalloc(1, sizeof(NodeList)); + if(newNode == NULL){ + return -1; + } + newNode->data.key = key; + newNode->data.keyLen = len; + newNode->data.value = value; + newNode->data.used = true; + newNode->next = *list; + *list = newNode; + } + return 0; +} + +static int nodeListSize(NodeList* list){ + int cnt = 0; + while(list){ + if(list->data.used) cnt++; + else break; + list = list->next; + } + return cnt; +} +/*********************** list end *********************************/ + +typedef struct { + char *measure; + char *tags; + char *cols; + char *timestamp; int32_t measureLen; int32_t measureTagsLen; int32_t tagsLen; int32_t colsLen; int32_t timestampLen; + + SArray *colArray; } SSmlLineInfo; typedef struct { @@ -109,12 +207,13 @@ typedef struct { int32_t sTableNameLen; char childTableName[TSDB_TABLE_NAME_LEN]; uint64_t uid; + void *key; // for openTsdb telnet SArray *tags; - // if info->formatData is true, elements are SArray. - // if info->formatData is false, elements are SHashObj for find by key quickly + // elements are SHashObj for find by key quickly SArray *cols; + STableDataCxt *tableDataCtx; } SSmlTableInfo; typedef struct { @@ -149,26 +248,18 @@ typedef struct { int64_t endTime; } SSmlCostInfo; -typedef struct { - SRequestObj *request; - tsem_t sem; - int32_t cnt; - int32_t total; - TdThreadSpinlock lock; -} Params; - typedef struct { int64_t id; - Params *params; SMLProtocolType protocol; int8_t precision; + bool reRun; bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) bool isRawLine; int32_t ttl; - SHashObj *childTables; - SHashObj *superTables; + NodeList *childTables; + NodeList *superTables; SHashObj *pVgHash; STscObj *taos; @@ -177,25 +268,33 @@ typedef struct { SQuery *pQuery; SSmlCostInfo cost; - int32_t affectedRows; + int32_t lineNum; SSmlMsgBuf msgBuf; - SHashObj *dumplicateKey; // for dumplicate key - SArray *colsContainer; // for cols parse, if dataFormat == false cJSON *root; // for parse json + SSmlLineInfo *lines; // element is SSmlLineInfo + + // + SArray *preLineTagKV; + SArray *preLineColKV; + + SSmlLineInfo preLine; + STableMeta *currSTableMeta; + STableDataCxt *currTableDataCtx; + bool needModifySchema; } SSmlHandle; //================================================================================================= //================================================================================================= static volatile int64_t linesSmlHandleId = 0; static int64_t smlGenId() { - int64_t id; + int64_t id; - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); } while (id == 0); - return id; + return id; } static inline bool smlDoubleToInt64OverFlow(double num) { @@ -279,7 +378,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH int32_t code = TSDB_CODE_SUCCESS; for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); if (code != TSDB_CODE_SUCCESS) { return code; @@ -301,7 +400,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool i = 1; } for (; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { taosHashCleanup(hashTmp); return -1; @@ -322,7 +421,7 @@ static int32_t getBytes(uint8_t type, int32_t length) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *results, int32_t numOfCols, bool isTag) { for (int j = 0; j < taosArrayGetSize(cols); ++j) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); ESchemaAction action = SCHEMA_ACTION_NULL; smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { @@ -423,13 +522,16 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); -end: + end: destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); return code; } static int32_t smlModifyDBSchemas(SSmlHandle *info) { + if(info->dataFormat && !info->needModifySchema){ + return TSDB_CODE_SUCCESS; + } int32_t code = 0; SHashObj *hashTmp = NULL; STableMeta *pTableMeta = NULL; @@ -443,13 +545,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - SSmlSTableMeta **tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (tableMetaSml) { - SSmlSTableMeta *sTableData = *tableMetaSml; + NodeList *tmp = info->superTables; + while (tmp) { + SSmlSTableMeta *sTableData = tmp->data.value; bool needCheckMeta = false; // for multi thread - size_t superTableLen = 0; - void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); + size_t superTableLen = (size_t)tmp->data.keyLen; + const void *superTable = tmp->data.key; memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); memcpy(pName.tname, superTable, superTableLen); @@ -598,17 +700,18 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { sTableData->tableMeta = pTableMeta; - tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, tableMetaSml); + tmp = tmp->next; } return 0; -end: + end: taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); // catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); return code; } +/******************************* parse basic type function **********************/ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; @@ -747,50 +850,38 @@ static bool smlIsNchar(const char *pVal, uint16_t len) { if (len < 3) { return false; } - if ((pVal[0] == 'l' || pVal[0] == 'L') && pVal[1] == '"' && pVal[len - 1] == '"') { + if (pVal[1] == '"' && pVal[len - 1] == '"' && (pVal[0] == 'l' || pVal[0] == 'L')) { return true; } return false; } +/******************************* parse basic type function end **********************/ -static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { +/******************************* time function **********************/ +static uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, + TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, + TSDB_TIME_PRECISION_NANO}; +static int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; +static int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; +static int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL}; + +static int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) { char *endPtr = NULL; int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); - if (value + len != endPtr) { - return -1; - } - double ts = tsInt64; - switch (type) { - case TSDB_TIME_PRECISION_HOURS: - ts *= NANOSECOND_PER_HOUR; - tsInt64 *= NANOSECOND_PER_HOUR; - break; - case TSDB_TIME_PRECISION_MINUTES: - ts *= NANOSECOND_PER_MINUTE; - tsInt64 *= NANOSECOND_PER_MINUTE; - break; - case TSDB_TIME_PRECISION_SECONDS: - ts *= NANOSECOND_PER_SEC; - tsInt64 *= NANOSECOND_PER_SEC; - break; - case TSDB_TIME_PRECISION_MILLI: - ts *= NANOSECOND_PER_MSEC; - tsInt64 *= NANOSECOND_PER_MSEC; - break; - case TSDB_TIME_PRECISION_MICRO: - ts *= NANOSECOND_PER_USEC; - tsInt64 *= NANOSECOND_PER_USEC; - break; - case TSDB_TIME_PRECISION_NANO: - break; - default: - ASSERT(0); - } - if (ts >= (double)INT64_MAX || ts < 0) { + if (unlikely(value + len != endPtr)) { return -1; } - return tsInt64; + if(unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)){ + int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS]; + if(unit > INT64_MAX / tsInt64){ + return -1; + } + tsInt64 *= unit; + fromPrecision = TSDB_TIME_PRECISION_MILLI; + } + + return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } static int8_t smlGetTsTypeByLen(int32_t len) { @@ -803,39 +894,17 @@ static int8_t smlGetTsTypeByLen(int32_t len) { } } -static int8_t smlGetTsTypeByPrecision(int8_t precision) { - switch (precision) { - case TSDB_SML_TIMESTAMP_HOURS: - return TSDB_TIME_PRECISION_HOURS; - case TSDB_SML_TIMESTAMP_MILLI_SECONDS: - return TSDB_TIME_PRECISION_MILLI; - case TSDB_SML_TIMESTAMP_NANO_SECONDS: - case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: - return TSDB_TIME_PRECISION_NANO; - case TSDB_SML_TIMESTAMP_MICRO_SECONDS: - return TSDB_TIME_PRECISION_MICRO; - case TSDB_SML_TIMESTAMP_SECONDS: - return TSDB_TIME_PRECISION_SECONDS; - case TSDB_SML_TIMESTAMP_MINUTES: - return TSDB_TIME_PRECISION_MINUTES; - default: - return -1; - } -} - static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { - if (len == 0 || (len == 1 && data[0] == '0')) { - return taosGetTimestampNs(); + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if(unlikely(len == 0 || (len == 1 && data[0] == '0'))){ + return taosGetTimestampNs()/smlFactorNS[toPrecision]; } - int8_t tsType = smlGetTsTypeByPrecision(info->precision); - if (tsType == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL); - return -1; - } + uint8_t fromPrecision = smlPrecisionConvert[info->precision]; - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); return -1; } @@ -843,28 +912,30 @@ static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t le } static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { - if (!data) { + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if (unlikely(!data)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); return -1; } - if (len == 1 && data[0] == '0') { - return taosGetTimestampNs(); + if (unlikely(len == 1 && data[0] == '0')) { + return taosGetTimestampNs()/smlFactorNS[toPrecision]; } - int8_t tsType = smlGetTsTypeByLen(len); - if (tsType == -1) { + uint8_t fromPrecision = smlGetTsTypeByLen(len); + if (unlikely(fromPrecision == -1)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); return -1; } - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); return -1; } return ts; } -static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) { +static int64_t smlParseTS(SSmlHandle *info, const char *data, int32_t len) { int64_t ts = 0; if (info->protocol == TSDB_SML_LINE_PROTOCOL) { // uError("SML:data:%s,len:%d", data, len); @@ -876,27 +947,175 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra } uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts); - if (ts <= 0) { - uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); - return TSDB_CODE_INVALID_TIMESTAMP; + return ts; +} +/******************************* time function end **********************/ + +/******************************* Sml struct related function **********************/ +static SSmlTableInfo *smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen) { + SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); + if (!tag) { + return NULL; } - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; + tag->sTableName = measure; + tag->sTableNameLen = measureLen; + + tag->cols = taosArrayInit(numRows, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:smlBuildTableInfo failed to allocate memory"); + goto cleanup; } - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = ts; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); + tag->tags = taosArrayInit(16, sizeof(SSmlKv)); + if (tag->tags == NULL) { + uError("SML:smlBuildTableInfo failed to allocate memory"); + goto cleanup; + } + return tag; + + cleanup: + taosMemoryFree(tag); + return NULL; +} + +static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){ + for(int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = taosArrayGet(tags, i); + if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) { + smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key); + return TSDB_CODE_TSC_DUP_NAMES; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) { + SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + int ret = smlCheckDupUnit(dumplicateKey, cols, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; + } + ret = smlCheckDupUnit(dumplicateKey, tags, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; + } + + end: + taosHashCleanup(dumplicateKey); + return ret; +} + +static int32_t smlParseTableName(SArray *tags, char *childTableName) { + size_t childTableNameLen = strlen(tsSmlChildTableName); + if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; + + for(int i = 0; i < taosArrayGetSize(tags); i++){ + SSmlKv *tag = taosArrayGet(tags, i); + // handle child table name + if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + break; + } + } return TSDB_CODE_SUCCESS; } +static int32_t smlSetCTableName(SSmlTableInfo *oneTable){ + smlParseTableName(oneTable->tags, oneTable->childTableName); + + if (strlen(oneTable->childTableName) == 0) { + SArray* dst = taosArrayDup(oneTable->tags, NULL); + RandTableName rName = {dst, oneTable->sTableName, (uint8_t)oneTable->sTableNameLen, + oneTable->childTableName, 0}; + + buildChildTableName(&rName); + taosArrayDestroy(dst); + oneTable->uid = rName.uid; + } else { + oneTable->uid = *(uint64_t *)(oneTable->childTableName); + } + return TSDB_CODE_SUCCESS; +} + +static SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { + SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + if (!meta) { + return NULL; + } + + if(unlikely(!isDataFormat)){ + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->tagHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + + meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->colHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + } + + meta->tags = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->tags == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + + meta->cols = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->cols == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + return meta; + + cleanup: + taosMemoryFree(meta); + return NULL; +} + +static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols){ + for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + taosArrayPush(metaArray, kv); + if(unlikely(metaHash != NULL)) { + taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); + } + } +} + +static STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen){ + STableMeta *pTableMeta = NULL; + + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); + + SRequestConnInfo conn = {0}; + conn.pTrans = info->taos->pAppInfo->pTransporter; + conn.requestId = info->pRequest->requestId; + conn.requestObjRefId = info->pRequest->self; + conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + memcpy(pName.tname, measure, measureLen); + + catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + return pTableMeta; +} + +static void smlDestroySTableMeta(SSmlSTableMeta *meta) { + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->colHash); + taosArrayDestroy(meta->tags); + taosArrayDestroy(meta->cols); + taosMemoryFree(meta->tableMeta); + taosMemoryFree(meta); +} +/******************************* Sml struct related function end **********************/ + static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { // binary if (smlIsBinary(pVal->value, pVal->length)) { @@ -934,10 +1153,437 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return TSDB_CODE_TSC_INVALID_VALUE; } -static int32_t smlParseInfluxString(const char *sql, const char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { +int32_t is_same_child_table_json(const void *a, const void *b){ + return (cJSON_Compare((const cJSON *)a, (const cJSON *)b, true)) ? 0 : 1; +} + +int32_t is_same_child_table_telnet(const void *a, const void *b){ + SSmlLineInfo *t1 = (SSmlLineInfo *)a; + SSmlLineInfo *t2 = (SSmlLineInfo *)b; + return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) + && ((t1->tagsLen == t2->tagsLen) && memcmp(t1->tags, t2->tags, t1->tagsLen) == 0)) ? 0 : 1; +} + +#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) + +#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) + +#define IS_SAME_KEY (preKV->keyLen == kv.keyLen && memcmp(preKV->key, kv.key, kv.keyLen) == 0) + +static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, + SSmlLineInfo* currElement, bool isSameMeasure, bool isSameCTable){ + if(isSameCTable){ + return TSDB_CODE_SUCCESS; + } + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(!isSameMeasure){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + }else{ + taosArraySetSize(preLineKV, 0); + } + + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + int32_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + int32_t valueLen = 0; + hasSlash = false; + while (*sql < sqlEnd) { + // parse value + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + }else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + + if(unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + SSmlKv kv = {.key = key, .type = TSDB_DATA_TYPE_NCHAR, .keyLen = keyLen, .value = value, .length = valueLen}; + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = taosArrayGet(preLineKV, cnt); + if(unlikely(kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = taosArrayGet(superKV, cnt); + if(unlikely(kv.length > preKV->length)) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + taosArrayPush(preLineKV, &kv); + } + + cnt++; + if(IS_SPACE(*sql)){ + break; + } + (*sql)++; + } + + void* oneTable = nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL); + if ((oneTable != NULL)) { + return TSDB_CODE_SUCCESS; + } + + SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen); + if (!tinfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for(int i = 0; i < taosArrayGetSize(preLineKV); i++){ + taosArrayPush(tinfo->tags, taosArrayGet(preLineKV, i)); + } + smlSetCTableName(tinfo); + if(info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if(tinfo->tableDataCtx == NULL){ + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + nodeListSet(&info->childTables, currElement->measure, currElement->measureTagsLen, tinfo, NULL); + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, + SSmlLineInfo* currElement, bool isSameMeasure, bool isSameCTable){ + int cnt = 0; + SArray *preLineKV = info->preLineColKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameCTable)){ + SSmlTableInfo *oneTable = (SSmlTableInfo *)nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL); + if (unlikely(oneTable == NULL)) { + smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure); + return TSDB_CODE_SML_INVALID_DATA; + } + info->currTableDataCtx = oneTable->tableDataCtx; + } + + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->cols; + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + } + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + int32_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + int32_t valueLen = 0; + hasSlash = false; + bool isInQuote = false; + while (*sql < sqlEnd) { + // parse value + if (IS_QUOTE(*sql)) { + isInQuote = !isInQuote; + (*sql)++; + continue; + } + if (!isInQuote){ + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + } else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(isInQuote)) { + smlBuildInvalidDataMsg(&info->msgBuf, "only one quote", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; + int32_t ret = smlParseValue(&kv, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + if(info->dataFormat){ + //cnt begin 0, add ts so + 2 + if(unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + // bind data + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("smlBuildCol error, retry"); + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(cnt >= taosArrayGetSize(preLineKV)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = taosArrayGet(preLineKV, cnt); + if(kv.type != preKV->type){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = taosArrayGet(tableMeta->cols, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = taosArrayGet(superKV, cnt); + if(unlikely(kv.type != preKV->type)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(IS_VAR_DATA_TYPE(kv.type)){ + if(kv.length > preKV->length) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + if(currElement->colArray == NULL){ + currElement->colArray = taosArrayInit(16, sizeof(SSmlKv)); + taosArraySetSize(currElement->colArray, 1); + } + taosArrayPush(currElement->colArray, &kv); //reserve for timestamp + } + + cnt++; + if(IS_SPACE(*sql)){ + break; + } + (*sql)++; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { if (!sql) return TSDB_CODE_SML_INVALID_DATA; JUMP_SPACE(sql, sqlEnd) - if (*sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; + if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA; elements->measure = sql; // parse measure @@ -957,47 +1603,62 @@ static int32_t smlParseInfluxString(const char *sql, const char *sqlEnd, SSmlLin sql++; } elements->measureLen = sql - elements->measure; - if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - smlBuildInvalidDataMsg(msg, "measure is empty or too large than 192", NULL); + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } - // parse tag - if (*sql == SPACE) { - elements->tagsLen = 0; - } else { - if (*sql == COMMA) sql++; - elements->tags = sql; - while (sql < sqlEnd) { - if (IS_SPACE(sql)) { - break; - } - sql++; + // to get measureTagsLen before + const char* tmp = sql; + while (tmp < sqlEnd){ + if (IS_SPACE(tmp)) { + break; } - elements->tagsLen = sql - elements->tags; + tmp++; } - elements->measureTagsLen = sql - elements->measure; + elements->measureTagsLen = tmp - elements->measure; + + bool isSameCTable = false; + bool isSameMeasure = false; + if(IS_SAME_CHILD_TABLE){ + isSameCTable = true; + isSameMeasure = true; + }else if(info->dataFormat) { + isSameMeasure = IS_SAME_SUPER_TABLE; + } + // parse tag + if (*sql == COMMA) sql++; + elements->tags = sql; + + // tinfo != NULL means child table has never occur before + int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if(unlikely(ret != TSDB_CODE_SUCCESS)){ + return ret; + } + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + sql = elements->measure + elements->measureTagsLen; + + elements->tagsLen = sql - elements->tags; // parse cols JUMP_SPACE(sql, sqlEnd) elements->cols = sql; - bool isInQuote = false; - while (sql < sqlEnd) { - if (IS_QUOTE(sql)) { - isInQuote = !isInQuote; - } - if (!isInQuote && IS_SPACE(sql)) { - break; - } - sql++; + + ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if(unlikely(ret != TSDB_CODE_SUCCESS)){ + return ret; } - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", elements->cols); - return TSDB_CODE_SML_INVALID_DATA; + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; } + elements->colsLen = sql - elements->cols; - if (elements->colsLen == 0) { - smlBuildInvalidDataMsg(msg, "cols is empty", NULL); + if (unlikely(elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "cols is empty", NULL); return TSDB_CODE_SML_INVALID_DATA; } @@ -1012,14 +1673,29 @@ static int32_t smlParseInfluxString(const char *sql, const char *sqlEnd, SSmlLin } elements->timestampLen = sql - elements->timestamp; - return TSDB_CODE_SUCCESS; + int64_t ts = smlParseTS(info, elements->timestamp, elements->timestampLen); + if (ts <= 0) { + uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); + return TSDB_CODE_INVALID_TIMESTAMP; + } + // add ts to + SSmlKv kv = { .key = TS, .keyLen = TS_LEN, .i = ts, .type = TSDB_DATA_TYPE_TIMESTAMP, .length = (int16_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + if(info->dataFormat){ + smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); + smlBuildRow(info->currTableDataCtx); + }else{ + taosArraySet(elements->colArray, 0, &kv); + } + info->preLine = *elements; + + return ret; } -static void smlParseTelnetElement(const char **sql, const char *sqlEnd, const char **data, int32_t *len) { +static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t *len) { while (*sql < sqlEnd) { - if (**sql != SPACE && !(*data)) { + if (unlikely((**sql != SPACE && !(*data)))) { *data = *sql; - } else if (**sql == SPACE && *data) { + } else if (unlikely(**sql == SPACE && *data)) { *len = *sql - *data; break; } @@ -1027,25 +1703,59 @@ static void smlParseTelnetElement(const char **sql, const char *sqlEnd, const ch } } -static int32_t smlParseTelnetTags(const char *data, const char *sqlEnd, SArray *cols, char *childTableName, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; +static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { + if(is_same_child_table_telnet(elements, &info->preLine) == 0){ + return TSDB_CODE_SUCCESS; + } + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(!isSameMeasure){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + }else{ + taosArraySetSize(preLineKV, 0); + } + const char *sql = data; - size_t childTableNameLen = strlen(tsSmlChildTableName); while (sql < sqlEnd) { JUMP_SPACE(sql, sqlEnd) - if (*sql == '\0') break; + if (unlikely(*sql == '\0')) break; const char *key = sql; int32_t keyLen = 0; // parse key while (sql < sqlEnd) { - if (*sql == SPACE) { + if (unlikely(*sql == SPACE)) { smlBuildInvalidDataMsg(msg, "invalid data", sql); return TSDB_CODE_SML_INVALID_DATA; } - if (*sql == EQUAL) { + if (unlikely(*sql == EQUAL)) { keyLen = sql - key; sql++; break; @@ -1053,24 +1763,24 @@ static int32_t smlParseTelnetTags(const char *data, const char *sqlEnd, SArray * sql++; } - if (IS_INVALID_COL_LEN(keyLen)) { + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; - } +// if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { +// smlBuildInvalidDataMsg(msg, "dumplicate key", key); +// return TSDB_CODE_TSC_DUP_NAMES; +// } // parse value const char *value = sql; int32_t valueLen = 0; while (sql < sqlEnd) { // parse value - if (*sql == SPACE) { + if (unlikely(*sql == SPACE)) { break; } - if (*sql == EQUAL) { + if (unlikely(*sql == EQUAL)) { smlBuildInvalidDataMsg(msg, "invalid data", sql); return TSDB_CODE_SML_INVALID_DATA; } @@ -1078,217 +1788,213 @@ static int32_t smlParseTelnetTags(const char *data, const char *sqlEnd, SArray * } valueLen = sql - value; - if (valueLen == 0) { + if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(msg, "invalid value", value); return TSDB_CODE_TSC_INVALID_VALUE; } - // handle child table name - if (childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } - - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - kv->type = TSDB_DATA_TYPE_NCHAR; + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen, .type = TSDB_DATA_TYPE_NCHAR}; - taosArrayPush(cols, &kv); + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = taosArrayGet(preLineKV, cnt); + if(unlikely(kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = taosArrayGet(superKV, cnt); + if(unlikely(kv.length > preKV->length)) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + taosArrayPush(preLineKV, &kv); + } + cnt++; } + SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + if (unlikely(tinfo == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (!tinfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for(int i = 0; i < taosArrayGetSize(preLineKV); i++){ + taosArrayPush(tinfo->tags, taosArrayGet(preLineKV, i)); + } + smlSetCTableName(tinfo); + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + SSmlLineInfo *key = taosMemoryMalloc(sizeof(SSmlLineInfo)); + *key = *elements; + tinfo->key = key; + nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); + } + info->currTableDataCtx = tinfo->tableDataCtx; return TSDB_CODE_SUCCESS; } // format: =[ =] -static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, const char *sqlEnd, SSmlTableInfo *tinfo, - SArray *cols) { +static int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { if (!sql) return TSDB_CODE_SML_INVALID_DATA; // parse metric - smlParseTelnetElement(&sql, sqlEnd, &tinfo->sTableName, &tinfo->sTableNameLen); - if (!(tinfo->sTableName) || IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { + smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen); + if (unlikely((!(elements->measure) || IS_INVALID_TABLE_LEN(elements->measureLen)))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } // parse timestamp - const char *timestamp = NULL; - int32_t tLen = 0; - smlParseTelnetElement(&sql, sqlEnd, ×tamp, &tLen); - if (!timestamp || tLen == 0) { + smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen); + if (unlikely(!elements->timestamp || elements->timestampLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); return TSDB_CODE_SML_INVALID_DATA; } - int32_t ret = smlParseTS(info, timestamp, tLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return ret; + bool needConverTime = false; // get TS before parse tag(get meta), so need conver time + if(info->dataFormat && info->currSTableMeta == NULL){ + needConverTime = true; } + int64_t ts = smlParseTS(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts < 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .i = ts, .type = TSDB_DATA_TYPE_TIMESTAMP, .length = (int16_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; // parse value - const char *value = NULL; - int32_t valueLen = 0; - smlParseTelnetElement(&sql, sqlEnd, &value, &valueLen); - if (!value || valueLen == 0) { + smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); + if (unlikely(!elements->cols || elements->colsLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); return TSDB_CODE_TSC_INVALID_VALUE; } - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(cols, &kv); - kv->key = VALUE; - kv->keyLen = VALUE_LEN; - kv->value = value; - kv->length = valueLen; - if ((ret = smlParseValue(kv, &info->msgBuf)) != TSDB_CODE_SUCCESS) { + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = elements->colsLen}; + if (smlParseNumber(&kv, &info->msgBuf)) { + kv.length = (int16_t)tDataTypes[kv.type].bytes; + }else{ + return TSDB_CODE_TSC_INVALID_VALUE; + } + JUMP_SPACE(sql, sqlEnd) + + elements->tags = sql; + elements->tagsLen = sqlEnd - sql; + + int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } - // parse tags - ret = smlParseTelnetTags(sql, sqlEnd, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return ret; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *childTableName, bool isTag, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (len == 0) { + if(unlikely(info->reRun)){ return TSDB_CODE_SUCCESS; } - size_t childTableNameLen = strlen(tsSmlChildTableName); - const char *sql = data; - while (sql < data + len) { - const char *key = sql; - int32_t keyLen = 0; - - while (sql < data + len) { - // parse key - if (IS_COMMA(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - if (IS_EQUAL(sql)) { - keyLen = sql - key; - sql++; - break; - } - sql++; + if(info->dataFormat){ + if(needConverTime) { + kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); } - - if (IS_INVALID_COL_LEN(keyLen)) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); } - - // parse value - const char *value = sql; - int32_t valueLen = 0; - bool isInQuote = false; - while (sql < data + len) { - // parse value - if (!isTag && IS_QUOTE(sql)) { - isInQuote = !isInQuote; - sql++; - continue; - } - if (!isInQuote && IS_COMMA(sql)) { - break; - } - if (!isInQuote && IS_EQUAL(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - sql++; + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; } - valueLen = sql - value; - sql++; - - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", value); - return TSDB_CODE_SML_INVALID_DATA; - } - if (valueLen == 0) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_SML_INVALID_DATA; - } - PROCESS_SLASH(key, keyLen) - PROCESS_SLASH(value, valueLen) - - // handle child table name - if (childTableName && childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } - - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - if (cols) taosArrayPush(cols, &kv); - - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - if (isTag) { - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - kv->type = TSDB_DATA_TYPE_NCHAR; - } else { - int32_t ret = smlParseValue(kv, msg); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); } + info->preLine = *elements; return TSDB_CODE_SUCCESS; } -static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg) { +static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); if (index) { - SSmlKv **value = (SSmlKv **)taosArrayGet(metaArray, *index); - if (kv->type != (*value)->type) { + SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); + if (isTag){ + if (kv->length > value->length) { + value->length = kv->length; + } + continue; + } + if (kv->type != value->type) { smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); return TSDB_CODE_SML_NOT_SAME_TYPE; - } else { - if (IS_VAR_DATA_TYPE(kv->type)) { // update string len, if bigger - if (kv->length > (*value)->length) { - *value = kv; - } - } + } + + if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) { // update string len, if bigger + value->length = kv->length; } } else { size_t tmp = taosArrayGetSize(metaArray); ASSERT(tmp <= INT16_MAX); int16_t size = tmp; - taosArrayPush(metaArray, &kv); + taosArrayPush(metaArray, kv); taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); } } @@ -1296,374 +2002,189 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols return TSDB_CODE_SUCCESS; } -static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { - for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosArrayPush(metaArray, &kv); - taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); - } -} - -static SSmlTableInfo *smlBuildTableInfo() { - SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); - if (!tag) { - return NULL; +static void smlDestroyTableInfo(SSmlTableInfo *tag) { + for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { + SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); + taosHashCleanup(kvHash); } - tag->cols = taosArrayInit(16, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } - - tag->tags = taosArrayInit(16, POINTER_BYTES); - if (tag->tags == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } - return tag; - -cleanup: - taosMemoryFree(tag); - return NULL; -} - -static void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { - if (info->dataFormat) { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SArray *kvArray = (SArray *)taosArrayGetP(tag->cols, i); - for (int j = 0; j < taosArrayGetSize(kvArray); ++j) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(kvArray, j); - taosMemoryFree(p); - } - taosArrayDestroy(kvArray); - } - } else { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); - void **p1 = (void **)taosHashIterate(kvHash, NULL); - while (p1) { - taosMemoryFree(*p1); - p1 = (void **)taosHashIterate(kvHash, p1); - } - taosHashCleanup(kvHash); - } - } - for (size_t i = 0; i < taosArrayGetSize(tag->tags); i++) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(tag->tags, i); - taosMemoryFree(p); - } + taosMemoryFree(tag->key); taosArrayDestroy(tag->cols); taosArrayDestroy(tag->tags); taosMemoryFree(tag); } -static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) { - SArray *s1 = *(SArray **)key1; - SArray *s2 = *(SArray **)key2; - SSmlKv *kv1 = (SSmlKv *)taosArrayGetP(s1, 0); - SSmlKv *kv2 = (SSmlKv *)taosArrayGetP(s2, 0); - ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; - } -} - -static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { - SHashObj *s1 = *(SHashObj **)key1; - SHashObj *s2 = *(SHashObj **)key2; - SSmlKv **kv1pp = (SSmlKv **)taosHashGet(s1, TS, TS_LEN); - SSmlKv **kv2pp = (SSmlKv **)taosHashGet(s2, TS, TS_LEN); - if (!kv1pp || !kv2pp) { - uError("smlKvTimeHashCompare kv is null"); - return -1; - } - SSmlKv *kv1 = *kv1pp; - SSmlKv *kv2 = *kv2pp; - if (!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv1"); - return -1; - } - if (!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv2"); - return -1; - } - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; - } -} - -static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *cols) { - if (dataFormat) { - void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &cols); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); - } - return TSDB_CODE_SUCCESS; - } - +static int32_t smlPushCols(SArray *colsArray, SArray *cols) { SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (!kvHash) { uError("SML:smlDealCols failed to allocate memory"); return TSDB_CODE_OUT_OF_MEMORY; } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } - void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &kvHash); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash); - } + taosArrayPush(colsArray, &kvHash); return TSDB_CODE_SUCCESS; } -static SSmlSTableMeta *smlBuildSTableMeta() { - SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); - if (!meta) { - return NULL; - } - meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->tagHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } - - meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->colHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } - - meta->tags = taosArrayInit(32, POINTER_BYTES); - if (meta->tags == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } - - meta->cols = taosArrayInit(32, POINTER_BYTES); - if (meta->cols == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } - return meta; - -cleanup: - taosMemoryFree(meta); - return NULL; -} - -static void smlDestroySTableMeta(SSmlSTableMeta *meta) { - taosHashCleanup(meta->tagHash); - taosHashCleanup(meta->colHash); - taosArrayDestroy(meta->tags); - taosArrayDestroy(meta->cols); - taosMemoryFree(meta->tableMeta); - taosMemoryFree(meta); -} - -static void smlDestroyCols(SArray *cols) { - if (!cols) return; - for (int i = 0; i < taosArrayGetSize(cols); ++i) { - void *kv = taosArrayGetP(cols, i); - taosMemoryFree(kv); - } -} - -static void smlDestroyInfo(SSmlHandle *info) { +void smlDestroyInfo(SSmlHandle *info) { if (!info) return; qDestroyQuery(info->pQuery); // destroy info->childTables - void **p1 = (void **)taosHashIterate(info->childTables, NULL); - while (p1) { - smlDestroyTableInfo(info, (SSmlTableInfo *)(*p1)); - p1 = (void **)taosHashIterate(info->childTables, p1); + NodeList* tmp = info->childTables; + while (tmp) { + if(tmp->data.used) { + smlDestroyTableInfo(tmp->data.value); + } + NodeList* t = tmp->next; + taosMemoryFree(tmp); + tmp = t; } - taosHashCleanup(info->childTables); // destroy info->superTables - p1 = (void **)taosHashIterate(info->superTables, NULL); - while (p1) { - smlDestroySTableMeta((SSmlSTableMeta *)(*p1)); - p1 = (void **)taosHashIterate(info->superTables, p1); + tmp = info->superTables; + while (tmp) { + if(tmp->data.used) { + smlDestroySTableMeta(tmp->data.value); + } + NodeList* t = tmp->next; + taosMemoryFree(tmp); + tmp = t; } - taosHashCleanup(info->superTables); // destroy info->pVgHash taosHashCleanup(info->pVgHash); - taosHashCleanup(info->dumplicateKey); - if (!info->dataFormat) { - taosArrayDestroy(info->colsContainer); + + taosArrayDestroy(info->preLineTagKV); + taosArrayDestroy(info->preLineColKV); + + if(!info->dataFormat){ + for(int i = 0; i < info->lineNum; i++){ + taosArrayDestroy(info->lines[i].colArray); + } + taosMemoryFree(info->lines); } - destroyRequest(info->pRequest); cJSON_Delete(info->root); taosMemoryFreeClear(info); } -static SSmlHandle *smlBuildSmlInfo(STscObj *pTscObj, SRequestObj *request, SMLProtocolType protocol, int8_t precision) { +static SSmlHandle *smlBuildSmlInfo(TAOS *taos) { int32_t code = TSDB_CODE_SUCCESS; SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); if (NULL == info) { return NULL; } - info->id = smlGenId(); - - info->pQuery = smlInitHandle(); - - if (pTscObj) { - info->taos = pTscObj; - code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto cleanup; - } + info->taos = acquireTscObj(*(int64_t *)taos); + code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); + goto cleanup; } - - info->precision = precision; - info->protocol = protocol; - if (protocol == TSDB_SML_LINE_PROTOCOL) { - info->dataFormat = tsSmlDataFormat; - } else { - info->dataFormat = true; - } - - if (request) { - info->pRequest = request; - info->msgBuf.buf = info->pRequest->msgBuf; - info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; - } - - info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + info->id = smlGenId(); + info->pQuery = smlInitHandle(); + info->dataFormat = true; - info->dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!info->dataFormat) { - info->colsContainer = taosArrayInit(32, POINTER_BYTES); - if (NULL == info->colsContainer) { - uError("SML:0x%" PRIx64 " create info failed", info->id); - goto cleanup; - } - } - if (NULL == info->pQuery || NULL == info->childTables || NULL == info->superTables || NULL == info->pVgHash || - NULL == info->dumplicateKey) { - uError("SML:0x%" PRIx64 " create info failed", info->id); + info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); + info->preLineColKV = taosArrayInit(8, sizeof(SSmlKv)); + + if (NULL == info->pVgHash) { + uError("create SSmlHandle failed"); goto cleanup; } return info; + cleanup: smlDestroyInfo(info); return NULL; } /************* TSDB_SML_JSON_PROTOCOL function start **************/ -static int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo) { +static int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { cJSON *metric = cJSON_GetObjectItem(root, "metric"); if (!cJSON_IsString(metric)) { return TSDB_CODE_TSC_INVALID_JSON; } - tinfo->sTableNameLen = strlen(metric->valuestring); - if (IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { + elements->measureLen = strlen(metric->valuestring); + if (IS_INVALID_TABLE_LEN(elements->measureLen)) { uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } - tinfo->sTableName = metric->valuestring; + elements->measure = metric->valuestring; return TSDB_CODE_SUCCESS; } -static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsVal) { +static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { int32_t size = cJSON_GetArraySize(root); - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; + if (unlikely(size != OTD_JSON_SUB_FIELDS_NUM)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; } cJSON *value = cJSON_GetObjectItem(root, "value"); - if (!cJSON_IsNumber(value)) { - return TSDB_CODE_TSC_INVALID_JSON; + if (unlikely(!cJSON_IsNumber(value))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; } cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; + if (unlikely(!cJSON_IsString(type))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; } double timeDouble = value->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { + if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; + return -1; } if (timeDouble == 0) { - *tsVal = taosGetTimestampNs(); - return TSDB_CODE_SUCCESS; + return taosGetTimestampNs()/smlFactorNS[toPrecision]; } if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; + return timeDouble; } - *tsVal = timeDouble; + int64_t tsInt64 = timeDouble; size_t typeLen = strlen(type->valuestring); if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { // seconds - *tsVal = *tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; + int8_t fromPrecision = TSDB_TIME_PRECISION_SECONDS; + if(smlFactorS[toPrecision] < INT64_MAX / tsInt64){ + return tsInt64 * smlFactorS[toPrecision]; } + return -1; } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { switch (type->valuestring[0]) { case 'm': case 'M': // milliseconds - *tsVal = *tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MILLI, toPrecision); break; case 'u': case 'U': // microseconds - *tsVal = *tsVal * NANOSECOND_PER_USEC; - timeDouble = timeDouble * NANOSECOND_PER_USEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MICRO, toPrecision); break; case 'n': case 'N': + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_NANO, toPrecision); break; default: - return TSDB_CODE_TSC_INVALID_JSON; + return -1; } } else { - return TSDB_CODE_TSC_INVALID_JSON; + return -1; } - - return TSDB_CODE_SUCCESS; } static uint8_t smlGetTimestampLen(int64_t num) { @@ -1675,66 +2196,49 @@ static uint8_t smlGetTimestampLen(int64_t num) { return len; } -static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { +static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root) { // Timestamp must be the first KV to parse - int64_t tsVal = 0; - + int32_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); if (cJSON_IsNumber(timestamp)) { // timestamp value 0 indicates current system time double timeDouble = timestamp->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { + if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; + return -1; } - if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; + if (unlikely(timeDouble < 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp is negative", NULL); + return timeDouble; + }else if (unlikely(timeDouble == 0)) { + return taosGetTimestampNs()/smlFactorNS[toPrecision]; } uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); - tsVal = (int64_t)timeDouble; - if (tsLen == TSDB_TIME_PRECISION_SEC_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; + int8_t fromPrecision = smlGetTsTypeByLen(tsLen); + if (unlikely(fromPrecision == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", NULL); + return -1; + } + int64_t tsInt64 = timeDouble; + if(fromPrecision == TSDB_TIME_PRECISION_SECONDS){ + if(smlFactorS[toPrecision] < INT64_MAX / tsInt64){ + return tsInt64 * smlFactorS[toPrecision]; } - } else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (timeDouble == 0) { - tsVal = taosGetTimestampNs(); - } else { - return TSDB_CODE_INVALID_TIMESTAMP; + return -1; + }else{ + return convertTimePrecision(timeDouble, fromPrecision, toPrecision); } } else if (cJSON_IsObject(timestamp)) { - int32_t ret = smlParseTSFromJSONObj(info, timestamp, &tsVal); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Failed to parse timestamp from JSON Obj", info->id); - return ret; - } + return smlParseTSFromJSONObj(info, timestamp, toPrecision); } else { - return TSDB_CODE_TSC_INVALID_JSON; + smlBuildInvalidDataMsg(&info->msgBuf, + "invalidate json", NULL); + return -1; } - - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; - } - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = tsVal; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); - return TSDB_CODE_SUCCESS; } static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { @@ -1930,323 +2434,311 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; +static int32_t smlParseColsFromJSON(cJSON *root, SSmlKv *kv) { cJSON *metricVal = cJSON_GetObjectItem(root, "value"); if (metricVal == NULL) { return TSDB_CODE_TSC_INVALID_JSON; } - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; - } - taosArrayPush(cols, &kv); - - kv->key = VALUE; - kv->keyLen = VALUE_LEN; int32_t ret = smlParseValueFromJSON(metricVal, kv); if (ret != TSDB_CODE_SUCCESS) { return ret; } + return TSDB_CODE_SUCCESS; } -static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, - SSmlMsgBuf *msg) { +static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { int32_t ret = TSDB_CODE_SUCCESS; - if (!pKVs) { - return TSDB_CODE_OUT_OF_MEMORY; - } + cJSON *tags = cJSON_GetObjectItem(root, "tags"); - if (tags == NULL || tags->type != cJSON_Object) { + if (unlikely(tags == NULL || tags->type != cJSON_Object)) { return TSDB_CODE_TSC_INVALID_JSON; } - size_t childTableNameLen = strlen(tsSmlChildTableName); + // add measure to tags to identify one child table + cJSON *cMeasure = cJSON_AddStringToObject(tags, JSON_METERS_NAME, elements->measure); + if(unlikely(cMeasure == NULL)){ + return TSDB_CODE_TSC_INVALID_JSON; + } + elements->tags = (char*)tags; + if(is_same_child_table_json(elements->tags, info->preLine.tags) == 0){ + cJSON_DeleteItemFromObjectCaseSensitive(tags, JSON_METERS_NAME); + return TSDB_CODE_SUCCESS; + } + cJSON_DeleteItemFromObjectCaseSensitive(tags, JSON_METERS_NAME); + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + }else{ + taosArraySetSize(preLineKV, 0); + } + int32_t tagNum = cJSON_GetArraySize(tags); for (int32_t i = 0; i < tagNum; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); - if (tag == NULL) { + if (unlikely(tag == NULL)) { return TSDB_CODE_TSC_INVALID_JSON; } + if(unlikely(tag == cMeasure)) continue; size_t keyLen = strlen(tag->string); - if (IS_INVALID_COL_LEN(keyLen)) { + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { uError("OTD:Tag key length is 0 or too large than 64"); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } - // check duplicate keys - if (smlCheckDuplicateKey(tag->string, keyLen, dumplicateKey)) { - return TSDB_CODE_TSC_DUP_NAMES; - } - - // handle child table name - if (childTableNameLen != 0 && strcmp(tag->string, tsSmlChildTableName) == 0) { - if (!cJSON_IsString(tag)) { - uError("OTD:ID must be JSON string"); - return TSDB_CODE_TSC_INVALID_JSON; - } - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); - continue; - } // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(pKVs, &kv); - - // key - kv->keyLen = keyLen; - kv->key = tag->string; - + SSmlKv kv ={.key = tag->string, .keyLen = keyLen}; // value - ret = smlParseValueFromJSON(tag, kv); - if (ret != TSDB_CODE_SUCCESS) { + ret = smlParseValueFromJSON(tag, &kv); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } + + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = taosArrayGet(preLineKV, cnt); + if(unlikely(kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = taosArrayGet(superKV, cnt); + if(unlikely(kv.length > preKV->length)) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + taosArrayPush(preLineKV, &kv); + } + cnt++; } + SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->tags, POINTER_BYTES, is_same_child_table_json); + if (unlikely(tinfo == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int i = 0; i < taosArrayGetSize(preLineKV); i++) { + taosArrayPush(tinfo->tags, taosArrayGet(preLineKV, i)); + } + smlSetCTableName(tinfo); + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + nodeListSet(&info->childTables, tags, POINTER_BYTES, tinfo, is_same_child_table_json); + } + info->currTableDataCtx = tinfo->tableDataCtx; + return ret; } -static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo, SArray *cols) { +static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { int32_t ret = TSDB_CODE_SUCCESS; - if (!cJSON_IsObject(root)) { - uError("OTD:0x%" PRIx64 " data point needs to be JSON object", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - int32_t size = cJSON_GetArraySize(root); // outmost json fields has to be exactly 4 - if (size != OTD_JSON_FIELDS_NUM) { + if (unlikely(size != OTD_JSON_FIELDS_NUM)) { uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); return TSDB_CODE_TSC_INVALID_JSON; } // Parse metric - ret = smlParseMetricFromJSON(info, root, tinfo); - if (ret != TSDB_CODE_SUCCESS) { + ret = smlParseMetricFromJSON(info, root, elements); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); return ret; } uDebug("OTD:0x%" PRIx64 " Parse metric from JSON payload finished", info->id); - // Parse timestamp - ret = smlParseTSFromJSON(info, root, cols); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); - return ret; - } - uDebug("OTD:0x%" PRIx64 " Parse timestamp from JSON payload finished", info->id); - // Parse metric value - ret = smlParseColsFromJSON(root, cols); - if (ret) { + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN}; + ret = smlParseColsFromJSON(root, &kv); + if (unlikely(ret)) { uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); return ret; } uDebug("OTD:0x%" PRIx64 " Parse metric value from JSON payload finished", info->id); // Parse tags - ret = smlParseTagsFromJSON(root, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret) { + ret = smlParseTagsFromJSON(info, root, elements); + if (unlikely(ret)) { uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); return ret; } uDebug("OTD:0x%" PRIx64 " Parse tags from JSON payload finished", info->id); + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + // Parse timestamp + // notice!!! put ts back to tag to ensure get meta->precision + int64_t ts = smlParseTSFromJSON(info, root); + if (unlikely(ts < 0)) { + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + return TSDB_CODE_INVALID_TIMESTAMP; + } + uDebug("OTD:0x%" PRIx64 " Parse timestamp from JSON payload finished", info->id); + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .i = ts, .type = TSDB_DATA_TYPE_TIMESTAMP, .length = (int16_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + if(info->dataFormat){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + return TSDB_CODE_SUCCESS; } /************* TSDB_SML_JSON_PROTOCOL function end **************/ +static int32_t smlParseLineBottom(SSmlHandle *info) { + if(info->dataFormat) return TSDB_CODE_SUCCESS; -static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { - SSmlLineInfo elements = {0}; - uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? "rawdata" : sql)); - - int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); - return ret; - } - - SArray *cols = NULL; - if (info->dataFormat) { // if dataFormat, cols need new memory to save data - cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed to allocate memory", info->id); - return TSDB_CODE_OUT_OF_MEMORY; - } - } else { // if dataFormat is false, cols do not need to save data, there is another new memory to save data - cols = info->colsContainer; - } - - ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTS failed", info->id); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - ret = smlParseCols(elements.cols, elements.colsLen, cols, NULL, false, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse cloums fields failed", info->id); - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - - bool hasTable = true; - SSmlTableInfo *tinfo = NULL; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); - if (!oneTable) { - tinfo = smlBuildTableInfo(); - if (!tinfo) { - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return TSDB_CODE_OUT_OF_MEMORY; - } - taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); - oneTable = &tinfo; - hasTable = false; - } - - ret = smlDealCols(*oneTable, info->dataFormat, cols); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - if (!hasTable) { - ret = smlParseCols(elements.tags, elements.tagsLen, (*oneTable)->tags, (*oneTable)->childTableName, true, - info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse tag fields failed", info->id); - return ret; + for(int32_t i = 0; i < info->lineNum; i ++){ + SSmlLineInfo* elements = info->lines + i; + SSmlTableInfo *tinfo = NULL; + if(info->protocol == TSDB_SML_LINE_PROTOCOL){ + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->measure, elements->measureTagsLen, NULL); + }else if(info->protocol == TSDB_SML_TELNET_PROTOCOL){ + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + }else{ + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->tags, POINTER_BYTES, is_same_child_table_json); } - if (taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_TAGS) { + if(tinfo == NULL){ + uError("SML:0x%" PRIx64 "get oneTable failed, line num:%d", info->id, i); + smlBuildInvalidDataMsg(&info->msgBuf, "get oneTable failed", elements->measure); + return TSDB_CODE_SML_INVALID_DATA; + } + + if (taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); return TSDB_CODE_PAR_INVALID_TAGS_NUM; } - if (taosArrayGetSize(cols) + taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_COLUMNS) { + if (taosArrayGetSize(elements->colArray) + taosArrayGetSize(tinfo->tags) > TSDB_MAX_COLUMNS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); return TSDB_CODE_PAR_TOO_MANY_COLUMNS; } - (*oneTable)->sTableName = elements.measure; - (*oneTable)->sTableNameLen = elements.measureLen; - if (strlen((*oneTable)->childTableName) == 0) { - RandTableName rName = {(*oneTable)->tags, (*oneTable)->sTableName, (uint8_t)(*oneTable)->sTableNameLen, - (*oneTable)->childTableName, 0}; + int ret = smlPushCols(tinfo->cols, elements->colArray); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } - buildChildTableName(&rName); - (*oneTable)->uid = rName.uid; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + if (tableMeta) { // update meta + ret = smlUpdateMeta(tableMeta->colHash, tableMeta->cols, elements->colArray, false, &info->msgBuf); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlUpdateMeta(tableMeta->tagHash, tableMeta->tags, tinfo->tags, true, &info->msgBuf); + } + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + return ret; + } } else { - (*oneTable)->uid = *(uint64_t *)((*oneTable)->childTableName); + ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + return ret; + } + + SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); + smlInsertMeta(meta->colHash, meta->cols, elements->colArray); + nodeListSet(&info->superTables, elements->measure, elements->measureLen, meta, NULL); } } - SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements.measure, elements.measureLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - return ret; - } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); - } - - if (!info->dataFormat) { - taosArrayClear(info->colsContainer); - } - taosHashClear(info->dumplicateKey); - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { - int ret = TSDB_CODE_SUCCESS; - SSmlTableInfo *tinfo = smlBuildTableInfo(); - if (!tinfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed to allocate memory", info->id); - return TSDB_CODE_OUT_OF_MEMORY; - } - - if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ret = smlParseTelnetString(info, (const char *)data, (char *)data + len, tinfo, cols); - } else if (info->protocol == TSDB_SML_JSON_PROTOCOL) { - ret = smlParseJSONString(info, (cJSON *)data, tinfo, cols); - } else { - ASSERT(0); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed", info->id); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return ret; - } - - if (taosArrayGetSize(tinfo->tags) <= 0 || taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalidate tags length:[1,128]", NULL); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return TSDB_CODE_PAR_INVALID_TAGS_NUM; - } - taosHashClear(info->dumplicateKey); - - if (strlen(tinfo->childTableName) == 0) { - RandTableName rName = {tinfo->tags, tinfo->sTableName, (uint8_t)tinfo->sTableNameLen, tinfo->childTableName, 0}; - buildChildTableName(&rName); - tinfo->uid = rName.uid; - } else { - tinfo->uid = *(uint64_t *)(tinfo->childTableName); // generate uid by name simple - } - - bool hasTable = true; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName)); - if (!oneTable) { - taosHashPut(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName), &tinfo, POINTER_BYTES); - oneTable = &tinfo; - hasTable = false; - } else { - smlDestroyTableInfo(info, tinfo); - } - - taosArrayPush((*oneTable)->cols, &cols); - SSmlSTableMeta **tableMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - return ret; - } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen, &meta, POINTER_BYTES); - } - return TSDB_CODE_SUCCESS; } @@ -2254,46 +2746,91 @@ static int32_t smlParseJSON(SSmlHandle *info, char *payload) { int32_t payloadNum = 0; int32_t ret = TSDB_CODE_SUCCESS; - if (payload == NULL) { + if (unlikely(payload == NULL)) { uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); return TSDB_CODE_TSC_INVALID_JSON; } info->root = cJSON_Parse(payload); - if (info->root == NULL) { + if (unlikely(info->root == NULL)) { uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); return TSDB_CODE_TSC_INVALID_JSON; } // multiple data points must be sent in JSON array - if (cJSON_IsObject(info->root)) { - payloadNum = 1; - } else if (cJSON_IsArray(info->root)) { + if (cJSON_IsArray(info->root)) { payloadNum = cJSON_GetArraySize(info->root); + } else if (cJSON_IsObject(info->root)) { + payloadNum = 1; } else { uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - ret = TSDB_CODE_TSC_INVALID_JSON; - goto end; + return TSDB_CODE_TSC_INVALID_JSON; } - for (int32_t i = 0; i < payloadNum; ++i) { + int32_t i = 0; + while (i < payloadNum) { cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : cJSON_GetArrayItem(info->root, i); - ret = smlParseTelnetLine(info, dataPoint, -1); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - goto end; + if(info->dataFormat) { + SSmlLineInfo element = {0}; + ret = smlParseJSONString(info, dataPoint, &element); + }else{ + ret = smlParseJSONString(info, dataPoint, info->lines + i); } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); + return ret; + } + + if(unlikely(info->reRun)){ + i = 0; + info->reRun = false; + // clear info->childTables + NodeList* pList = info->childTables; + while (pList) { + if(pList->data.used) { + smlDestroyTableInfo(pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + // clear info->superTables + pList = info->superTables; + while (pList) { + if(pList->data.used) { + smlDestroySTableMeta(pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + if(unlikely(info->lines != NULL)){ + uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); + return TSDB_CODE_SML_INVALID_DATA; + } + info->lineNum = payloadNum; + info->lines = taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); + + memset(&info->preLine, 0, sizeof(SSmlLineInfo)); + info->currSTableMeta = NULL; + info->currTableDataCtx = NULL; + + SVnodeModifOpStmt* stmt= (SVnodeModifOpStmt*)(info->pQuery->pRoot); + stmt->freeHashFunc(stmt->pTableBlockHashObj); + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + continue; + } + i++; } -end: - return ret; + return TSDB_CODE_SUCCESS; } static int32_t smlInsertData(SSmlHandle *info) { int32_t code = TSDB_CODE_SUCCESS; - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - SSmlTableInfo *tableData = *oneTable; + NodeList* tmp = info->childTables; + while (tmp) { + SSmlTableInfo *tableData = (SSmlTableInfo *)tmp->data.value; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); @@ -2313,22 +2850,22 @@ static int32_t smlInsertData(SSmlHandle *info) { } taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); - SSmlSTableMeta **pMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); - ASSERT(NULL != pMeta && NULL != *pMeta); + SSmlSTableMeta *pMeta = + (SSmlSTableMeta *)nodeListGet(info->superTables, tableData->sTableName, tableData->sTableNameLen, NULL); + ASSERT(NULL != pMeta); // use tablemeta of stable to save vgid and uid of child table - (*pMeta)->tableMeta->vgId = vg.vgId; - (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid + pMeta->tableMeta->vgId = vg.vgId; + pMeta->tableMeta->uid = tableData->uid; // one table merge data block together according uid - code = smlBindData(info->pQuery, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat, - (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, + code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, pMeta->cols, tableData->cols, + pMeta->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->ttl, info->msgBuf.buf, info->msgBuf.len); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBindData failed", info->id); return code; } - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); + tmp = tmp->next; } code = smlBuildOutput(info->pQuery, info->pVgHash); @@ -2338,27 +2875,18 @@ static int32_t smlInsertData(SSmlHandle *info) { } info->cost.insertRpcTime = taosGetTimestampUs(); - // launchQueryImpl(info->pRequest, info->pQuery, false, NULL); - // info->affectedRows = taos_affected_rows(info->pRequest); - // return info->pRequest->code; - SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); - SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); - if (pWrapper == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pWrapper->pRequest = info->pRequest; - launchAsyncQuery(info->pRequest, info->pQuery, NULL, pWrapper); - return TSDB_CODE_SUCCESS; + launchQueryImpl(info->pRequest, info->pQuery, true, NULL); + return info->pRequest->code; } static void smlPrintStatisticInfo(SSmlHandle *info) { uError("SML:0x%" PRIx64 - " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ + " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 - "", + "", info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime, @@ -2381,7 +2909,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char return code; } - for (int32_t i = 0; i < numLines; ++i) { + char *oldRaw = rawLine; + int32_t i = 0; + while (i < numLines) { char *tmp = NULL; int len = 0; if (lines) { @@ -2400,10 +2930,23 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } } + uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? "rawdata" : tmp)); + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - code = smlParseInfluxLine(info, tmp, len); + if(info->dataFormat){ + SSmlLineInfo element = {0}; + code = smlParseInfluxString(info, tmp, tmp + len, &element); + }else{ + code = smlParseInfluxString(info, tmp, tmp + len, info->lines + i); + } } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - code = smlParseTelnetLine(info, tmp, len); + if(info->dataFormat) { + SSmlLineInfo element = {0}; + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, &element); + }else{ + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i); + } + } else { ASSERT(0); } @@ -2411,7 +2954,48 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); return code; } + if(info->reRun){ + i = 0; + rawLine = oldRaw; + info->reRun = false; + // clear info->childTables + NodeList* pList = info->childTables; + while (pList) { + if(pList->data.used) { + smlDestroyTableInfo(pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + // clear info->superTables + pList = info->superTables; + while (pList) { + if(pList->data.used) { + smlDestroySTableMeta(pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + if(info->lines != NULL){ + uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); + return TSDB_CODE_SML_INVALID_DATA; + } + info->lines = taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); + + memset(&info->preLine, 0, sizeof(SSmlLineInfo)); + info->currSTableMeta = NULL; + info->currTableDataCtx = NULL; + + SVnodeModifOpStmt* stmt= (SVnodeModifOpStmt*)(info->pQuery->pRoot); + stmt->freeHashFunc(stmt->pTableBlockHashObj); + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + continue; + } + i++; } + return code; } @@ -2427,16 +3011,22 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL return code; } + code = smlParseLineBottom(info); + if (code != 0) { + uError("SML:0x%" PRIx64 " smlParseLineBottom error : %s", info->id, tstrerror(code)); + return code; + } + info->cost.lineNum = numLines; - info->cost.numOfSTables = taosHashGetSize(info->superTables); - info->cost.numOfCTables = taosHashGetSize(info->childTables); + info->cost.numOfSTables = nodeListSize(info->superTables); + info->cost.numOfCTables = nodeListSize(info->childTables); info->cost.schemaTime = taosGetTimestampUs(); do { code = smlModifyDBSchemas(info); if (code == 0) break; - } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); + } while (retryNum++ < nodeListSize(info->superTables) * MAX_RETRY_TIMES); if (code != 0) { uError("SML:0x%" PRIx64 " smlModifyDBSchemas error : %s", info->id, tstrerror(code)); @@ -2453,92 +3043,41 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL return code; } -static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) { - // SCatalog *catalog = NULL; - // int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog); - // if (code != TSDB_CODE_SUCCESS) { - // uError("SML get catalog error %d", code); - // return code; - // } - // - // SName name; - // tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db)); - // char dbFname[TSDB_DB_FNAME_LEN] = {0}; - // tNameGetFullDbName(&name, dbFname); - // SDbCfgInfo pInfo = {0}; - // - // SRequestConnInfo conn = {0}; - // conn.pTrans = taos->pAppInfo->pTransporter; - // conn.requestId = request->requestId; - // conn.requestObjRefId = request->self; - // conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp); - // - // code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo); - // if (code != TSDB_CODE_SUCCESS) { - // return code; - // } - // taosArrayDestroy(pInfo.pRetensions); - // - // if (!pInfo.schemaless) { - // return TSDB_CODE_SML_INVALID_DB_CONF; - // } - return TSDB_CODE_SUCCESS; -} - -static void smlInsertCallback(void *param, void *res, int32_t code) { - SRequestObj *pRequest = (SRequestObj *)res; - SSmlHandle *info = (SSmlHandle *)param; - int32_t rows = taos_affected_rows(pRequest); - - uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); - Params *pParam = info->params; - // lock - taosThreadSpinLock(&pParam->lock); - pParam->cnt++; - if (code != TSDB_CODE_SUCCESS) { - pParam->request->code = code; - pParam->request->body.resInfo.numOfRows += rows; - } else { - pParam->request->body.resInfo.numOfRows += info->affectedRows; +TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, + int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; } - // unlock - taosThreadSpinUnlock(&pParam->lock); - if (pParam->cnt == pParam->total) { - tsem_post(&pParam->sem); + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); + if (request == NULL) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; } - uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows); - info->cost.endTime = taosGetTimestampUs(); - info->cost.code = code; - smlPrintStatisticInfo(info); - smlDestroyInfo(info); -} -TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd, - int numLines, int protocol, int precision, int32_t ttl) { - int batchs = 0; - STscObj *pTscObj = request->pTscObj; + SSmlHandle *info = smlBuildSmlInfo(taos); + if (info == NULL) { + request->code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML:taos_schemaless_insert error SSmlHandle is null"); + goto end; + } + info->pRequest = request; + info->isRawLine = rawLine != NULL; + info->ttl = ttl; + info->precision = precision; + info->protocol = protocol; + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + info->lineNum = numLines; - pTscObj->schemalessType = 1; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - - Params params = {0}; - params.request = request; - tsem_init(¶ms.sem, 0, 0); - taosThreadSpinInit(&(params.lock), 0); - if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); goto end; } - if (isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS) { - request->code = TSDB_CODE_SML_INVALID_DB_CONF; - smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL); - goto end; - } - if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); @@ -2560,65 +3099,15 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char goto end; } - batchs = ceil(((double)numLines) / tsSmlBatchSize); - params.total = batchs; - for (int i = 0; i < batchs; ++i) { - SRequestObj *req = (SRequestObj *)createRequest(pTscObj->id, TSDB_SQL_INSERT, 0); - if (!req) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error request is null"); - goto end; - } - SSmlHandle *info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision); - if (!info) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error SSmlHandle is null"); - goto end; - } - - info->isRawLine = (rawLine == NULL); - info->ttl = ttl; - - int32_t perBatch = tsSmlBatchSize; - - if (numLines > perBatch) { - numLines -= perBatch; - } else { - perBatch = numLines; - numLines = 0; - } - - info->params = ¶ms; - info->affectedRows = perBatch; - info->pRequest->body.queryFp = smlInsertCallback; - info->pRequest->body.param = info; - int32_t code = smlProcess(info, lines, rawLine, rawLineEnd, perBatch); - if (lines) { - lines += perBatch; - } - if (rawLine) { - int num = 0; - while (rawLine < rawLineEnd) { - if (*(rawLine++) == '\n') { - num++; - } - if (num == perBatch) { - break; - } - } - } - if (code != TSDB_CODE_SUCCESS) { - info->pRequest->body.queryFp(info, req, code); - } - } - tsem_wait(¶ms.sem); + int32_t code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); + request->code = code; + info->cost.endTime = taosGetTimestampUs(); + info->cost.code = code; + smlPrintStatisticInfo(info); end: - taosThreadSpinDestroy(¶ms.lock); - tsem_destroy(¶ms.sem); - // ((STscObj *)taos)->schemalessType = 0; - pTscObj->schemalessType = 1; uDebug("resultend:%s", request->msgBuf); + smlDestroyInfo(info); return (TAOS_RES *)request; } @@ -2643,25 +3132,7 @@ end: TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid); } TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { @@ -2677,25 +3148,7 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi } TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, - int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines || len <= 0) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - + int precision, int32_t ttl, int64_t reqid) { int numLines = 0; *totalRows = 0; char *tmp = lines; @@ -2708,7 +3161,7 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int tmp = lines + i + 1; } } - return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid); } TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int64_t reqid) { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ade0c95227..766a94caf1 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -875,6 +875,7 @@ void tmqFreeImpl(void* handle) { tmq_t* tmq = (tmq_t*)handle; // TODO stop timer + tmqClearUnhandleMsg(tmq); if (tmq->mqueue) taosCloseQueue(tmq->mqueue); if (tmq->delayedTask) taosCloseQueue(tmq->delayedTask); if (tmq->qall) taosFreeQall(tmq->qall); @@ -884,8 +885,7 @@ void tmqFreeImpl(void* handle) { int32_t sz = taosArrayGetSize(tmq->clientTopics); for (int32_t i = 0; i < sz; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema); - int32_t vgSz = taosArrayGetSize(pTopic->vgs); + taosMemoryFreeClear(pTopic->schema.pSchema); taosArrayDestroy(pTopic->vgs); } taosArrayDestroy(tmq->clientTopics); @@ -1304,7 +1304,6 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { for (int32_t i = 0; i < sz; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema); - int32_t vgSz = taosArrayGetSize(pTopic->vgs); taosArrayDestroy(pTopic->vgs); } taosArrayDestroy(tmq->clientTopics); @@ -1410,7 +1409,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { return -1; } void* pReq = taosMemoryCalloc(1, tlen); - if (tlen < 0) { + if (pReq == NULL) { tscError("failed to malloc askEpReq msg, size:%d", tlen); return -1; } @@ -1738,7 +1737,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { taosFreeQitem(pollRspWrapper); return pRsp; } else { - tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n", + tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); taosFreeQitem(pollRspWrapper); } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a02b5d11da..d96e7c9be0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2384,24 +2384,15 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { return NULL; } - SSmlKv* pTag = taosMemoryCalloc(1, sizeof(SSmlKv)); - if (pTag == NULL) { - taosArrayDestroy(tags); - return NULL; - } - void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); if (cname == NULL) { taosArrayDestroy(tags); - taosMemoryFree(pTag); return NULL; } - pTag->key = "group_id"; - pTag->keyLen = strlen(pTag->key); - pTag->type = TSDB_DATA_TYPE_UBIGINT; - pTag->u = groupId; - pTag->length = sizeof(uint64_t); + SSmlKv pTag = {.key = "group_id", .keyLen = sizeof("group_id") - 1, + .type = TSDB_DATA_TYPE_UBIGINT, .u = groupId, + .length = sizeof(uint64_t)}; taosArrayPush(tags, &pTag); RandTableName rname = { @@ -2413,7 +2404,6 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { buildChildTableName(&rname); - taosMemoryFree(pTag); taosArrayDestroy(tags); ASSERT(rname.ctbShortName && rname.ctbShortName[0]); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d1a3f38143..7e9b28939b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -333,6 +333,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr) != 0) return -1; if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; + if (cfgAddBool(pCfg, "assert", 1, 1) != 0) return -1; if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1; if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1; @@ -693,6 +694,8 @@ static void taosSetSystemCfg(SConfig *pCfg) { bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; taosSetCoreDump(enableCore); + tsAssert = cfgGetItem(pCfg, "assert")->bval; + // todo tsVersion = 30000000; } @@ -788,6 +791,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { case 'a': { if (strcasecmp("asyncLog", name) == 0) { tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + } else if (strcasecmp("assert", name) == 0) { + tsAssert = cfgGetItem(pCfg, "assert")->bval; } break; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 22ed4e4aa3..95a72843e8 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -28,6 +28,8 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" +#include "tlog.h" + int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 0d47ef1e7f..644b253cc2 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -298,8 +298,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { } static int compareKv(const void* p1, const void* p2) { - SSmlKv* kv1 = *(SSmlKv**)p1; - SSmlKv* kv2 = *(SSmlKv**)p2; + SSmlKv* kv1 = (SSmlKv*)p1; + SSmlKv* kv2 = (SSmlKv*)p2; int32_t kvLen1 = kv1->keyLen; int32_t kvLen2 = kv2->keyLen; int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); @@ -320,7 +320,7 @@ void buildChildTableName(RandTableName* rName) { taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); - SSmlKv* tagKv = taosArrayGetP(rName->tags, j); + SSmlKv* tagKv = taosArrayGet(rName->tags, j); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendChar(&sb, '='); if (IS_VAR_DATA_TYPE(tagKv->type)) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 52ebd7f879..ca2c056743 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "trow.h" +#include "tlog.h" const uint8_t tdVTypeByte[2][3] = {{ // 2 bits diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index a106a09a69..ec05ef4c44 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -23,6 +23,8 @@ #define _DEFAULT_SOURCE #include "ttime.h" +#include "tlog.h" + /* * mktime64 - Converts date to seconds. * Converts Gregorian date to seconds since 1970-01-01 00:00:00. diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 188677656a..a8103351b4 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -17,6 +17,7 @@ #include "dmMgmt.h" #include "mnode.h" #include "tconfig.h" +#include "tglobal.h" // clang-format off #define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'." @@ -45,9 +46,30 @@ static struct { SArray *pArgs; // SConfigPair } global = {0}; -static void dmStopDnode(int signum, void *info, void *ctx) { dmStop(); } +static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143, true); } +static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert = 1; } + +static void dmStopDnode(int signum, void *sigInfo, void *context) { + // taosIgnSignal(SIGUSR1); + // taosIgnSignal(SIGUSR2); + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGHUP); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGBREAK); + + dInfo("shut down signal is %d", signum); +#ifndef WINDOWS + dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, + taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); +#endif + + dmStop(); +} static void dmSetSignalHandle() { + taosSetSignal(SIGUSR1, dmSetDebugFlag); + taosSetSignal(SIGUSR2, dmSetAssert); taosSetSignal(SIGTERM, dmStopDnode); taosSetSignal(SIGHUP, dmStopDnode); taosSetSignal(SIGINT, dmStopDnode); @@ -105,6 +127,19 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { return 0; } +static void dmPrintArgs(int32_t argc, char const *argv[]) { + char path[1024] = {0}; + taosGetCwd(path, sizeof(path)); + + char args[1024] = {0}; + int32_t arglen = snprintf(args, sizeof(args), "%s", argv[0]); + for (int32_t i = 1; i < argc; ++i) { + arglen = arglen + snprintf(args + arglen, sizeof(args) - arglen, " %s", argv[i]); + } + + dInfo("startup path:%s args:%s", path, args); +} + static void dmGenerateGrant() { mndGenerateMachineCode(); } static void dmPrintVersion() { @@ -194,6 +229,8 @@ int mainWindows(int argc, char **argv) { return -1; } + dmPrintArgs(argc, argv); + if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) { dError("failed to start since read config error"); taosCloseLog(); @@ -201,7 +238,12 @@ int mainWindows(int argc, char **argv) { return -1; } - taosConvInit(); + if (taosConvInit() != 0) { + dError("failed to init conv"); + taosCloseLog(); + taosCleanupArgs(); + return -1; + } if (global.dumpConfig) { dmDumpCfg(); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index a7ad983b0c..39eeeb32ea 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -103,7 +103,12 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { tSerializeSStatusReq(pHead, contLen, &req); tFreeSStatusReq(&req); - SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527}; + SRpcMsg rpcMsg = {.pCont = pHead, + .contLen = contLen, + .msgType = TDMT_MND_STATUS, + .info.ahandle = (void *)0x9527, + .info.refId = 0, + .info.noResp = 0}; SRpcMsg rpcRsp = {0}; dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq); @@ -150,7 +155,8 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SServerStatusRsp statusRsp = {0}; SMonMloadInfo minfo = {0}; (*pMgmt->getMnodeLoadsFp)(&minfo); - if (minfo.isMnode && (minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) { + if (minfo.isMnode && + (minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); return; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 30ef7b9542..80c040a5e8 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -20,7 +20,9 @@ static void *dmStatusThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-status"); - + + const static int16_t TRIM_FREQ = 30; + int32_t trimCount = 0; while (1) { taosMsleep(200); if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; @@ -28,9 +30,13 @@ static void *dmStatusThreadFp(void *param) { int64_t curTime = taosGetTimestampMs(); float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsStatusInterval) { - taosMemoryTrim(0); dmSendStatusReq(pMgmt); lastTime = curTime; + + trimCount = (trimCount + 1) % TRIM_FREQ; + if (trimCount == 0) { + taosMemoryTrim(0); + } } } diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 2d2a121795..2e66aeae37 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -139,7 +139,7 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SSnode *pSnode = pMgmt->pSnode; if (pSnode == NULL) { - dError("snode: msg:%p failed to put into vnode queue since %s, type:%s qtype:%d", pMsg, terrstr(), + dError("msg:%p failed to put into snode queue since %s, type:%s qtype:%d", pMsg, terrstr(), TMSG_INFO(pMsg->msgType), qtype); taosFreeQitem(pMsg); rpcFreeCont(pRpc->pCont); @@ -161,7 +161,8 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { smPutNodeMsgToWriteQueue(pMgmt, pMsg); break; default: - ASSERT(0); + ASSERTS(0, "msg:%p failed to put into snode queue since %s, type:%s qtype:%d", pMsg, terrstr(), + TMSG_INFO(pMsg->msgType), qtype); } return 0; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 2383a09343..99f8bd002a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -301,6 +301,7 @@ int32_t dmInitServer(SDnode *pDnode) { rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.parent = pDnode; + rpcInit.compressSize = tsCompressMsgSize; pTrans->serverRpc = rpcOpen(&rpcInit); if (pTrans->serverRpc == NULL) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 972907cc95..4aaa96cb71 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -555,6 +555,12 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { } if (mndCheckDbPrivilegeByName(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + mndReleaseTopic(pMnode, pTopic); + goto SUBSCRIBE_OVER; + } + + if (mndCheckTopicPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_SUBSCRIBE, pTopic) != 0) { + mndReleaseTopic(pMnode, pTopic); goto SUBSCRIBE_OVER; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5fa86dffe2..43155124c1 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -825,7 +825,13 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); code = mndAlterDb(pMnode, pReq, pDb, &dbObj); - if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + + if (dbObj.cfg.replications != pDb->cfg.replications) { + // return quickly, operation executed asynchronously + mInfo("db:%s, alter db replica from %d to %d", pDb->name, pDb->cfg.replications, dbObj.cfg.replications); + } else { + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + } _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 895d7fedc4..10b1e36496 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -119,7 +119,13 @@ int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta } int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - int32_t code = mndProcessWriteMsg(pFsm, pMsg, pMeta); + int32_t code = 0; + if (!syncUtilUserCommit(pMsg->msgType)) { + goto _out; + } + code = mndProcessWriteMsg(pFsm, pMsg, pMeta); + +_out: rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; return code; @@ -143,9 +149,13 @@ void mndRestoreFinish(const SSyncFSM *pFsm) { SMnode *pMnode = pFsm->data; if (!pMnode->deploy) { - mInfo("vgId:1, sync restore finished, and will handle outstanding transactions"); - mndTransPullup(pMnode); - mndSetRestored(pMnode, true); + if (!pMnode->restored) { + mInfo("vgId:1, sync restore finished, and will handle outstanding transactions"); + mndTransPullup(pMnode); + mndSetRestored(pMnode, true); + } else { + mInfo("vgId:1, sync restore finished, repeat call"); + } } else { mInfo("vgId:1, sync restore finished"); } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 83392049b8..03069d5404 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -174,7 +174,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockSMA, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock* pDataBlock, bool *allHave); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 6a704d0425..0ed68dce95 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -454,7 +454,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK SListNode* pNode = NULL; while ((pNode = tdListNext(&iter)) != NULL) { - memcpy(pBuf + sizeof(suid), pNode->data, keyLen); + memcpy(&pBuf[1], pNode->data, keyLen); // check whether it is existed in LRU cache, and remove it from linked list if not. LRUHandle* pRes = taosLRUCacheLookup(pCache, pBuf, len); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0b00f036de..722a93137c 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -549,8 +549,8 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { } static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME) { - int64_t ttlDays; - int64_t ctime; + int64_t ttlDays = 0; + int64_t ctime = 0; if (pME->type == TSDB_CHILD_TABLE) { ctime = pME->ctbEntry.ctime; ttlDays = pME->ctbEntry.ttlDays; @@ -1353,6 +1353,10 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { goto end; } + if (stbEntry.stbEntry.schemaTag.pSchema == NULL) { + goto end; + } + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; STagVal tagVal = {.cid = pTagColumn->colId}; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 3e772b1e1c..a6393b6413 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -750,12 +750,12 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d } void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { - const SArray* pBlocks = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; - int64_t suid = pTask->tbSink.stbUid; - char* stbFullName = pTask->tbSink.stbFullName; - STSchema* pTSchema = pTask->tbSink.pTSchema; - SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper; + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + /*SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper;*/ int32_t blockSz = taosArrayGetSize(pBlocks); @@ -780,7 +780,6 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* } for (int32_t i = 0; i < blockSz; i++) { - bool createTb = true; SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); if (pDataBlock->info.type == STREAM_DELETE_RESULT) { SBatchDeleteReq deleteReq = {0}; @@ -818,13 +817,31 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* tqDebug("failed to put delete req into write-queue since %s", terrstr()); } } else { - SSubmitTbData* pTbData = (SSubmitTbData*)taosMemoryCalloc(1, sizeof(SSubmitTbData)); - if (!pTbData) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + SSubmitTbData tbData = {0}; + int32_t rows = pDataBlock->info.rows; + tqDebug("tq sink, convert block1 %d, rows: %d", i, rows); + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { goto _end; } - // create table req - if (createTb) { + + tbData.suid = suid; + tbData.uid = 0; // uid is assigned by vnode + tbData.sver = pTSchema->version; + + char* ctbName = NULL; + if (pDataBlock->info.parTbName[0]) { + ctbName = strdup(pDataBlock->info.parTbName); + } else { + ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, pVnode->pMeta, 0); + if (metaGetTableEntryByName(&mr, ctbName) < 0) { + metaReaderClear(&mr); + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); + SVCreateTbReq* pCreateTbReq = NULL; if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { @@ -867,31 +884,36 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* pCreateTbReq->ctb.tagName = tagName; // set table name - if (pDataBlock->info.parTbName[0]) { - pCreateTbReq->name = strdup(pDataBlock->info.parTbName); - } else { - pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + pCreateTbReq->name = ctbName; + ctbName = NULL; + + tbData.pCreateTbReq = pCreateTbReq; + tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + } else { + if (mr.me.type != TSDB_CHILD_TABLE) { + tqError("vgId:%d, failed to write into %s, since table type incorrect, type %d", TD_VID(pVnode), ctbName, + mr.me.type); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; } - pTbData->pCreateTbReq = pCreateTbReq; - pTbData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + + if (mr.me.ctbEntry.suid != suid) { + tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid: %" PRId64 + ", actual suid %" PRId64 "", + TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + } + + tbData.uid = mr.me.uid; + metaReaderClear(&mr); + taosMemoryFreeClear(ctbName); } - pTbData->suid = suid; - pTbData->uid = 0; // uid is assigned by vnode - pTbData->sver = pTSchema->version; - - int32_t rows = pDataBlock->info.rows; - - if (!(pTbData->aRowP = taosArrayInit(rows, sizeof(SRow*)))) { - taosMemoryFreeClear(pTbData); - goto _end; - } - - tqDebug("tq sink, convert block1 %d, rows: %d", i, rows); - + // rows if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { - taosArrayDestroy(pTbData->aRowP); - taosMemoryFree(pTbData); + taosArrayDestroy(tbData.aRowP); goto _end; } @@ -904,14 +926,15 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); taosArrayPush(pVals, &cv); } else { - void* data = colDataGetData(pColData, j); + void* colData = colDataGetData(pColData, j); if (IS_STR_DATA_TYPE(pCol->type)) { - SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SValue sv = + (SValue){.nData = varDataLen(colData), .pData = varDataVal(colData)}; // address copy, no value SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); taosArrayPush(pVals, &cv); } else { SValue sv; - memcpy(&sv.val, data, tDataTypes[pCol->type].bytes); + memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes); SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); taosArrayPush(pVals, &cv); } @@ -919,15 +942,14 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* } SRow* pRow = NULL; if ((terrno = tRowBuild(pVals, (STSchema*)pTSchema, &pRow)) < 0) { - tDestroySSubmitTbData(pTbData, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); goto _end; } ASSERT(pRow); - taosArrayPush(pTbData->aRowP, &pRow); + taosArrayPush(tbData.aRowP, &pRow); } - taosArrayPush(pReq->aSubmitTbData, pTbData); - taosMemoryFree(pTbData); + taosArrayPush(pReq->aSubmitTbData, &tbData); // encode int32_t len; @@ -945,6 +967,9 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* if (tEncodeSSubmitReq2(&encoder, pReq) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to encode submit req since %s", terrstr()); + tEncoderClear(&encoder); + rpcFreeCont(pBuf); + continue; } tEncoderClear(&encoder); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 571fbb088b..8aca60fbd9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1025,7 +1025,10 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs SArray *pDelIdxArray = taosArrayInit(32, sizeof(SDelIdx)); code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); - if (code) goto _err; + if (code) { + tsdbDelFReaderClose(&pDelFReader); + goto _err; + } SDelIdx *delIdx = taosArraySearch(pDelIdxArray, &(SDelIdx){.suid = suid, .uid = uid}, tCmprDelIdx, TD_EQ); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 40932f077c..92451dcf58 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -53,6 +53,7 @@ typedef struct { // -------------- TSKEY nextKey; // reset by each table commit int32_t commitFid; + int32_t expLevel; TSKEY minKey; TSKEY maxKey; // commit file data @@ -503,6 +504,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { // memory pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision); + pCommitter->expLevel = tsdbFidLevel(pCommitter->commitFid, &pCommitter->pTsdb->keepCfg, taosGetTimestampSec()); tsdbFidKeyRange(pCommitter->commitFid, pCommitter->minutes, pCommitter->precision, &pCommitter->minKey, &pCommitter->maxKey); #if 0 @@ -556,7 +558,10 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { } } else { SDiskID did = {0}; - tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did); + if (tfsAllocDisk(pTsdb->pVnode->pTfs, pCommitter->expLevel, &did) < 0) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); wSet.diskId = did; wSet.nSttF = 1; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 72d9c4f69e..7dc839773f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -962,6 +962,7 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } } + pDFileSet->diskId = pSet->diskId; goto _exit; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 823125a905..d6931e1b01 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3128,7 +3128,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 return false; } else if (pKey->ts == last->ts) { TSDBKEY* prev = taosArrayGet(pDelList, num - 2); - return (prev->version >= pKey->version); + return (prev->version >= pKey->version && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); @@ -4134,8 +4134,9 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockSMA, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { int32_t code = 0; + SColumnDataAgg ***pBlockSMA = &pDataBlock->pBlockAgg; *allHave = false; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { @@ -4183,6 +4184,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS int32_t i = 0, j = 0; size_t size = taosArrayGetSize(pSup->pColAgg); + // ensure capacity + if(pDataBlock->pDataBlock) { + size_t colsNum = taosArrayGetSize(pDataBlock->pDataBlock); + taosArrayEnsureCap(pSup->pColAgg, colsNum); + } + SSDataBlock* pResBlock = pReader->pResBlock; if (pResBlock->pBlockAgg == NULL) { size_t num = taosArrayGetSize(pResBlock->pDataBlock); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 048d3cc22b..bd17a2593b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -155,7 +155,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { if (rowVer >= pReader->sver && rowVer <= pReader->ever) { pIter->rInfo.suid = pIter->bData.suid; - pIter->rInfo.uid = pIter->bData.uid; + pIter->rInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); goto _add_iter; } @@ -179,16 +179,14 @@ _err: return code; } -static SRowInfo* tsdbSnapGetRow(STsdbSnapReader* pReader) { return pReader->pIter ? &pReader->pIter->rInfo : NULL; } - static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) { int32_t code = 0; if (pReader->pIter) { - SFDataIter* pIter = pReader->pIter; - + SFDataIter* pIter = NULL; while (true) { _find_row: + pIter = pReader->pIter; for (pIter->iRow++; pIter->iRow < pIter->bData.nRow; pIter->iRow++) { int64_t rowVer = pIter->bData.aVersion[pIter->iRow]; @@ -224,6 +222,7 @@ static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) { } pReader->pIter = NULL; + break; } else if (pIter->type == SNAP_STT_FILE_ITER) { for (pIter->iSttBlk++; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) { SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); @@ -238,6 +237,7 @@ static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) { } pReader->pIter = NULL; + break; } else { ASSERT(0); } @@ -269,6 +269,20 @@ _err: return code; } +static SRowInfo* tsdbSnapGetRow(STsdbSnapReader* pReader) { + if (pReader->pIter) { + return &pReader->pIter->rInfo; + } else { + tsdbSnapNextRow(pReader); + + if (pReader->pIter) { + return &pReader->pIter->rInfo; + } else { + return NULL; + } + } +} + static int32_t tsdbSnapCmprData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; @@ -1356,7 +1370,7 @@ _exit: taosMemoryFree(pWriter); } } else { - tsdbDebug("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); + tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); *ppWriter = pWriter; } return code; @@ -1421,7 +1435,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { tFree(pWriter->aBuf[iBuf]); } - tsdbInfo("vgId:%d, vnode snapshot tsdb writer close for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); + tsdbInfo("vgId:%d %s done", TD_VID(pWriter->pTsdb->pVnode), __func__); taosMemoryFree(pWriter); *ppWriter = NULL; return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 63391ec0f4..1c92e922b0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -637,6 +637,7 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { } } else { ASSERT(0); + return NULL; // suppress error report by compiler } } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index cbfc16ce4a..bb5063cfea 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -210,9 +210,13 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp version); ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); + ASSERT(pVnode->state.applied + 1 == version); + pVnode->state.applied = version; pVnode->state.applyTerm = pMsg->info.conn.applyTerm; + if (!syncUtilUserCommit(pMsg->msgType)) goto _exit; + // skip header pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); len = pMsg->contLen - sizeof(SMsgHead); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0b198afb0d..1fcee1992e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -224,7 +224,7 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { bool allColumnsHaveAgg = true; - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pBlock->pBlockAgg, &allColumnsHaveAgg); + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 2374d80bbf..072a96fa83 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -95,6 +95,8 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo // TODO: optimize to ignore null values for linear interpolation. if (!pLinearInfo->isStartSet) { if (!colDataIsNull_s(pColInfoData, rowIndex)) { + ASSERT(IS_MATHABLE_TYPE(pColInfoData->info.type)); + pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d9a011a892..6f1ab8ca65 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -863,19 +863,20 @@ static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) { int32_t compareWinRes(void* pKey, void* data, int32_t index) { SArray* res = (SArray*)data; - SWinKey* pos = taosArrayGet(res, index); - SResKeyPos* pData = (SResKeyPos*)pKey; - if (*(int64_t*)pData->key == pos->ts) { - if (pData->groupId > pos->groupId) { - return 1; - } else if (pData->groupId < pos->groupId) { - return -1; - } - return 0; - } else if (*(int64_t*)pData->key > pos->ts) { + SWinKey* pDataPos = taosArrayGet(res, index); + SResKeyPos* pRKey = (SResKeyPos*)pKey; + if (pRKey->groupId > pDataPos->groupId) { return 1; + } else if (pRKey->groupId < pDataPos->groupId) { + return -1; } - return -1; + + if (*(int64_t*)pRKey->key > pDataPos->ts) { + return 1; + } else if (*(int64_t*)pRKey->key < pDataPos->ts){ + return -1; + } + return 0; } static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { @@ -1400,19 +1401,21 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { int32_t compareWinKey(void* pKey, void* data, int32_t index) { SArray* res = (SArray*)data; - SWinKey* pos = taosArrayGet(res, index); - SWinKey* pData = (SWinKey*)pKey; - if (pData->ts == pos->ts) { - if (pData->groupId > pos->groupId) { - return 1; - } else if (pData->groupId < pos->groupId) { - return -1; - } - return 0; - } else if (pData->ts > pos->ts) { + SWinKey* pDataPos = taosArrayGet(res, index); + SWinKey* pWKey = (SWinKey*)pKey; + + if (pWKey->groupId > pDataPos->groupId) { return 1; + } else if (pWKey->groupId < pDataPos->groupId) { + return -1; } - return -1; + + if (pWKey->ts > pDataPos->ts) { + return 1; + } else if (pWKey->ts < pDataPos->ts) { + return -1; + } + return 0; } static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval, diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 4204692514..d97f81c994 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -17,6 +17,7 @@ #include "taoserror.h" #include "tdef.h" #include "tpagedbuf.h" +#include "tlog.h" #define LHASH_CAP_RATIO 0.85 diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index e5727f1472..9c3e1f5604 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -22,6 +22,7 @@ #include "tpagedbuf.h" #include "tpercentile.h" #include "ttypes.h" +#include "tlog.h" #define DEFAULT_NUM_OF_SLOT 1024 @@ -367,11 +368,13 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { pSlot->info.data = NULL; } - SArray *pPageIdList = (SArray *)taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId)); - if (pPageIdList == NULL) { - SArray *pList = taosArrayInit(4, sizeof(int32_t)); - taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pList, POINTER_BYTES); - pPageIdList = pList; + SArray *pPageIdList; + void *p = taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId)); + if (p == NULL) { + pPageIdList = taosArrayInit(4, sizeof(int32_t)); + taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES); + } else { + pPageIdList = *(SArray **)p; } pSlot->info.data = getNewBufPage(pBucket->pBuffer, &pageId); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index c78ec5b999..32e57565d4 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -88,11 +88,13 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { } #ifdef WINDOWS if (strlen(path) == 0) { - strcat(path, "udfd.exe"); - } else { - strcat(path, "\\udfd.exe"); + strcat(path, "C:\\TDengine"); } + strcat(path, "\\udfd.exe"); #else + if (strlen(path) == 0) { + strcat(path, "/usr/bin"); + } strcat(path, "/udfd"); #endif char *argsUdfd[] = {path, "-c", configDir, NULL}; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 66b5ce3e58..2ab061a4c1 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -56,7 +56,7 @@ static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSche int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv* kv = taosArrayGetP(cols, i); + SSmlKv* kv = taosArrayGet(cols, i); SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema)); @@ -111,7 +111,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < tags->numOfBound; ++i) { SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; - SSmlKv* kv = taosArrayGetP(cols, i); + SSmlKv* kv = taosArrayGet(cols, i); taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; @@ -158,9 +158,68 @@ end: return code; } -int32_t smlBindData(SQuery* query, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, - char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, - int16_t msgBufLen) { +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta){ + STableDataCxt* pTableCxt = NULL; + SVCreateTbReq *pCreateTbReq = NULL; + int ret = insGetTableDataCxt(((SVnodeModifOpStmt *)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid), + pTableMeta, &pCreateTbReq, &pTableCxt, false); + if (ret != TSDB_CODE_SUCCESS) { + return NULL; + } + + ret = initTableColSubmitData(pTableCxt); + if (ret != TSDB_CODE_SUCCESS) { + return NULL; + } + return pTableCxt; +} + +int32_t smlBuildRow(STableDataCxt* pTableCxt){ + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS != ret) { + return ret; + } + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + return TSDB_CODE_SUCCESS; +} + +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void *data, int32_t index){ + int ret = TSDB_CODE_SUCCESS; + SSchema* pColSchema = schema + index; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); + SSmlKv* kv = (SSmlKv*)data; + if (kv->type == TSDB_DATA_TYPE_NCHAR){ + int32_t len = 0; + char* pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE); + if (NULL == pUcs4) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + if (errno == E2BIG) { + ret = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } + ret = TSDB_CODE_TSC_INVALID_VALUE; + goto end; + } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + } else if(kv->type == TSDB_DATA_TYPE_BINARY) { + pVal->value.nData = kv->length; + pVal->value.pData = (uint8_t *)kv->value; + } else { + memcpy(&pVal->value.val, &(kv->value), kv->length); + } + pVal->flag = CV_FLAG_VALUE; + +end: + return ret; +} + +int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, + char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SSchema* pTagsSchema = getTableTagSchema(pTableMeta); @@ -193,6 +252,20 @@ int32_t smlBindData(SQuery* query, SArray* tags, SArray* colsSchema, SArray* col pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); + if(dataFormat){ + STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifOpStmt *)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); + if (NULL == pTableCxt) { + ret = buildInvalidOperationMsg(&pBuf, "dataformat true. get tableDataCtx error"); + goto end; + } + (*pTableCxt)->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; + (*pTableCxt)->pData->pCreateTbReq = pCreateTblReq; + (*pTableCxt)->pMeta->uid = pTableMeta->uid; + (*pTableCxt)->pMeta->vgId = pTableMeta->vgId; + pCreateTblReq = NULL; + goto end; + } + STableDataCxt* pTableCxt = NULL; ret = insGetTableDataCxt(((SVnodeModifOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false); @@ -227,15 +300,12 @@ int32_t smlBindData(SQuery* query, SArray* tags, SArray* colsSchema, SArray* col for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); - SSmlKv* kv = NULL; - if (!format) { - void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); - if (p) kv = *p; - } - - if (kv == NULL) { + void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); + if (p == NULL) { continue; } + SSmlKv *kv = *(SSmlKv **)p; + if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); } @@ -297,7 +367,7 @@ SQuery* smlInitHandle() { qDestroyQuery(pQuery); return NULL; } - stmt->pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); stmt->freeHashFunc = insDestroyTableDataCxtHashMap; stmt->freeArrayFunc = insDestroyVgroupDataCxtList; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 85a4e70124..10e45901e5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3879,12 +3879,17 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt pOptions->keep[2] = getBigintFromValueNode((SValueNode*)nodesListGetNode(pOptions->pKeep, 2)); } + int64_t tsdbMaxKeep = TSDB_MAX_KEEP; + if (pOptions->precision == TSDB_TIME_PRECISION_NANO) { + tsdbMaxKeep = TSDB_MAX_KEEP_NS; + } + if (pOptions->keep[0] < TSDB_MIN_KEEP || pOptions->keep[1] < TSDB_MIN_KEEP || pOptions->keep[2] < TSDB_MIN_KEEP || - pOptions->keep[0] > TSDB_MAX_KEEP || pOptions->keep[1] > TSDB_MAX_KEEP || pOptions->keep[2] > TSDB_MAX_KEEP) { + pOptions->keep[0] > tsdbMaxKeep || pOptions->keep[1] > tsdbMaxKeep || pOptions->keep[2] > tsdbMaxKeep) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 " valid range: [%dm, %dm]", pOptions->keep[0], pOptions->keep[1], pOptions->keep[2], TSDB_MIN_KEEP, - TSDB_MAX_KEEP); + tsdbMaxKeep); } if (!((pOptions->keep[0] <= pOptions->keep[1]) && (pOptions->keep[1] <= pOptions->keep[2]))) { @@ -4036,7 +4041,10 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName TSDB_MAX_MINROWS_FBLOCK); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbKeepOption(pCxt, pOptions); + code = checkDbPrecisionOption(pCxt, pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbKeepOption(pCxt, pOptions); // use precision } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE); @@ -4049,9 +4057,6 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbRangeOption(pCxt, "tsdbPagesize", pOptions->tsdbPageSize, TSDB_MIN_TSDB_PAGESIZE, TSDB_MAX_TSDB_PAGESIZE); } - if (TSDB_CODE_SUCCESS == code) { - code = checkDbPrecisionOption(pCxt, pOptions); - } if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "replications", pOptions->replica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA); } diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 1f9675a3cd..ee83636192 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -44,6 +44,7 @@ typedef struct SSyncSnapshotSender { SyncTerm term; int64_t startTime; int64_t endTime; + int64_t lastSendTime; bool finish; // init when create diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index f198f3809d..be14ef91a4 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -79,7 +79,6 @@ char* syncUtilPrintBin2(char* ptr, uint32_t len); void syncUtilMsgHtoN(void* msg); void syncUtilMsgNtoH(void* msg); bool syncUtilUserPreCommit(tmsg_t msgType); -bool syncUtilUserCommit(tmsg_t msgType); bool syncUtilUserRollback(tmsg_t msgType); void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNode* pNode, const char* format, ...); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 07b1101256..5fdcbeb91c 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -84,7 +84,7 @@ void syncOneReplicaAdvance(SSyncNode* pSyncNode) { } void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { - ASSERT(false && "deprecated"); + ASSERTS(false, "deprecated"); if (pSyncNode == NULL) { sError("pSyncNode is NULL"); return; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 59e5e968e7..6fabab18cb 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -151,7 +151,7 @@ int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { } syncNodeStartHeartbeatTimer(pSyncNode); - syncNodeReplicate(pSyncNode); + //syncNodeReplicate(pSyncNode); } syncNodeRelease(pSyncNode); @@ -791,9 +791,9 @@ static int32_t syncHbTimerStop(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer) { } int32_t syncNodeLogStoreRestoreOnNeed(SSyncNode* pNode) { - ASSERT(pNode->pLogStore != NULL && "log store not created"); - ASSERT(pNode->pFsm != NULL && "pFsm not registered"); - ASSERT(pNode->pFsm->FpGetSnapshotInfo != NULL && "FpGetSnapshotInfo not registered"); + ASSERTS(pNode->pLogStore != NULL, "log store not created"); + ASSERTS(pNode->pFsm != NULL, "pFsm not registered"); + ASSERTS(pNode->pFsm->FpGetSnapshotInfo != NULL, "FpGetSnapshotInfo not registered"); SSnapshot snapshot; if (pNode->pFsm->FpGetSnapshotInfo(pNode->pFsm, &snapshot) < 0) { sError("vgId:%d, failed to get snapshot info since %s", pNode->vgId, terrstr()); @@ -1144,8 +1144,8 @@ void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode) { } int32_t syncNodeRestore(SSyncNode* pSyncNode) { - ASSERT(pSyncNode->pLogStore != NULL && "log store not created"); - ASSERT(pSyncNode->pLogBuf != NULL && "ring log buffer not created"); + ASSERTS(pSyncNode->pLogStore != NULL, "log store not created"); + ASSERTS(pSyncNode->pLogBuf != NULL, "ring log buffer not created"); SyncIndex lastVer = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); SyncIndex commitIndex = pSyncNode->pLogStore->syncLogCommitIndex(pSyncNode->pLogStore); @@ -1839,7 +1839,8 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { #endif // close receiver - if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { + if (pSyncNode != NULL && pSyncNode->pNewNodeReceiver != NULL && + snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); } @@ -2663,7 +2664,7 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn int32_t code = syncNodeAppend(ths, pEntry); if (code < 0 && ths->vgId != 1 && vnodeIsMsgBlock(pEntry->originalRpcType)) { - ASSERT(false && "failed to append blocking msg"); + ASSERTS(false, "failed to append blocking msg"); } return code; } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 5d6905f99e..7bd8d75dd1 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -50,7 +50,7 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt // initial log buffer with at least one item, e.g. commitIndex SSyncRaftEntry* pMatch = pBuf->entries[(index - 1 + pBuf->size) % pBuf->size].pItem; - ASSERT(pMatch != NULL && "no matched log entry"); + ASSERTS(pMatch != NULL, "no matched log entry"); ASSERT(pMatch->index + 1 == index); SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = pMatch->index, .prevLogTerm = pMatch->term}; @@ -86,14 +86,14 @@ SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, S if (prevIndex >= pBuf->startIndex) { pEntry = pBuf->entries[(prevIndex + pBuf->size) % pBuf->size].pItem; - ASSERT(pEntry != NULL && "no log entry found"); + ASSERTS(pEntry != NULL, "no log entry found"); prevLogTerm = pEntry->term; return prevLogTerm; } if (pMgr && pMgr->startIndex <= prevIndex && prevIndex < pMgr->endIndex) { int64_t timeMs = pMgr->states[(prevIndex + pMgr->size) % pMgr->size].timeMs; - ASSERT(timeMs != 0 && "no log entry found"); + ASSERTS(timeMs != 0, "no log entry found"); prevLogTerm = pMgr->states[(prevIndex + pMgr->size) % pMgr->size].term; ASSERT(prevIndex == 0 || prevLogTerm != 0); return prevLogTerm; @@ -141,9 +141,9 @@ int32_t syncLogValidateAlignmentOfCommit(SSyncNode* pNode, SyncIndex commitIndex } int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { - ASSERT(pNode->pLogStore != NULL && "log store not created"); - ASSERT(pNode->pFsm != NULL && "pFsm not registered"); - ASSERT(pNode->pFsm->FpGetSnapshotInfo != NULL && "FpGetSnapshotInfo not registered"); + ASSERTS(pNode->pLogStore != NULL, "log store not created"); + ASSERTS(pNode->pFsm != NULL, "pFsm not registered"); + ASSERTS(pNode->pFsm->FpGetSnapshotInfo != NULL, "FpGetSnapshotInfo not registered"); SSnapshot snapshot; if (pNode->pFsm->FpGetSnapshotInfo(pNode->pFsm, &snapshot) < 0) { @@ -437,7 +437,7 @@ _out: } int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry) { - ASSERT(pFsm->FpCommitCb != NULL && "No commit cb registered for the FSM"); + ASSERTS(pFsm->FpCommitCb != NULL, "No commit cb registered for the FSM"); if ((pNode->replicaNum == 1) && pNode->restoreFinish && pNode->vgId != 1) { return 0; @@ -513,13 +513,8 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm if (!syncUtilUserCommit(pEntry->originalRpcType)) { sInfo("vgId:%d, commit sync barrier. index: %" PRId64 ", term:%" PRId64 ", type: %s", vgId, pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType)); - pBuf->commitIndex = index; - if (!inBuf) { - syncEntryDestroy(pEntry); - pEntry = NULL; - } - continue; } + if (syncLogFsmExecute(pNode, pFsm, role, term, pEntry) != 0) { sError("vgId:%d, failed to execute sync log entry. index:%" PRId64 ", term:%" PRId64 ", role: %d, current term: %" PRId64, @@ -905,7 +900,7 @@ int32_t syncNodeLogReplMgrInit(SSyncNode* pNode) { ASSERT(pNode->logReplMgrs[i] == NULL); pNode->logReplMgrs[i] = syncLogReplMgrCreate(); pNode->logReplMgrs[i]->peerId = i; - ASSERT(pNode->logReplMgrs[i] != NULL && "Out of memory."); + ASSERTS(pNode->logReplMgrs[i] != NULL, "Out of memory."); } return 0; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index b8ecbe7515..540f40a4c0 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -103,6 +103,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pSender->sendingMS = 0; pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; pSender->startTime = taosGetTimestampMs(); + pSender->lastSendTime = pSender->startTime; pSender->finish = false; // build begin msg @@ -201,6 +202,8 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg); syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, ""); + pSender->lastSendTime = taosGetTimestampMs(); + // event log if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { sSTrace(pSender, "snapshot sender finish"); @@ -213,33 +216,36 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { // send snapshot data from cache int32_t snapshotReSend(SSyncSnapshotSender *pSender) { // send current block data + + // build msg + SRpcMsg rpcMsg = {0}; + (void)syncBuildSnapshotSend(&rpcMsg, pSender->blockLen, pSender->pSyncNode->vgId); + + SyncSnapshotSend *pMsg = rpcMsg.pCont; + pMsg->srcId = pSender->pSyncNode->myRaftId; + pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->beginIndex = pSender->snapshotParam.start; + pMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; + pMsg->lastConfig = pSender->lastConfig; + pMsg->seq = pSender->seq; + if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) { - // build msg - SRpcMsg rpcMsg = {0}; - (void)syncBuildSnapshotSend(&rpcMsg, pSender->blockLen, pSender->pSyncNode->vgId); - - SyncSnapshotSend *pMsg = rpcMsg.pCont; - pMsg->srcId = pSender->pSyncNode->myRaftId; - pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; - pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; - pMsg->beginIndex = pSender->snapshotParam.start; - pMsg->lastIndex = pSender->snapshot.lastApplyIndex; - pMsg->lastTerm = pSender->snapshot.lastApplyTerm; - pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; - pMsg->lastConfig = pSender->lastConfig; - pMsg->seq = pSender->seq; - // pMsg->privateTerm = pSender->privateTerm; memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); - - // send msg - syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg); - syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, ""); - - // event log - sSTrace(pSender, "snapshot sender resend"); } + // send msg + syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg); + syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, ""); + + pSender->lastSendTime = taosGetTimestampMs(); + + // event log + sSTrace(pSender, "snapshot sender resend"); + return 0; } @@ -339,6 +345,8 @@ bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceive void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { // force close, abandon incomplete data if (pReceiver->pWriter != NULL) { + // event log + sRTrace(pReceiver, "snapshot receiver force stop"); int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false, &(pReceiver->snapshot)); ASSERT(ret == 0); @@ -348,7 +356,7 @@ void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { pReceiver->start = false; // event log - sRTrace(pReceiver, "snapshot receiver force stop"); + // sRTrace(pReceiver, "snapshot receiver force stop"); } int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { @@ -669,6 +677,7 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs sNTrace(pSyncNode, "snapshot receiver finish waitting for true time, now:%" PRId64 ", stime:%" PRId64, timeNow, pMsg->startTime); taosMsleep(10); + timeNow = taosGetTimestampMs(); } int32_t code = snapshotReceiverFinish(pReceiver, pMsg); diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 9c39fc1e8e..16e593d0e4 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -20,6 +20,7 @@ #include "syncRaftLog.h" #include "syncReplication.h" #include "syncRespMgr.h" +#include "syncSnapshot.h" #include "syncUtil.h" static void syncNodeCleanConfigIndex(SSyncNode* ths) { @@ -70,6 +71,20 @@ static int32_t syncNodeTimerRoutine(SSyncNode* ths) { } int64_t timeNow = taosGetTimestampMs(); + + for (int i = 0; i < ths->peersNum; ++i) { + SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(ths->peersId[i])); + if (pSender != NULL) { + if (ths->isStart && ths->state == TAOS_SYNC_STATE_LEADER && pSender->start && + timeNow - pSender->lastSendTime > SYNC_SNAP_RESEND_MS) { + snapshotReSend(pSender); + } else { + sTrace("vgId:%d, do not resend: nstart%d, now:%" PRId64 ", lstsend:%" PRId64 ", diff:%" PRId64, ths->vgId, + ths->isStart, timeNow, pSender->lastSendTime, timeNow - pSender->lastSendTime); + } + } + } + if (atomic_load_64(&ths->snapshottingIndex) != SYNC_INDEX_INVALID) { // end timeout wal snapshot if (timeNow - ths->snapshottingTime > SYNC_DEL_WAL_MS && diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 7787438dfe..a034e6ad83 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -160,8 +160,6 @@ void syncUtilMsgNtoH(void* msg) { bool syncUtilUserPreCommit(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; } -bool syncUtilUserCommit(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; } - bool syncUtilUserRollback(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; } void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) { @@ -568,7 +566,7 @@ void syncLogSendSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* p syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-snapshot-send from %s:%d {term:%" PRId64 ", begin:%" PRId64 ", end:%" PRId64 ", lterm:%" PRId64 + "send sync-snapshot-send to %s:%d {term:%" PRId64 ", begin:%" PRId64 ", end:%" PRId64 ", lterm:%" PRId64 ", stime:%" PRId64 ", seq:%d}, %s", host, port, pMsg->term, pMsg->beginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime, pMsg->seq, s); } @@ -595,7 +593,7 @@ void syncLogSendSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMs syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-snapshot-rsp from %s:%d {term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64 ", lterm:%" PRId64 + "send sync-snapshot-rsp to %s:%d {term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64 ", lterm:%" PRId64 ", stime:%" PRId64 ", ack:%d}, %s", host, port, pMsg->term, pMsg->snapBeginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime, pMsg->ack, s); } diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index 1eb803e846..e1a8a5c832 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -19,7 +19,7 @@ SyncAppendEntriesReply *createMsg() { pMsg->success = true; pMsg->matchIndex = 77; pMsg->term = 33; - pMsg->privateTerm = 44; + // pMsg->privateTerm = 44; pMsg->startTime = taosGetTimestampMs(); return pMsg; } diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 9ae79e11fb..7b636085f2 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,5 +1,5 @@ #include "syncTest.h" -#include +// #include /* typedef enum { diff --git a/source/libs/sync/test/sync_test_lib/inc/syncIO.h b/source/libs/sync/test/sync_test_lib/inc/syncIO.h index 19f896182e..98b013b46d 100644 --- a/source/libs/sync/test/sync_test_lib/inc/syncIO.h +++ b/source/libs/sync/test/sync_test_lib/inc/syncIO.h @@ -81,6 +81,8 @@ int32_t syncIOQTimerStop(); int32_t syncIOPingTimerStart(); int32_t syncIOPingTimerStop(); +void syncEntryDestory(SSyncRaftEntry* pEntry); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/test/sync_test_lib/src/syncIO.c b/source/libs/sync/test/sync_test_lib/src/syncIO.c index 2c24451713..415fecd9f2 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncIO.c +++ b/source/libs/sync/test/sync_test_lib/src/syncIO.c @@ -469,3 +469,5 @@ static void syncIOTickPing(void *param, void *tmrId) { taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); } + +void syncEntryDestory(SSyncRaftEntry* pEntry) {} \ No newline at end of file diff --git a/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c index 1ea7629601..3b6007df6b 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c @@ -1583,8 +1583,8 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); - cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); + // cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index c728e29641..10a99bb1fa 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -81,6 +81,8 @@ void tdbFree(void *); typedef struct hashset_st *hashset_t; +void hashset_destroy(hashset_t set); + struct STxn { int flags; int64_t txnId; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8c62f89b64..5c1f264460 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -110,6 +110,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg ret = tdbBegin(pEnv, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); if (ret < 0) { + tdbOsFree(pBt); return -1; } @@ -119,6 +120,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, txn); if (ret < 0) { tdbAbort(pEnv, txn); + tdbOsFree(pBt); return -1; } @@ -126,6 +128,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg if (ret < 0) { tdbError("failed to write page since %s", terrstr()); tdbAbort(pEnv, txn); + tdbOsFree(pBt); return -1; } @@ -138,6 +141,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pBt->info, sizeof(pBt->info), txn); if (ret < 0) { tdbAbort(pEnv, txn); + tdbOsFree(pBt); return -1; } } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index f638ec25a3..e3d886b046 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -379,9 +379,6 @@ int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { return -1; } - if (pTxn->jPageSet) { - hashset_destroy(pTxn->jPageSet); - } // pPager->inTran = 0; return 0; @@ -549,8 +546,6 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { return -1; } - hashset_destroy(pTxn->jPageSet); - // pPager->inTran = 0; return 0; @@ -922,6 +917,8 @@ int tdbPagerRestoreJournals(SPager *pPager, SBTree *pBt) { char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { if (tdbPagerRestore(pPager, pBt, name) < 0) { + tdbCloseDir(&pDir); + tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), name); return -1; } @@ -946,6 +943,8 @@ int tdbPagerRollback(SPager *pPager) { if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { if (tdbOsRemove(name) < 0 && errno != ENOENT) { + tdbCloseDir(&pDir); + tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name); terrno = TAOS_SYSTEM_ERROR(errno); return -1; diff --git a/source/libs/tdb/src/db/tdbTxn.c b/source/libs/tdb/src/db/tdbTxn.c index 77c87d18f2..055d9c7f98 100644 --- a/source/libs/tdb/src/db/tdbTxn.c +++ b/source/libs/tdb/src/db/tdbTxn.c @@ -30,6 +30,11 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void int tdbTxnClose(TXN *pTxn) { if (pTxn) { + if (pTxn->jPageSet) { + hashset_destroy(pTxn->jPageSet); + pTxn->jPageSet = NULL; + } + tdbOsFree(pTxn); } diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp index f4f09b20b8..b16bc643d3 100644 --- a/source/libs/tdb/test/tdbExOVFLTest.cpp +++ b/source/libs/tdb/test/tdbExOVFLTest.cpp @@ -8,6 +8,7 @@ #include #include #include +#include "tlog.h" typedef struct SPoolMem { int64_t size; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 54e80a0009..cd02eb8d5e 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -8,6 +8,7 @@ #include #include #include +#include "tlog.h" typedef struct SPoolMem { int64_t size; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 50594f1a5a..060d1d3a98 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1052,9 +1052,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); - char tbuf[256] = {0}; - EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - if (!EPSET_IS_VALID(&pCtx->epSet)) { tError("invalid epset"); destroyCmsg(pMsg); @@ -1493,18 +1490,35 @@ FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { bool noDelay = true; if (hasEpSet == false) { - // assert(pResp->contLen == 0); if (pResp->contLen == 0) { if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { noDelay = false; } else { EPSET_FORWARD_INUSE(&pCtx->epSet); } - } else { - if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { - noDelay = false; + } else if (pResp->contLen != 0) { + SEpSet epSet; + int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + if (valid < 0) { + tDebug("get invalid epset, epset equal, continue"); + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } } else { - EPSET_FORWARD_INUSE(&pCtx->epSet); + if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + tDebug("epset not equal, retry new epset"); + pCtx->epSet = epSet; + noDelay = false; + } else { + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + tDebug("epset equal, continue"); + EPSET_FORWARD_INUSE(&pCtx->epSet); + } + } } } } else { @@ -1584,7 +1598,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { addConnToPool(pThrd->pool, pConn); } else if (code == TSDB_CODE_SYN_RESTORING) { tTrace("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); - noDelay = cliResetEpset(pCtx, pResp, false); + noDelay = cliResetEpset(pCtx, pResp, true); addConnToPool(pThrd->pool, pConn); transFreeMsg(pResp->pCont); } else { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 73c6dc4011..2b1f68d5f6 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1195,6 +1195,8 @@ void transCloseServer(void* arg) { sendQuitToWorkThrd(srv->pThreadObj[i]); destroyWorkThrd(srv->pThreadObj[i]); } + } else { + uv_loop_close(srv->loop); } taosMemoryFree(srv->pThreadObj); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 421901184b..9175067568 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -163,7 +163,7 @@ int32_t taosMulMkDir(const char *dirname) { code = mkdir(temp, 0755); #endif if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return code; } *pos = TD_DIRSEP[0]; @@ -179,7 +179,7 @@ int32_t taosMulMkDir(const char *dirname) { code = mkdir(temp, 0755); #endif if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return code; } } @@ -225,7 +225,7 @@ int32_t taosMulModeMkDir(const char *dirname, int mode) { code = mkdir(temp, mode); #endif if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return code; } *pos = TD_DIRSEP[0]; @@ -241,7 +241,7 @@ int32_t taosMulModeMkDir(const char *dirname, int mode) { code = mkdir(temp, mode); #endif if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return code; } } @@ -497,3 +497,11 @@ int32_t taosCloseDir(TdDirPtr *ppDir) { return 0; #endif } + +void taosGetCwd(char *buf, int32_t len) { +#if !defined(WINDOWS) + (void)getcwd(buf, len - 1); +#else + strncpy(buf, "not implemented on windows", len -1); +#endif +} diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 68365be481..8c2170239f 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -313,7 +313,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { assert(!(tdFileOptions & TD_FILE_EXCL)); fp = fopen(path, mode); if (fp == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } } else { @@ -336,14 +336,14 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); #endif if (fd == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } } TdFilePtr pFile = (TdFilePtr)taosMemoryMalloc(sizeof(TdFile)); if (pFile == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + // terrno = TSDB_CODE_OUT_OF_MEMORY; if (fd >= 0) close(fd); if (fp != NULL) fclose(fp); return NULL; diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 230ca5b968..2e68bd5ccd 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -348,7 +348,7 @@ void taosMemoryTrim(int32_t size) { void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #ifdef USE_TD_MEMORY - ASSERT(0); + assert(0); #else #if defined(LINUX) void* p = memalign(alignment, size); diff --git a/source/os/src/osString.c b/source/os/src/osString.c index e2d8ce95db..f03778de2f 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -143,15 +143,17 @@ SConv *gConv = NULL; int32_t convUsed = 0; int32_t gConvMaxNum = 0; -void taosConvInit(void) { +int32_t taosConvInit(void) { gConvMaxNum = 512; gConv = taosMemoryCalloc(gConvMaxNum, sizeof(SConv)); for (int32_t i = 0; i < gConvMaxNum; ++i) { gConv[i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); if ((iconv_t)-1 == gConv[i].conv || (iconv_t)0 == gConv[i].conv) { - ASSERT(0); + return -1; } } + + return 0; } void taosConvDestroy() { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 7ec1da0530..89b84ad395 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -97,6 +97,7 @@ LONG WINAPI exceptionHandler(LPEXCEPTION_POINTERS exception); #include #include +#include #else @@ -275,34 +276,34 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { #endif } + + int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { #ifdef WINDOWS snprintf(releaseName, maxLen, "Windows"); return 0; #elif defined(_TD_DARWIN_64) - char line[1024]; - size_t size = 0; - int32_t code = -1; + char osversion[32]; + size_t osversion_len = sizeof(osversion) - 1; + int osversion_name[] = { CTL_KERN, KERN_OSRELEASE }; - TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) return false; - - while ((size = taosGetsFile(pFile, sizeof(line), line)) != -1) { - line[size - 1] = '\0'; - if (strncmp(line, "PRETTY_NAME", 11) == 0) { - const char *p = strchr(line, '=') + 1; - if (*p == '"') { - p++; - line[size - 2] = 0; - } - tstrncpy(releaseName, p, maxLen); - code = 0; - break; - } + if (sysctl(osversion_name, 2, osversion, &osversion_len, NULL, 0) == -1) { + return -1; } - taosCloseFile(&pFile); - return code; + uint32_t major, minor; + if (sscanf(osversion, "%u.%u", &major, &minor) != 2) { + return -1; + } + if (major >= 20) { + major -= 9; // macOS 11 and newer + sprintf(releaseName, "macOS %u.%u", major, minor); + } else { + major -= 4; // macOS 10.1.1 and newer + sprintf(releaseName, "macOS 10.%d.%d", major, minor); + } + + return 0; #else char line[1024]; size_t size = 0; @@ -617,14 +618,14 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { return 0; } else { // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return -1; } #elif defined(_TD_DARWIN_64) struct statvfs info; if (statvfs(dataDir, &info)) { // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return -1; } else { diskSize->total = info.f_blocks * info.f_frsize; @@ -635,7 +636,7 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { #else struct statvfs info; if (statvfs(dataDir, &info)) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return -1; } else { diskSize->total = info.f_blocks * info.f_frsize; diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index 4d6875d263..057d3a620c 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "talgo.h" +#include "tlog.h" #define doswap(__left, __right, __size, __buf) \ do { \ diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index cc823decf4..48638af8d5 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -181,7 +181,7 @@ int32_t tjsonGetObjectValueString(const SJson* pJson, char** pValueString) { int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal) { char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); if (NULL == p) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } strcpy(pVal, p); return TSDB_CODE_SUCCESS; @@ -190,7 +190,7 @@ int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal) { int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal) { char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); if (NULL == p) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } *pVal = strdup(p); return TSDB_CODE_SUCCESS; @@ -199,7 +199,7 @@ int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal) int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal) { char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); if (NULL == p) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } #ifdef WINDOWS sscanf(p, "%" PRId64, pVal); @@ -233,7 +233,7 @@ int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal) { char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); if (NULL == p) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } #ifdef WINDOWS sscanf(p, "%" PRIu64, pVal); @@ -259,6 +259,9 @@ int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pV int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal) { const SJson* pObject = tjsonGetObjectItem(pJson, pName); + if (NULL == pObject) { + return TSDB_CODE_SUCCESS; + } if (!cJSON_IsBool(pObject)) { return TSDB_CODE_FAILED; } @@ -268,6 +271,9 @@ int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal) { int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal) { const SJson* pObject = tjsonGetObjectItem(pJson, pName); + if (NULL == pObject) { + return TSDB_CODE_SUCCESS; + } if (!cJSON_IsNumber(pObject)) { return TSDB_CODE_FAILED; } @@ -282,7 +288,7 @@ SJson* tjsonGetArrayItem(const SJson* pJson, int32_t index) { return cJSON_GetAr int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj) { SJson* pJsonObj = tjsonGetObjectItem(pJson, pName); if (NULL == pJsonObj) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } return func(pJsonObj, pObj); } @@ -294,7 +300,7 @@ int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, v SJson* pJsonObj = tjsonGetObjectItem(pJson, pName); if (NULL == pJsonObj) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } *pObj = taosMemoryCalloc(1, objSize); if (NULL == *pObj) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index be1db74f1a..3825b91c6d 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -72,6 +72,7 @@ static int32_t tsDaylightActive; /* Currently in daylight saving time. */ bool tsLogEmbedded = 0; bool tsAsyncLog = true; +bool tsAssert = true; int32_t tsNumOfLogLines = 10000000; int32_t tsLogKeepDays = 0; LogFp tsLogFp = NULL; @@ -778,3 +779,37 @@ cmp_end: return ret; } + +bool taosAssert(bool condition, const char *file, int32_t line, const char *format, ...) { + if (condition) return false; + + const char *flags = "UTL FATAL "; + ELogLevel level = DEBUG_FATAL; + int32_t dflag = 255; // tsLogEmbedded ? 255 : uDebugFlag + char buffer[LOG_MAX_LINE_BUFFER_SIZE]; + int32_t len = taosBuildLogHead(buffer, flags); + + va_list argpointer; + va_start(argpointer, format); + len = len + vsnprintf(buffer + len, LOG_MAX_LINE_BUFFER_SIZE - len, format, argpointer); + va_end(argpointer); + buffer[len++] = '\n'; + buffer[len] = 0; + taosPrintLogImp(1, 255, buffer, len); + + taosPrintLog(flags, level, dflag, "tAssert at file %s:%d exit:%d", file, line, tsAssert); + taosPrintTrace(flags, level, dflag); + + if (tsAssert) { + taosCloseLog(); + taosMsleep(300); + +#ifdef NDEBUG + abort(); +#else + assert(0); +#endif + } + + return true; +} \ No newline at end of file diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index 76fb9c7067..00379a755b 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -6,6 +6,7 @@ #include "taos.h" #include "taosdef.h" #include "thash.h" +#include "tlog.h" namespace { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c90b2499a4..28ba7ea60f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1022,9 +1022,9 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py -#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py -#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py diff --git a/tests/script/tsim/db/alter_replica_13.sim b/tests/script/tsim/db/alter_replica_13.sim index 4eafb86198..d75acb50ad 100644 --- a/tests/script/tsim/db/alter_replica_13.sim +++ b/tests/script/tsim/db/alter_replica_13.sim @@ -116,6 +116,25 @@ endi print ============= step4: alter database sql alter database db replica 3 +$wt = 0 +stepwt1: + $wt = $wt + 1 + sleep 1000 + if $wt == 200 then + print ====> dnode not ready! + return -1 + endi +sql show transactions +if $rows != 0 then + print wait 1 seconds to alter + goto stepwt1 +endi + +sql show db.vgroups +print ---> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data27 $data28 $data29 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data26 $data37 $data38 $data39 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data36 $data47 $data48 $data49 $leaderIndex = 0 diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim index 47e1fda79f..74e8007791 100644 --- a/tests/script/tsim/db/alter_replica_31.sim +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -148,6 +148,26 @@ endi print ============= step3: alter database sql alter database db replica 1 +$wt = 0 + stepwt1: + $wt = $wt + 1 + sleep 1000 + if $wt == 200 then + print ====> dnode not ready! + return -1 + endi +sql show transactions +if $rows != 0 then + print wait 1 seconds to alter + goto stepwt1 +endi + +sql show db.vgroups +print ---> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data27 $data28 $data29 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data26 $data37 $data38 $data39 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data36 $data47 $data48 $data49 + $hasleader = 0 $x = 0 diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index 94bb909e29..add3ccc51f 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -223,7 +223,7 @@ void shellRunSingleCommandWebsocketImp(char *command) { if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) { fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n"); } else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) { - fprintf(stderr, "TDengine server is down, will try to reconnect\n"); + fprintf(stderr, "TDengine server is disconnected, will try to reconnect\n"); shell.ws_conn = NULL; } ws_free_result(res); diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 47b7adbf18..2597a26243 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -20,6 +20,7 @@ #include #include "taos.h" #include "types.h" +#include "tlog.h" int smlProcess_influx_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -77,11 +78,20 @@ int smlProcess_telnet_Test() { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - const char *sql[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", + char *sql[4] = {0}; + sql[0] = taosMemoryCalloc(1, 128); + sql[1] = taosMemoryCalloc(1, 128); + sql[2] = taosMemoryCalloc(1, 128); + sql[3] = taosMemoryCalloc(1, 128); + const char *sql1[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", "sys.if.bytes.out 1479496102 1.3E3 network=tcp", " sys.procs.running 1479496100 42 host=web01 "}; + for(int i = 0; i < 4; i++){ + strncpy(sql[i], sql1[i], 128); + } + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -1017,7 +1027,7 @@ int smlProcess_18784_Test() { printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); int code = taos_errno(pRes); ASSERT(!code); - ASSERT(taos_affected_rows(pRes) == 2); + ASSERT(taos_affected_rows(pRes) == 1); taos_free_result(pRes); pRes = taos_query(taos, "select * from disk"); @@ -1091,9 +1101,10 @@ int sml_ts2164_Test() { taos_free_result(pRes); const char *sql[] = { +// "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "meters,location=la,groupid=ca current=11.8,voltage=221", "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", +// "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; pRes = taos_query(taos, "use line_test"); @@ -1118,6 +1129,9 @@ int sml_ttl_Test() { const char *sql[] = { "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833739000000", }; + const char *sql1[] = { + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833339000000", + }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -1127,6 +1141,11 @@ int sml_ttl_Test() { printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); + pRes = taos_schemaless_insert_ttl(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + + printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + pRes = taos_query(taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); TAOS_ROW row = taos_fetch_row(pRes);