xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into fix/TS-2349

This commit is contained in:
Shengliang Guan 2022-12-30 17:42:56 +08:00
parent 9c1f2997f1 fcf64fa3d5
commit 929be865f1
71 changed files with 3911 additions and 2751 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/client/taos.h
View File

@ -161,6 +161,8 @@ DLL_EXPORT int taos_stmt_set_tags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags)
DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
// let stmt to reclaim TAOS_FIELD_E that was allocated by `taos_stmt_get_tag_fields`/`taos_stmt_get_col_fields`
DLL_EXPORT void taos_stmt_reclaim_fields(TAOS_STMT *stmt, TAOS_FIELD_E *fields);
DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums);

9
include/libs/function/function.h
View File

@ -138,7 +138,7 @@ typedef struct SqlFunctionCtx {
char *pOutput; // final result output buffer, point to sdata->data
int32_t numOfParams;
// input parameter, e.g., top(k, 20), the number of results of top query is kept in param
SFunctParam *param;
SFunctParam *param;
// corresponding output buffer for timestamp of each result, e.g., diff/csum
SColumnInfoData *pTsOutput;
int32_t offset;
@ -152,6 +152,7 @@ typedef struct SqlFunctionCtx {
struct SSDataBlock *pSrcBlock;
struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity
SSerializeDataHandle saveHandle;
int32_t exprIdx;
char udfName[TSDB_FUNC_NAME_LEN];
} SqlFunctionCtx;
@ -182,9 +183,9 @@ struct SScalarParam {
int32_t numOfQualified; // number of qualified elements in the final results
};
void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell);
bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry);
bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry);
void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell);
bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry);
bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry);
typedef struct SPoint {
int64_t key;

2
include/libs/nodes/plannodes.h
View File

@ -121,6 +121,7 @@ typedef struct SAggLogicNode {
bool hasLast;
bool hasTimeLineFunc;
bool onlyHasKeepOrderFunc;
bool hasGroupKeyOptimized;
} SAggLogicNode;
typedef struct SProjectLogicNode {
@ -402,6 +403,7 @@ typedef struct SAggPhysiNode {
SNodeList* pGroupKeys;
SNodeList* pAggFuncs;
bool mergeDataBlock;
bool groupKeyOptimized;
} SAggPhysiNode;
typedef struct SDownstreamSourceNode {

16
include/libs/stream/tstream.h
View File

@ -392,21 +392,6 @@ static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) {
atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
}
static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) {
if (pTask->outputType == TASK_OUTPUT__TABLE) {
pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
taosFreeQitem(pBlock);
} else if (pTask->outputType == TASK_OUTPUT__SMA) {
pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
taosFreeQitem(pBlock);
} else {
taosWriteQitem(pTask->outputQueue->queue, pBlock);
}
return 0;
}
typedef struct {
SMsgHead head;
int64_t streamId;
@ -584,6 +569,7 @@ int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp);
int32_t streamTryExec(SStreamTask* pTask);
int32_t streamSchedExec(SStreamTask* pTask);
int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock);
int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz);

6
include/os/osMemory.h
View File

@ -22,12 +22,16 @@ extern "C" {
// If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following sectio
#if !defined(WINDOWS)
#ifndef ALLOW_FORBID_FUNC
#define malloc MALLOC_FUNC_TAOS_FORBID
#define calloc CALLOC_FUNC_TAOS_FORBID
#define realloc REALLOC_FUNC_TAOS_FORBID
#define free FREE_FUNC_TAOS_FORBID
#endif
#endif // ifndef ALLOW_FORBID_FUNC
#endif // if !defined(WINDOWS)
void *taosMemoryMalloc(int64_t size);
void *taosMemoryCalloc(int64_t num, int64_t size);

1
include/os/osSysinfo.h
View File

@ -70,6 +70,7 @@ typedef struct {
SysNameInfo taosGetSysNameInfo();
bool taosCheckCurrentInDll();
int taosGetlocalhostname(char *hostname, size_t maxLen);
#ifdef __cplusplus
}

33
include/os/osSystem.h
View File

@ -62,11 +62,38 @@ void taosResetTerminalMode();
taosMemoryFree(strings); \
}
#else
#include <windows.h>
#include <dbghelp.h>
#define STACKSIZE 64
#define taosPrintTrace(flags, level, dflag) \
{ \
taosPrintLog(flags, level, dflag, \
"backtrace not implemented on windows, so detailed stack information cannot be printed"); \
}
unsigned int i; \
void* stack[STACKSIZE]; \
unsigned short frames; \
SYMBOL_INFO* symbol; \
HANDLE process; \
\
process = GetCurrentProcess(); \
\
SymInitialize(process, NULL, TRUE); \
\
frames = CaptureStackBackTrace(0, STACKSIZE, stack, NULL); \
symbol = (SYMBOL_INFO*)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \
if (symbol != NULL) { \
symbol->MaxNameLen = 255; \
symbol->SizeOfStruct = sizeof(SYMBOL_INFO); \
\
if (frames > 0) { \
taosPrintLog(flags, level, dflag, "obtained %d stack frames", frames); \
for (i = 0; i < frames; i++) { \
SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \
taosPrintLog(flags, level, dflag, "frame:%i: %s - 0x%0X", frames - i - 1, symbol->Name, symbol->Address); \
} \
} \
free(symbol); \
} \
}
#endif
#ifdef __cplusplus

29
include/util/tcompare.h
View File

@ -36,17 +36,18 @@ extern "C" {
#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y)))
#define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y)))
#define PATTERN_COMPARE_INFO_INITIALIZER \
{ '%', '_' }
#define PATTERN_COMPARE_INFO_INITIALIZER { '%', '_', L'%', L'_' }
typedef struct SPatternCompareInfo {
char matchAll; // symbol for match all wildcard, default: '%'
char matchOne; // symbol for match one wildcard, default: '_'
char matchAll; // symbol for match all wildcard, default: '%'
char matchOne; // symbol for match one wildcard, default: '_'
TdUcs4 umatchAll; // unicode version matchAll
TdUcs4 umatchOne; // unicode version matchOne
} SPatternCompareInfo;
int32_t patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, const SPatternCompareInfo *pInfo);
int32_t WCSPatternMatch(const TdUcs4 *pattern, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, const SPatternCompareInfo *pInfo);
int32_t taosArrayCompareString(const void *a, const void *b);
@ -79,9 +80,11 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
int32_t compareStrRegexComp(const void *pLeft, const void *pRight);
int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight);
int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight);
int32_t comparestrRegexMatch(const void *pLeft, const void *pRight);
int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight);
int32_t comparewcsRegexMatch(const void *pLeft, const void *pRight);
int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight);
int32_t compareInt8ValDesc(const void *pLeft, const void *pRight);
int32_t compareInt16ValDesc(const void *pLeft, const void *pRight);
@ -99,11 +102,11 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight);
int32_t compareStrPatternMatch(const void *pLeft, const void *pRight);
int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t comparestrPatternMatch(const void *pLeft, const void *pRight);
int32_t comparestrPatternNMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t comparewcsPatternMatch(const void *pLeft, const void *pRight);
int32_t comparewcsPatternNMatch(const void *pLeft, const void *pRight);
int32_t compareInt8Int16(const void *pLeft, const void *pRight);
int32_t compareInt8Int32(const void *pLeft, const void *pRight);

19
include/util/tdef.h
View File

@ -189,12 +189,13 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_COLUMNS 4096
#define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns
#define TSDB_NODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string
#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string
#define TSDB_CGROUP_LEN 193 // it is a null-terminated string
#define TSDB_DB_NAME_LEN 65
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_NODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string
#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string
#define TSDB_CGROUP_LEN 193 // it is a null-terminated string
#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string
#define TSDB_DB_NAME_LEN 65
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_FUNC_NAME_LEN 65
#define TSDB_FUNC_COMMENT_LEN 1024 * 1024
@ -212,8 +213,8 @@ typedef enum ELogicConditionType {
#define TSDB_INDEX_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_INDEX_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TYPE_STR_MAX_LEN 32
#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TOPIC_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_STREAM_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TOPIC_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TOPIC_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_STREAM_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_STREAM_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
#define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20)
#define TSDB_COL_NAME_LEN 65
@ -310,7 +311,7 @@ typedef enum ELogicConditionType {
#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute
#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved.
#define TSDB_MAX_KEEP_NS (365 * 292 * 1440) // data in db to be reserved.
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_MIN_MINROWS_FBLOCK 10
#define TSDB_MAX_MINROWS_FBLOCK 1000
#define TSDB_DEFAULT_MINROWS_FBLOCK 100

11
include/util/tlog.h
View File

@ -83,9 +83,14 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
#endif
;
bool taosAssert(bool condition, const char *file, int32_t line, const char *format, ...);
#define ASSERTS(condition, ...) taosAssert(condition, __FILE__, __LINE__, __VA_ARGS__)
#define ASSERT(condition) ASSERTS(condition, "assert info not provided")
bool taosAssertDebug(bool condition, const char *file, int32_t line, const char *format, ...);
bool taosAssertRelease(bool condition);
#define ASSERTS(condition, ...) taosAssertDebug(condition, __FILE__, __LINE__, __VA_ARGS__)
#ifdef NDEBUG
#define ASSERT(condition) taosAssertRelease(condition)
#else
#define ASSERT(condition) taosAssertDebug(condition, __FILE__, __LINE__, "assert info not provided")
#endif
// clang-format off
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}

6
include/util/tutil.h
View File

@ -29,11 +29,17 @@ extern "C" {
int32_t strdequote(char *src);
size_t strtrim(char *src);
char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote);
TdUcs4* wcsnchr(const TdUcs4* haystack, TdUcs4 needle, size_t len);
char **strsplit(char *src, const char *delim, int32_t *num);
char *strtolower(char *dst, const char *src);
char *strntolower(char *dst, const char *src, int32_t n);
char *strntolower_s(char *dst, const char *src, int32_t n);
int64_t strnatoi(char *num, int32_t len);
size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize);
size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize);
char *strbetween(char *string, char *begin, char *end);
char *paGetToken(char *src, char **token, int32_t *tokenLen);

8
source/client/src/clientMain.c
View File

@ -1331,6 +1331,14 @@ int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fiel
return stmtGetColFields(stmt, fieldNum, fields);
}
// let stmt to reclaim TAOS_FIELD_E that was allocated by `taos_stmt_get_tag_fields`/`taos_stmt_get_col_fields`
void taos_stmt_reclaim_fields(TAOS_STMT *stmt, TAOS_FIELD_E *fields)
{
(void)stmt;
if (!fields) return;
taosMemoryFree(fields);
}
int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
if (stmt == NULL || bind == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);

2
source/common/src/tglobal.c
View File

@ -715,6 +715,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64;
tsSIMDBuiltins = (bool) cfgGetItem(pCfg, "SIMD-builtins")->bval;
tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;
tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN);

5
source/common/src/ttypes.c
View File

@ -131,7 +131,10 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
varDataCopy(val, src);
break;
default: {
memcpy(val, src, len);
if (len > 0) {
memcpy(val, src, len);
}
break;
}
}

3
source/dnode/mgmt/exe/dmMain.c
View File

@ -178,7 +178,7 @@ static int32_t dmInitLog() {
}
static void taosCleanupArgs() {
if (global.envCmd != NULL) taosMemoryFree(global.envCmd);
if (global.envCmd != NULL) taosMemoryFreeClear(global.envCmd);
}
int main(int argc, char const *argv[]) {
@ -271,7 +271,6 @@ int mainWindows(int argc, char **argv) {
taosCleanupCfg();
taosCloseLog();
taosCleanupArgs();
taosConvDestroy();
return -1;
}

18
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE
#include "vmInt.h"
#define MAX_CONTENT_LEN 1024 * 1024
#define MAX_CONTENT_LEN 2 * 1024 * 1024
SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) {
taosThreadRwlockRdlock(&pMgmt->lock);
@ -60,7 +60,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
dInfo("file %s not exist", file);
code = 0;
goto _OVER;
}
@ -133,7 +133,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
*numOfVnodes = vnodesNum;
code = 0;
dDebug("succcessed to read file %s, numOfVnodes:%d", file, vnodesNum);
dInfo("succcessed to read file %s, numOfVnodes:%d", file, vnodesNum);
_OVER:
if (content != NULL) taosMemoryFree(content);
@ -163,6 +163,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
if (ppVnodes == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
dError("failed to write %s while get vnodelist", file);
goto _OVER;
}
@ -172,6 +173,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
dError("failed to write %s while malloc content", file);
goto _OVER;
}
@ -213,6 +215,12 @@ _OVER:
if (code != 0) return -1;
dDebug("succeed to write %s, numOfVnodes:%d", realfile, numOfVnodes);
return taosRenameFile(file, realfile);
dInfo("succeed to write %s, numOfVnodes:%d", realfile, numOfVnodes);
code = taosRenameFile(file, realfile);
if (code != 0) {
dError("failed to rename %s to %s", file, realfile);
}
return code;
}

7
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -215,7 +215,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
if (pVnode != NULL) {
dDebug("vgId:%d, already exist", req.vgId);
dInfo("vgId:%d, already exist", req.vgId);
tFreeSCreateVnodeReq(&req);
vmReleaseVnode(pMgmt, pVnode);
terrno = TSDB_CODE_VND_ALREADY_EXIST;
@ -360,7 +360,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
int32_t vgId = dropReq.vgId;
dDebug("vgId:%d, start to drop vnode", vgId);
dInfo("vgId:%d, start to drop vnode", vgId);
if (dropReq.dnodeId != pMgmt->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_MSG;
@ -370,7 +370,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId);
if (pVnode == NULL) {
dDebug("vgId:%d, failed to drop since %s", vgId, terrstr());
dInfo("vgId:%d, failed to drop since %s", vgId, terrstr());
terrno = TSDB_CODE_VND_NOT_EXIST;
return -1;
}
@ -385,6 +385,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
vmCloseVnode(pMgmt, pVnode);
vmWriteVnodeListToFile(pMgmt);
dInfo("vgId:%d, is dropped", vgId);
return 0;
}

10
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -138,7 +138,7 @@ static void *vmOpenVnodeInThread(void *param) {
SVnodeMgmt *pMgmt = pThread->pMgmt;
char path[TSDB_FILENAME_LEN];
dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
dInfo("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
setThreadName("open-vnodes");
for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
@ -156,14 +156,14 @@ static void *vmOpenVnodeInThread(void *param) {
pThread->failed++;
} else {
vmOpenVnode(pMgmt, pCfg, pImpl);
dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
dInfo("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
pThread->opened++;
atomic_add_fetch_32(&pMgmt->state.openVnodes, 1);
}
}
dDebug("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
pThread->failed);
dInfo("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
pThread->failed);
return NULL;
}
@ -496,7 +496,7 @@ static void *vmRestoreVnodeInThread(void *param) {
dError("vgId:%d, failed to restore vnode by thread:%d", pVnode->vgId, pThread->threadIndex);
pThread->failed++;
} else {
dDebug("vgId:%d, is restored by thread:%d", pVnode->vgId, pThread->threadIndex);
dInfo("vgId:%d, is restored by thread:%d", pVnode->vgId, pThread->threadIndex);
pThread->opened++;
atomic_add_fetch_32(&pMgmt->state.openVnodes, 1);
}

4
source/dnode/vnode/src/meta/metaEntry.c
View File

@ -99,7 +99,9 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
}
if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1;
} else {
ASSERT(0);
metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type);
return -1;
}
tEndDecode(pCoder);

4
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -189,7 +189,8 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
SDecoder* pDecoder = &(SDecoder){0};
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
metaDecodeEntry(pDecoder, &metaEntry);
code = metaDecodeEntry(pDecoder, &metaEntry);
if (code) goto _err;
code = metaHandleEntry(pMeta, &metaEntry);
if (code) goto _err;
@ -198,6 +199,7 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
return code;
_err:
tDecoderClear(pDecoder);
metaError("vgId:%d, vnode snapshot meta write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
return code;
}

14
source/dnode/vnode/src/tq/tq.c
View File

@ -1112,6 +1112,11 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) {
// do recovery step 1
streamSourceRecoverScanStep1(pTask);
if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
streamMetaReleaseTask(pTq->pStreamMeta, pTask);
return 0;
}
// build msg to launch next step
SStreamRecoverStep2Req req;
code = streamBuildSourceRecover2Req(pTask, &req);
@ -1122,6 +1127,10 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) {
streamMetaReleaseTask(pTq->pStreamMeta, pTask);
if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
return 0;
}
// serialize msg
int32_t len = sizeof(SStreamRecoverStep1Req);
@ -1160,6 +1169,11 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t version, char* msg, int32_t m
return -1;
}
if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
streamMetaReleaseTask(pTq->pStreamMeta, pTask);
return 0;
}
// restore param
code = streamRestoreParam(pTask);
if (code < 0) {

4
source/libs/executor/inc/executorimpl.h
View File

@ -693,7 +693,7 @@ void cleanupExprSupp(SExprSupp* pSup);
void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs);
int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey);
const char* pkey, void* pState);
void cleanupAggSup(SAggSupporter* pAggSup);
void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows);
@ -852,6 +852,8 @@ int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResul
int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order);
int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo);
int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos,
int32_t order, int64_t* pData);
#ifdef __cplusplus
}

40
source/libs/executor/src/executorimpl.c
View File

@ -83,6 +83,7 @@ typedef struct SAggOperatorInfo {
uint64_t groupId;
SGroupResInfo groupResInfo;
SExprSupp scalarExprSup;
bool groupKeyOptimized;
} SAggOperatorInfo;
static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock);
@ -604,9 +605,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
}
}
bool isTaskKilled(SExecTaskInfo* pTaskInfo) {
return (0 != pTaskInfo->code) ? true : false;
}
bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code) ? true : false; }
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; }
@ -1349,20 +1348,25 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
}
}
static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock **ppBlock) {
static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock) {
if (!tsCountAlwaysReturnValue) {
return TSDB_CODE_SUCCESS;
}
SAggOperatorInfo* pAggInfo = pOperator->info;
if (pAggInfo->groupKeyOptimized) {
return TSDB_CODE_SUCCESS;
}
SOperatorInfo* downstream = pOperator->pDownstream[0];
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION ||
(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN &&
((STableScanInfo *)downstream->info)->hasGroupByTag == true)) {
((STableScanInfo*)downstream->info)->hasGroupByTag == true)) {
return TSDB_CODE_SUCCESS;
}
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
bool hasCountFunc = false;
bool hasCountFunc = false;
for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
@ -1411,7 +1415,7 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc
return TSDB_CODE_SUCCESS;
}
static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock **ppBlock) {
static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock) {
if (!blockAllocated) {
return;
}
@ -1437,8 +1441,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
int32_t order = TSDB_ORDER_ASC;
int32_t scanFlag = MAIN_SCAN;
bool hasValidBlock = false;
bool blockAllocated = false;
bool hasValidBlock = false;
bool blockAllocated = false;
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
@ -1481,7 +1485,6 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
}
destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
}
// the downstream operator may return with error code, so let's check the code before generating results.
@ -1636,7 +1639,7 @@ void cleanupAggSup(SAggSupporter* pAggSup) {
}
int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey) {
const char* pkey, void* pState) {
int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
if (code != TSDB_CODE_SUCCESS) {
return code;
@ -1648,7 +1651,13 @@ int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo
}
for (int32_t i = 0; i < numOfCols; ++i) {
pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf;
if (pState) {
pSup->pCtx[i].saveHandle.pBuf = NULL;
pSup->pCtx[i].saveHandle.pState = pState;
pSup->pCtx[i].exprIdx = i;
} else {
pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf;
}
}
return TSDB_CODE_SUCCESS;
@ -1733,7 +1742,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1755,11 +1765,13 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
}
pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock;
pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized;
pInfo->groupId = UINT64_MAX;
setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo,
optrDefaultBufFn, NULL);
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pTableScanInfo = downstream->info;

3
source/libs/executor/src/groupoperator.c
View File

@ -458,7 +458,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}

8
source/libs/executor/src/projectoperator.c
View File

@ -102,7 +102,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
}
initResultSizeInfo(&pOperator->resultInfo, numOfRows);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -317,7 +318,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
if (pProjectInfo->mergeDataBlocks) {
if (pRes->info.rows > 0) {
pFinalRes->info.id.groupId = 0; //clear groupId
pFinalRes->info.id.groupId = 0; // clear groupId
pFinalRes->info.version = pRes->info.version;
// continue merge data, ignore the group id
@ -402,7 +403,8 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
initResultSizeInfo(&pOperator->resultInfo, numOfRows);
blockDataEnsureCapacity(pResBlock, numOfRows);
int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}

183
source/libs/executor/src/sysscanoperator.c
View File

@ -65,6 +65,8 @@ typedef struct SSysTableScanInfo {
SSDataBlock* pRes;
int64_t numOfBlocks; // extract basic running information.
SLoadRemoteDataInfo loadInfo;
int32_t tbnameSlotId;
} SSysTableScanInfo;
typedef struct {
@ -346,6 +348,11 @@ static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result);
static int32_t optSysCheckOper(SNode* pOpear);
static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt);
static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableScanInfo* pInfo, const char* name,
SExecTaskInfo* pTaskInfo);
void extractTbnameSlotId(SSysTableScanInfo* pInfo, const SScanPhysiNode* pScanNode);
static SSDataBlock* sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo,
const char* name, SSDataBlock* pBlock);
__optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) {
if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) {
*reverse = true;
@ -1309,83 +1316,111 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
getDBNameFromCondition(pInfo->pCondition, dbName);
sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
}
SSDataBlock* pBlock = NULL;
if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
return sysTableScanUserTables(pOperator);
pBlock = sysTableScanUserTables(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) {
return sysTableScanUserTags(pOperator);
pBlock = sysTableScanUserTags(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite &&
IS_SYS_DBNAME(dbName)) {
return sysTableScanUserSTables(pOperator);
pBlock = sysTableScanUserSTables(pOperator);
} else { // load the meta from mnode of the given epset
if (pOperator->status == OP_EXEC_DONE) {
pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo);
}
return sysTableScanFillTbName(pOperator, pInfo, name, pBlock);
}
static SSDataBlock* sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo,
const char* name, SSDataBlock* pBlock) {
if (pBlock != NULL) {
if (pInfo->tbnameSlotId != -1) {
SColumnInfoData* pColumnInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, pInfo->tbnameSlotId);
char varTbName[TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE] = {0};
memcpy(varDataVal(varTbName), name, strlen(name));
varDataSetLen(varTbName, strlen(name));
for (int i = 0; i < pBlock->info.rows; ++i) {
colDataAppend(pColumnInfoData, i, varTbName, NULL);
}
doFilterResult(pBlock, pOperator->exprSupp.pFilterInfo);
}
}
if (pBlock && pBlock->info.rows != 0) {
return pBlock;
} else {
return NULL;
}
}
static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableScanInfo* pInfo, const char* name,
SExecTaskInfo* pTaskInfo) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
while (1) {
int64_t startTs = taosGetTimestampUs();
tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb));
tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user));
int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req);
char* buf1 = taosMemoryCalloc(1, contLen);
tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req);
// send the fetch remote task result reques
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) {
qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo));
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
while (1) {
int64_t startTs = taosGetTimestampUs();
tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb));
tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user));
int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE
: TDMT_MND_SYSTABLE_RETRIEVE;
int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req);
char* buf1 = taosMemoryCalloc(1, contLen);
tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req);
pMsgSendInfo->param = pOperator;
pMsgSendInfo->msgInfo.pData = buf1;
pMsgSendInfo->msgInfo.len = contLen;
pMsgSendInfo->msgType = msgType;
pMsgSendInfo->fp = loadSysTableCallback;
pMsgSendInfo->requestId = pTaskInfo->id.queryId;
// send the fetch remote task result reques
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) {
qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo));
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
int64_t transporterId = 0;
int32_t code =
asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo);
tsem_wait(&pInfo->ready);
if (pTaskInfo->code) {
qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo),
pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code));
return NULL;
}
SRetrieveMetaTableRsp* pRsp = pInfo->pRsp;
pInfo->req.showId = pRsp->handle;
if (pRsp->numOfRows == 0 || pRsp->completed) {
pOperator->status = OP_EXEC_DONE;
qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo),
pRsp->numOfRows, pInfo->loadInfo.totalRows);
if (pRsp->numOfRows == 0) {
taosMemoryFree(pRsp);
return NULL;
}
}
int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE
: TDMT_MND_SYSTABLE_RETRIEVE;
char* pStart = pRsp->data;
extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart);
updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator);
pMsgSendInfo->param = pOperator;
pMsgSendInfo->msgInfo.pData = buf1;
pMsgSendInfo->msgInfo.len = contLen;
pMsgSendInfo->msgType = msgType;
pMsgSendInfo->fp = loadSysTableCallback;
pMsgSendInfo->requestId = pTaskInfo->id.queryId;
int64_t transporterId = 0;
int32_t code =
asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo);
tsem_wait(&pInfo->ready);
if (pTaskInfo->code) {
qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo),
pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code));
return NULL;
}
SRetrieveMetaTableRsp* pRsp = pInfo->pRsp;
pInfo->req.showId = pRsp->handle;
if (pRsp->numOfRows == 0 || pRsp->completed) {
pOperator->status = OP_EXEC_DONE;
qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo),
pRsp->numOfRows, pInfo->loadInfo.totalRows);
if (pRsp->numOfRows == 0) {
taosMemoryFree(pRsp);
return NULL;
}
}
char* pStart = pRsp->data;
extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart);
updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator);
// todo log the filter info
doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo);
taosMemoryFree(pRsp);
if (pInfo->pRes->info.rows > 0) {
return pInfo->pRes;
} else if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
// todo log the filter info
doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo);
taosMemoryFree(pRsp);
if (pInfo->pRes->info.rows > 0) {
return pInfo->pRes;
} else if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
}
}
@ -1407,6 +1442,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
goto _error;
}
extractTbnameSlotId(pInfo, pScanNode);
pInfo->accountId = pScanPhyNode->accountId;
pInfo->pUser = taosMemoryStrDup((void*)pUser);
pInfo->sysInfo = pScanPhyNode->sysInfo;
@ -1449,6 +1486,26 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
return NULL;
}
void extractTbnameSlotId(SSysTableScanInfo* pInfo, const SScanPhysiNode* pScanNode) {
pInfo->tbnameSlotId = -1;
if (pScanNode->pScanPseudoCols != NULL) {
SNode* pNode = NULL;
FOREACH(pNode, pScanNode->pScanPseudoCols) {
STargetNode* pTargetNode = NULL;
if (nodeType(pNode) == QUERY_NODE_TARGET) {
pTargetNode = (STargetNode*)pNode;
SNode* expr = pTargetNode->pExpr;
if (nodeType(expr) == QUERY_NODE_FUNCTION) {
SFunctionNode* pFuncNode = (SFunctionNode*)expr;
if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) {
pInfo->tbnameSlotId = pTargetNode->slotId;
}
}
}
}
}
}
void destroySysScanOperator(void* param) {
SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param;
tsem_destroy(&pInfo->ready);

27
source/libs/executor/src/tfill.c
View File

@ -39,14 +39,10 @@ static void doSetVal(SColumnInfoData* pDstColInfoData, int32_t rowIndex, const S
static void setNotFillColumn(SFillInfo* pFillInfo, SColumnInfoData* pDstColInfo, int32_t rowIndex, int32_t colIdx) {
SRowVal* p = NULL;
if (FILL_IS_ASC_FILL(pFillInfo)) {
if (pFillInfo->prev.key != 0) {
p = &pFillInfo->prev; // prev has been set value
} else { // otherwise, use the value in the next row
p = &pFillInfo->next;
}
if (pFillInfo->type == TSDB_FILL_NEXT) {
p = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->next : &pFillInfo->prev;
} else {
p = &pFillInfo->next;
p = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next;
}
SGroupKeys* pKey = taosArrayGet(p->pRowVal, colIdx);
@ -257,9 +253,16 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVa
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
int32_t type = pFillInfo->pFillCol[i].pExpr->pExpr->nodeType;
if (type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) {
if ( type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) {
if (!pFillInfo->pFillCol[i].notFillCol && pFillInfo->type != TSDB_FILL_NEXT) {
continue;
}
int32_t srcSlotId = GET_DEST_SLOT_ID(&pFillInfo->pFillCol[i]);
if (srcSlotId == pFillInfo->srcTsSlotId && pFillInfo->type == TSDB_FILL_LINEAR) {
continue;
}
SColumnInfoData* pSrcCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, srcSlotId);
bool isNull = colDataIsNull_s(pSrcCol, rowIndex);
@ -288,8 +291,12 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t
int64_t ts = ((int64_t*)pTsCol->pData)[pFillInfo->index];
// set the next value for interpolation
if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) {
copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, &pFillInfo->next);
if (pFillInfo->currentKey < ts && ascFill) {
SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->next : &pFillInfo->prev;
copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal);
} else if (pFillInfo->currentKey > ts && !ascFill) {
SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->prev : &pFillInfo->next;
copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal);
}
if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) &&

122
source/libs/executor/src/timewindowoperator.c
View File

@ -22,7 +22,7 @@
#include "tfill.h"
#include "ttime.h"
#define IS_FINAL_OP(op) ((op)->isFinal)
#define IS_FINAL_OP(op) ((op)->isFinal)
#define DEAULT_DELETE_MARK (1000LL * 60LL * 60LL * 24LL * 365LL * 10LL);
typedef struct SSessionAggOperatorInfo {
@ -119,8 +119,8 @@ static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsL
pRowSup->groupId = groupId;
}
static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
int16_t pos, int16_t order, int64_t* pData) {
FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
int32_t pos, int32_t order, int64_t* pData) {
int32_t forwardRows = 0;
if (order == TSDB_ORDER_ASC) {
@ -666,8 +666,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, pBlock->info.rows,
numOfExprs);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0,
pBlock->info.rows, numOfExprs);
if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
closeResultRow(pr);
@ -817,13 +817,13 @@ static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {
} else {
int32_t code = comparePullWinKey(pPullInfo, pPullWins, index);
if (code == 0) {
SPullWindowInfo* pos = taosArrayGet(pPullWins ,index);
SPullWindowInfo* pos = taosArrayGet(pPullWins, index);
pos->window.skey = TMIN(pos->window.skey, pPullInfo->window.skey);
pos->window.ekey = TMAX(pos->window.ekey, pPullInfo->window.ekey);
pos->calWin.skey = TMIN(pos->calWin.skey, pPullInfo->calWin.skey);
pos->calWin.ekey = TMAX(pos->calWin.ekey, pPullInfo->calWin.ekey);
return TSDB_CODE_SUCCESS;
} else if (code > 0 ){
} else if (code > 0) {
index++;
}
}
@ -879,10 +879,10 @@ int32_t compareWinRes(void* pKey, void* data, int32_t index) {
} else if (pRKey->groupId < pDataPos->groupId) {
return -1;
}
if (*(int64_t*)pRKey->key > pDataPos->ts) {
return 1;
} else if (*(int64_t*)pRKey->key < pDataPos->ts){
} else if (*(int64_t*)pRKey->key < pDataPos->ts) {
return -1;
}
return 0;
@ -961,8 +961,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
numOfOutput);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
pBlock->info.rows, numOfOutput);
doCloseWindow(pResultRowInfo, pInfo, pResult);
@ -996,8 +996,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
}
#endif
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
numOfOutput);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
pBlock->info.rows, numOfOutput);
doCloseWindow(pResultRowInfo, pInfo, pResult);
}
@ -1164,7 +1164,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
// here we start a new session window
doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
@ -1188,8 +1188,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
pBlock->info.rows, numOfOutput);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
}
static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) {
@ -1394,8 +1394,8 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) {
size_t keyLen = 0;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
void* key = tSimpleHashGetKey(pIte, &keyLen);
uint64_t groupId = *(uint64_t*)key;
void* key = tSimpleHashGetKey(pIte, &keyLen);
uint64_t groupId = *(uint64_t*)key;
TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t));
SResultRowPosition* pPos = (SResultRowPosition*)pIte;
int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
@ -1656,7 +1656,7 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
// the primary timestamp column
bool needed = false;
for(int32_t i = 0; i < numOfCols; ++i) {
for (int32_t i = 0; i < numOfCols; ++i) {
SExprInfo* pExpr = pCtx[i].pExpr;
if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
needed = true;
@ -1724,7 +1724,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSuppor
void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
for (int32_t i = 0; i < numOfExpr; i++) {
// pCtx[i].isStream = true;
// pCtx[i].isStream = true;
}
}
@ -1748,7 +1748,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
int32_t code =
initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1800,8 +1801,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo,
optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -1869,7 +1870,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator
// pInfo->numOfRows data belong to the current session window
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
// here we start a new session window
doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
@ -1886,8 +1887,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
pBlock->info.rows, numOfOutput);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
}
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
@ -2000,7 +2001,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num);
initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -2018,8 +2020,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo,
optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -2068,7 +2070,8 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
SSDataBlock* pResBlock = createDataBlockFromDescNode(pSessionNode->window.node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -2091,8 +2094,8 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo,
optrDefaultBufFn, NULL);
pOperator->pTaskInfo = pTaskInfo;
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -2309,7 +2312,8 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) {
if (needDeleteWindowBuf(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) {
void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
if (!chIds) {
SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
SPullWindowInfo pull = {
.window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
// add pull data request
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
int32_t size1 = taosArrayGetSize(pInfo->pChildren);
@ -2395,7 +2399,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
};
void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && !chIds) {
SPullWindowInfo pull = {.window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
SPullWindowInfo pull = {
.window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
// add pull data request
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
int32_t size = taosArrayGetSize(pInfo->pChildren);
@ -2450,7 +2455,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
pSDataBlock->info.rows, numOfOutput);
pSDataBlock->info.rows, numOfOutput);
SWinKey key = {
.ts = nextWin.skey,
.groupId = groupId,
@ -2545,7 +2550,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
pOperator->status = OP_RES_TO_RETURN;
qDebug("===stream===return data:%s. recv datablock num:%" PRIu64 , IS_FINAL_OP(pInfo) ? "interval final" : "interval semi", pInfo->numOfDatapack);
qDebug("===stream===return data:%s. recv datablock num:%" PRIu64,
IS_FINAL_OP(pInfo) ? "interval final" : "interval semi", pInfo->numOfDatapack);
pInfo->numOfDatapack = 0;
break;
}
@ -2665,7 +2671,7 @@ int64_t getDeleteMark(SIntervalPhysiNode* pIntervalPhyNode) {
if (pIntervalPhyNode->window.deleteMark <= 0) {
return DEAULT_DELETE_MARK;
}
int64_t deleteMark = TMAX(pIntervalPhyNode->window.deleteMark,pIntervalPhyNode->window.watermark);
int64_t deleteMark = TMAX(pIntervalPhyNode->window.deleteMark, pIntervalPhyNode->window.watermark);
deleteMark = TMAX(deleteMark, pIntervalPhyNode->interval);
return deleteMark;
}
@ -2713,7 +2719,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -2778,8 +2785,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->fpSet =
createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo,
optrDefaultBufFn, NULL);
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
}
@ -3578,8 +3585,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo,
optrDefaultBufFn, NULL);
if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
@ -4083,8 +4090,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -4169,8 +4176,8 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR
}
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
pBlock->info.rows, pSup->numOfExprs);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos,
currPos - startPos, pBlock->info.rows, pSup->numOfExprs);
finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo);
resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow));
@ -4190,7 +4197,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
pBlock->info.rows, pSup->numOfExprs);
pBlock->info.rows, pSup->numOfExprs);
}
static void cleanupAfterGroupResultGen(SMergeAlignedIntervalAggOperatorInfo* pMiaInfo, SSDataBlock* pRes) {
@ -4341,7 +4348,8 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -4360,8 +4368,8 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
setOperatorInfo(pOperator, "TimeMergeAlignedIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL,
false, OP_NOT_OPENED, miaInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo,
optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -4496,7 +4504,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo*
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
pBlock->info.rows, numOfOutput);
pBlock->info.rows, numOfOutput);
doCloseWindow(pResultRowInfo, iaInfo, pResult);
// output previous interval results after this interval (&win) is closed
@ -4528,7 +4536,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo*
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
pBlock->info.rows, numOfOutput);
pBlock->info.rows, numOfOutput);
doCloseWindow(pResultRowInfo, iaInfo, pResult);
// output previous interval results after this interval (&nextWin) is closed
@ -4645,7 +4653,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
initResultSizeInfo(&pOperator->resultInfo, 4096);
int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -4665,8 +4674,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo);
setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false,
OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo,
optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -4843,7 +4852,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
initResultSizeInfo(&pOperator->resultInfo, 4096);
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}

13
source/libs/executor/test/executorTests.cpp
View File

@ -43,6 +43,19 @@ enum {
data_desc = 0x3,
};
TEST(testCase, windowFunctionTest) {
int64_t tsCol[100000];
int32_t rows = 100000;
for (int32_t i = 0; i < rows; i++) {
tsCol[i] = 1648791213000 + i;
}
int32_t ekeyNum = 50000;
int32_t pos = 40000;
int64_t ekey = tsCol[ekeyNum];
int32_t num = getForwardStepsInBlock(rows, binarySearchForKey, ekey, pos, TSDB_ORDER_ASC, tsCol);
ASSERT_EQ(num, ekeyNum - pos + 1);
}
typedef struct SDummyInputInfo {
int32_t totalPages; // numOfPages
int32_t current;

4
source/libs/function/inc/builtinsimpl.h
View File

@ -32,7 +32,7 @@ typedef struct SSumRes {
int16_t type;
int64_t prevTs;
bool isPrevTsSet;
bool overflow; // if overflow is true, dsum to be used for any type;
bool overflow; // if overflow is true, dsum to be used for any type;
} SSumRes;
typedef struct SMinmaxResInfo {
@ -46,7 +46,7 @@ typedef struct SMinmaxResInfo {
} SMinmaxResInfo;
int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc);
STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, const STupleKey* pKey);
STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock);
int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos);

137
source/libs/function/src/builtinsimpl.c
View File

@ -41,15 +41,15 @@
#define HLL_BUCKET_MASK (HLL_BUCKETS - 1)
#define HLL_ALPHA_INF 0.721347520444481703680 // constant for 0.5/ln(2)
//typedef struct SMinmaxResInfo {
// bool assign; // assign the first value or not
// int64_t v;
// STuplePos tuplePos;
// typedef struct SMinmaxResInfo {
// bool assign; // assign the first value or not
// int64_t v;
// STuplePos tuplePos;
//
// STuplePos nullTuplePos;
// bool nullTupleSaved;
// int16_t type;
//} SMinmaxResInfo;
// STuplePos nullTuplePos;
// bool nullTupleSaved;
// int16_t type;
// } SMinmaxResInfo;
typedef struct STopBotResItem {
SVariant v;
@ -540,7 +540,7 @@ int32_t countFunction(SqlFunctionCtx* pCtx) {
if (IS_NULL_TYPE(type)) {
// select count(NULL) returns 0
numOfElem = 1;
*((int64_t*)buf) = 0;
*((int64_t*)buf) += 0;
} else {
numOfElem = getNumOfElems(pCtx);
*((int64_t*)buf) += numOfElem;
@ -818,28 +818,31 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple
return;
}
if (pCtx->saveHandle.pBuf != NULL) {
if (pTuplePos->pageId != -1) {
int32_t numOfCols = pCtx->subsidiaries.num;
const char* p = loadTupleData(pCtx, pTuplePos);
if ((pCtx->saveHandle.pBuf != NULL && pTuplePos->pageId != -1) ||
(pCtx->saveHandle.pState && pTuplePos->streamTupleKey.ts > 0)) {
int32_t numOfCols = pCtx->subsidiaries.num;
const char* p = loadTupleData(pCtx, pTuplePos);
bool* nullList = (bool*)p;
char* pStart = (char*)(nullList + numOfCols * sizeof(bool));
bool* nullList = (bool*)p;
char* pStart = (char*)(nullList + numOfCols * sizeof(bool));
// todo set the offset value to optimize the performance.
for (int32_t j = 0; j < numOfCols; ++j) {
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];
int32_t dstSlotId = pc->pExpr->base.resSchema.slotId;
// todo set the offset value to optimize the performance.
for (int32_t j = 0; j < numOfCols; ++j) {
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];
int32_t dstSlotId = pc->pExpr->base.resSchema.slotId;
SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId);
ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes);
if (nullList[j]) {
colDataAppendNULL(pDstCol, rowIndex);
} else {
colDataAppend(pDstCol, rowIndex, pStart, false);
}
pStart += pDstCol->info.bytes;
SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId);
ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes);
if (nullList[j]) {
colDataAppendNULL(pDstCol, rowIndex);
} else {
colDataAppend(pDstCol, rowIndex, pStart, false);
}
pStart += pDstCol->info.bytes;
}
if (pCtx->saveHandle.pState) {
tdbFree((void*)p);
}
}
}
@ -2018,7 +2021,7 @@ static void firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowInde
}
if (!pInfo->hasResult) {
pInfo->pos = saveTupleData(pCtx, rowIndex, pSrcBlock, NULL);
pInfo->pos = saveTupleData(pCtx, rowIndex, pSrcBlock);
} else {
updateTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos);
}
@ -2071,8 +2074,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
// please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function.
// we will use this opt implementation in an new version that is only available in scan subplan
// please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first
// function. we will use this opt implementation in an new version that is only available in scan subplan
#if 0
if (blockDataOrder == TSDB_ORDER_ASC) {
// filter according to current result firstly
@ -2179,7 +2182,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
// please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function.
// please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first
// function.
#if 0
if (blockDataOrder == TSDB_ORDER_ASC) {
for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) {
@ -2236,9 +2240,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
int32_t round = pInput->numOfRows >> 2;
int32_t reminder = pInput->numOfRows & 0x03;
for (int32_t i = pInput->startRowIndex, tick = 0; tick < round; i += 4, tick += 1) {
int64_t cts = pts[i];
int32_t chosen = i;
for (int32_t i = pInput->startRowIndex, tick = 0; tick < round; i += 4, tick += 1) {
int64_t cts = pts[i];
int32_t chosen = i;
if (cts < pts[i + 1]) {
cts = pts[i + 1];
@ -2262,18 +2266,18 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
}
}
for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) {
char* data = colDataGetData(pInputCol, i);
doSaveCurrentVal(pCtx, i, pts[i], type, data);
pResInfo->numOfRes = 1;
}
for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) {
char* data = colDataGetData(pInputCol, i);
doSaveCurrentVal(pCtx, i, pts[i], type, data);
pResInfo->numOfRes = 1;
}
}
} else {
for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
if (colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
continue;
}
} else {
for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
if (colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
continue;
}
numOfElems++;
@ -2439,7 +2443,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pInputCol = pInput->pData[0];
int32_t type = pInputCol->info.type;
int32_t type = pInputCol->info.type;
int32_t bytes = pInputCol->info.bytes;
pInfo->bytes = bytes;
@ -2777,7 +2781,7 @@ int32_t topFunction(SqlFunctionCtx* pCtx) {
}
if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) {
pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
pRes->nullTupleSaved = true;
}
return TSDB_CODE_SUCCESS;
@ -2805,7 +2809,7 @@ int32_t bottomFunction(SqlFunctionCtx* pCtx) {
}
if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) {
pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
pRes->nullTupleSaved = true;
}
@ -2863,7 +2867,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
// save the data of this tuple
if (pCtx->subsidiaries.num > 0) {
pItem->tuplePos = saveTupleData(pCtx, rowIndex, pSrcBlock, NULL);
pItem->tuplePos = saveTupleData(pCtx, rowIndex, pSrcBlock);
}
#ifdef BUF_PAGE_DEBUG
qDebug("page_saveTuple i:%d, item:%p,pageId:%d, offset:%d\n", pEntryInfo->numOfRes, pItem, pItem->tuplePos.pageId,
@ -2937,8 +2941,7 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid
return buf;
}
static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length,
const STupleKey* pKey) {
static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STupleKey key) {
STuplePos p = {0};
if (pHandle->pBuf != NULL) {
SFilePage* pPage = NULL;
@ -2964,20 +2967,31 @@ static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf
releaseBufPage(pHandle->pBuf, pPage);
} else {
// other tuple save policy
if (streamStateFuncPut(pHandle->pState, pKey, pBuf, length) < 0) {
if (streamStateFuncPut(pHandle->pState, &key, pBuf, length) < 0) {
ASSERT(0);
}
p.streamTupleKey = *pKey;
p.streamTupleKey = key;
}
return p;
}
STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, const STupleKey* pKey) {
STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock) {
prepareBuf(pCtx);
STupleKey key;
if (pCtx->saveHandle.pBuf == NULL) {
SColumnInfoData* pColInfo = taosArrayGet(pSrcBlock->pDataBlock, 0);
ASSERT(pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
int64_t skey = *(int64_t*)colDataGetData(pColInfo, rowIndex);
key.groupId = pSrcBlock->info.id.groupId;
key.ts = skey;
key.exprIdx = pCtx->exprIdx;
}
char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf);
return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pKey);
return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, key);
}
static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) {
@ -3623,7 +3637,7 @@ bool histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultIn
pInfo->totalCount = 0;
pInfo->normalized = 0;
char *binTypeStr = strndup(varDataVal(pCtx->param[1].param.pz), varDataLen(pCtx->param[1].param.pz));
char* binTypeStr = strndup(varDataVal(pCtx->param[1].param.pz), varDataLen(pCtx->param[1].param.pz));
int8_t binType = getHistogramBinType(binTypeStr);
taosMemoryFree(binTypeStr);
@ -3947,7 +3961,6 @@ int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
char* data = colDataGetData(pCol, i);
SHLLInfo* pInputInfo = (SHLLInfo*)varDataVal(data);
@ -4423,7 +4436,7 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da
if (pInfo->numSampled < pInfo->samples) {
sampleAssignResult(pInfo, data, pInfo->numSampled);
if (pCtx->subsidiaries.num > 0) {
pInfo->tuplePos[pInfo->numSampled] = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pInfo->tuplePos[pInfo->numSampled] = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
pInfo->numSampled++;
} else {
@ -4454,7 +4467,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) {
}
if (pInfo->numSampled == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) {
pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
pInfo->nullTupleSaved = true;
}
@ -4758,7 +4771,7 @@ static void doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx,
pItem->count += 1;
if (pCtx->subsidiaries.num > 0) {
pItem->tuplePos = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock, NULL);
pItem->tuplePos = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock);
}
taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*));
@ -4798,7 +4811,7 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) {
}
if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) {
pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
pInfo->nullTupleSaved = true;
}
@ -5299,7 +5312,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t len = sprintf(st + VARSTR_HEADER_SIZE,
"Total_Blocks=[%d] Total_Size=[%.2f Kb] Average_size=[%.2f Kb] Compression_Ratio=[%.2f %c]",
pData->numOfBlocks, pData->totalSize / 1024.0, averageSize/1024.0, compRatio, '%');
pData->numOfBlocks, pData->totalSize / 1024.0, averageSize / 1024.0, compRatio, '%');
varDataSetLen(st, len);
colDataAppend(pColInfo, row++, st, false);

37
source/libs/function/src/detail/tminmax.c
View File

@ -64,7 +64,7 @@
static void calculateRounds(int32_t numOfRows, int32_t bytes, int32_t* remainder, int32_t* rounds, int32_t* width) {
const int32_t bitWidth = 256;
*width = (bitWidth>>3u) / bytes;
*width = (bitWidth >> 3u) / bytes;
*remainder = numOfRows % (*width);
*rounds = numOfRows / (*width);
}
@ -92,8 +92,7 @@ static void calculateRounds(int32_t numOfRows, int32_t bytes, int32_t* remainder
(_v) = (_sec)[j]; \
} \
}
static int8_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal) {
int8_t v = 0;
const int8_t* p = pData;
@ -116,7 +115,7 @@ static int8_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFu
const int8_t* q = (const int8_t*)&initVal;
EXTRACT_MAX_VAL(q, p, width, remain, v)
} else { // unsigned value
} else { // unsigned value
for (int32_t i = 0; i < rounds; ++i) {
next = _mm256_lddqu_si256((__m256i*)p);
initVal = _mm256_max_epu8(initVal, next);
@ -126,7 +125,7 @@ static int8_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFu
const uint8_t* q = (const uint8_t*)&initVal;
EXTRACT_MAX_VAL(q, p, width, remain, v)
}
} else { // min function
if (signVal) {
for (int32_t i = 0; i < rounds; ++i) {
@ -241,7 +240,7 @@ static int32_t i32VectorCmpAVX2(const int32_t* pData, int32_t numOfRows, bool is
// let compare the final results
const int32_t* q = (const int32_t*)&initVal;
EXTRACT_MAX_VAL(q, p, width, remain, v)
} else { // unsigned value
} else { // unsigned value
for (int32_t i = 0; i < rounds; ++i) {
next = _mm256_lddqu_si256((__m256i*)p);
initVal = _mm256_max_epi32(initVal, next);
@ -281,7 +280,7 @@ static int32_t i32VectorCmpAVX2(const int32_t* pData, int32_t numOfRows, bool is
}
static float floatVectorCmpAVX(const float* pData, int32_t numOfRows, bool isMinFunc) {
float v = 0;
float v = 0;
const float* p = pData;
int32_t width, remain, rounds;
@ -358,7 +357,7 @@ static double doubleVectorCmpAVX(const double* pData, int32_t numOfRows, bool is
static int32_t findFirstValPosition(const SColumnInfoData* pCol, int32_t start, int32_t numOfRows) {
int32_t i = start;
while (i < (start + numOfRows) && (colDataIsNull_f(pCol->nullbitmap, i) == true)) {
i += 1;
}
@ -495,7 +494,8 @@ static void handleInt64Col(const void* data, int32_t start, int32_t numOfRows, S
}
}
static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc) {
static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf,
bool isMinFunc) {
float* pData = (float*)pCol->pData;
float* val = (float*)&pBuf->v;
@ -525,7 +525,8 @@ static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRo
pBuf->assign = true;
}
static void handleDoubleCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc) {
static void handleDoubleCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf,
bool isMinFunc) {
double* pData = (double*)pCol->pData;
double* val = (double*)&pBuf->v;
@ -740,7 +741,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
}
} else {
@ -754,7 +755,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
}
}
@ -768,7 +769,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
}
}
@ -782,7 +783,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
}
}
@ -798,7 +799,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
}
}
@ -819,7 +820,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
memcpy(&pBuf->v, pCol->pData + (pCol->info.bytes * i), pCol->info.bytes);
if (pCtx->subsidiaries.num > 0) {
pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
}
pBuf->assign = true;
numOfElems = 1;
@ -883,9 +884,9 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
_over:
if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved) {
pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
pBuf->nullTupleSaved = true;
}
return numOfElems;
}
}

1
source/libs/function/src/tudf.c
View File

@ -1230,7 +1230,6 @@ int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *
if (uvTask->rspBuf.base != NULL) {
SUdfResponse rsp = {0};
void *buf = decodeUdfResponse(uvTask->rspBuf.base, &rsp);
assert(uvTask->rspBuf.len == POINTER_DISTANCE(buf, uvTask->rspBuf.base));
task->errCode = rsp.code;
switch (task->type) {

1
source/libs/function/src/udfd.c
View File

@ -400,7 +400,6 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->info.ahandle;
ASSERT(pMsg->info.ahandle != NULL);
if (pEpSet) {
if (!isEpsetEqual(&global.mgmtEp.epSet, pEpSet)) {

1
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -402,6 +402,7 @@ static int32_t logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
CLONE_NODE_LIST_FIELD(pGroupKeys);
CLONE_NODE_LIST_FIELD(pAggFuncs);
COPY_SCALAR_FIELD(hasGroupKeyOptimized);
return TSDB_CODE_SUCCESS;
}

7
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -1829,6 +1829,7 @@ static const char* jkAggPhysiPlanExprs = "Exprs";
static const char* jkAggPhysiPlanGroupKeys = "GroupKeys";
static const char* jkAggPhysiPlanAggFuncs = "AggFuncs";
static const char* jkAggPhysiPlanMergeDataBlock = "MergeDataBlock";
static const char* jkAggPhysiPlanGroupKeyOptimized = "GroupKeyOptimized";
static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) {
const SAggPhysiNode* pNode = (const SAggPhysiNode*)pObj;
@ -1846,6 +1847,9 @@ static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanMergeDataBlock, pNode->mergeDataBlock);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanGroupKeyOptimized, pNode->groupKeyOptimized);
}
return code;
}
@ -1866,6 +1870,9 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkAggPhysiPlanMergeDataBlock, &pNode->mergeDataBlock);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkAggPhysiPlanGroupKeyOptimized, &pNode->groupKeyOptimized);
}
return code;
}

9
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -2370,7 +2370,8 @@ enum {
PHY_AGG_CODE_EXPR,
PHY_AGG_CODE_GROUP_KEYS,
PHY_AGG_CODE_AGG_FUNCS,
PHY_AGG_CODE_MERGE_DATA_BLOCK
PHY_AGG_CODE_MERGE_DATA_BLOCK,
PHY_AGG_CODE_GROUP_KEY_OPTIMIZE
};
static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -2389,6 +2390,9 @@ static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_MERGE_DATA_BLOCK, pNode->mergeDataBlock);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_GROUP_KEY_OPTIMIZE, pNode->groupKeyOptimized);
}
return code;
}
@ -2415,6 +2419,9 @@ static int32_t msgToPhysiAggNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_AGG_CODE_MERGE_DATA_BLOCK:
code = tlvDecodeBool(pTlv, &pNode->mergeDataBlock);
break;
case PHY_AGG_CODE_GROUP_KEY_OPTIMIZE:
code = tlvDecodeBool(pTlv, &pNode->groupKeyOptimized);
break;
default:
break;
}

21
source/libs/parser/inc/parAst.h
View File

@ -186,6 +186,7 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force);
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue);
SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName);
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName,
SNode* pRealTable, SNodeList* pCols, SNode* pOptions);
SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding,
@ -193,15 +194,13 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt
SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pIndexName);
SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId);
SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId);
SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pQuery);
SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SToken* pSubDbName, bool withMeta);
SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pRealTable, bool withMeta);
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName);
SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId,
const SToken* pTopicName);
SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pQuery);
SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName,
bool withMeta);
SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable,
bool withMeta);
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName);
SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId, SToken* pTopicName);
SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue);
SNode* createDefaultExplainOptions(SAstCreateContext* pCxt);
SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal);
@ -214,9 +213,9 @@ SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool
const SToken* pLibPath, SDataType dataType, int32_t bufSize);
SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName);
SNode* createStreamOptions(SAstCreateContext* pCxt);
SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable,
SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable,
SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery);
SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pStreamName);
SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName);
SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId);
SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId);
SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt);

11
source/libs/parser/inc/sql.y
View File

@ -459,8 +459,11 @@ tag_item(A) ::= column_name(B) AS column_alias(C).
/************************************************ create index ********************************************************/
cmd ::= CREATE SMA INDEX not_exists_opt(D)
full_table_name(A) ON full_table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, A, B, NULL, C); }
cmd ::= DROP INDEX exists_opt(B) full_table_name(A). { pCxt->pRootNode = createDropIndexStmt(pCxt, B, A); }
full_index_name(A) ON full_table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, A, B, NULL, C); }
cmd ::= DROP INDEX exists_opt(B) full_index_name(A). { pCxt->pRootNode = createDropIndexStmt(pCxt, B, A); }
full_index_name(A) ::= index_name(B). { A = createRealTableNodeForIndexName(pCxt, NULL, &B); }
full_index_name(A) ::= db_name(B) NK_DOT index_name(C). { A = createRealTableNodeForIndexName(pCxt, &B, &C); }
index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL
NK_LP duration_literal(C) NK_RP sliding_opt(D) sma_stream_opt(E). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), NULL, D, E); }
@ -657,6 +660,10 @@ stream_name(A) ::= NK_ID(B).
%destructor cgroup_name { }
cgroup_name(A) ::= NK_ID(B). { A = B; }
%type index_name { SToken }
%destructor index_name { }
index_name(A) ::= NK_ID(B). { A = B; }
/************************************************ expression **********************************************************/
expr_or_subquery(A) ::= expression(B). { A = B; }
//expr_or_subquery(A) ::= subquery(B). { A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), NULL); }

64
source/libs/parser/src/parAstCreater.c
View File

@ -201,6 +201,24 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) {
return true;
}
static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) {
trimEscape(pTopicName);
if (pTopicName->n >= TSDB_TOPIC_NAME_LEN) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTopicName->z);
return false;
}
return true;
}
static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) {
trimEscape(pStreamName);
if (pStreamName->n >= TSDB_STREAM_NAME_LEN) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pStreamName->z);
return false;
}
return true;
}
static bool checkComment(SAstCreateContext* pCxt, const SToken* pCommentToken, bool demand) {
if (NULL == pCommentToken) {
pCxt->errCode = demand ? TSDB_CODE_PAR_SYNTAX_ERROR : TSDB_CODE_SUCCESS;
@ -1509,6 +1527,13 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const
return (SNode*)pStmt;
}
SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName) {
if (!checkIndexName(pCxt, pIndexName)) {
return NULL;
}
return createRealTableNode(pCxt, pDbName, pIndexName, NULL);
}
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName,
SNode* pRealTable, SNodeList* pCols, SNode* pOptions) {
CHECK_PARSER_STATUS(pCxt);
@ -1567,9 +1592,11 @@ SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, cons
return (SNode*)pStmt;
}
SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pQuery) {
SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pQuery) {
CHECK_PARSER_STATUS(pCxt);
if (!checkTopicName(pCxt, pTopicName)) {
return NULL;
}
SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
@ -1578,10 +1605,10 @@ SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists,
return (SNode*)pStmt;
}
SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SToken* pSubDbName, bool withMeta) {
SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName,
bool withMeta) {
CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pSubDbName, true)) {
if (!checkTopicName(pCxt, pTopicName) || !checkDbName(pCxt, pSubDbName, true)) {
return NULL;
}
SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
@ -1593,9 +1620,12 @@ SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, co
return (SNode*)pStmt;
}
SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pRealTable, bool withMeta) {
SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable,
bool withMeta) {
CHECK_PARSER_STATUS(pCxt);
if (!checkTopicName(pCxt, pTopicName)) {
return NULL;
}
SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
@ -1607,8 +1637,11 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists,
return (SNode*)pStmt;
}
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName) {
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkTopicName(pCxt, pTopicName)) {
return NULL;
}
SDropTopicStmt* pStmt = (SDropTopicStmt*)nodesMakeNode(QUERY_NODE_DROP_TOPIC_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
@ -1617,8 +1650,11 @@ SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const
}
SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId,
const SToken* pTopicName) {
SToken* pTopicName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkTopicName(pCxt, pTopicName)) {
return NULL;
}
SDropCGroupStmt* pStmt = (SDropCGroupStmt*)nodesMakeNode(QUERY_NODE_DROP_CGROUP_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->ignoreNotExists = ignoreNotExists;
@ -1730,9 +1766,12 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) {
return (SNode*)pOptions;
}
SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable,
SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable,
SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery) {
CHECK_PARSER_STATUS(pCxt);
if (!checkStreamName(pCxt, pStreamName)) {
return NULL;
}
SCreateStreamStmt* pStmt = (SCreateStreamStmt*)nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName);
@ -1749,8 +1788,11 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const
return (SNode*)pStmt;
}
SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pStreamName) {
SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkStreamName(pCxt, pStreamName)) {
return NULL;
}
SDropStreamStmt* pStmt = (SDropStreamStmt*)nodesMakeNode(QUERY_NODE_DROP_STREAM_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName);

6
source/libs/parser/src/parTranslater.c
View File

@ -4948,6 +4948,10 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
}
if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped");
}
int32_t tagsLen = 0;
for (int32_t i = 0; i < pTableMeta->tableInfo.numOfTags; ++i) {
tagsLen += pTagsSchema[i].bytes;
@ -7482,7 +7486,7 @@ static void destoryAlterTbReq(SVAlterTbReq* pReq) {
static int32_t rewriteAlterTableImpl(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta,
SQuery* pQuery) {
if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped");
}
if (TSDB_SUPER_TABLE == pTableMeta->tableType) {

4467
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

1
source/libs/planner/src/planLogicCreater.c
View File

@ -545,6 +545,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
pAgg->hasLastRow = pSelect->hasLastRowFunc;
pAgg->hasLast = pSelect->hasLastFunc;
pAgg->hasTimeLineFunc = pSelect->hasTimeLineFunc;
pAgg->hasGroupKeyOptimized = false;
pAgg->onlyHasKeepOrderFunc = pSelect->onlyHasKeepOrderFunc;
pAgg->node.groupAction = getGroupAction(pCxt, pSelect);
pAgg->node.requireDataOrder = getRequireDataOrder(pAgg->hasTimeLineFunc, pSelect);

11
source/libs/planner/src/planOptimizer.c
View File

@ -1544,6 +1544,11 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
code = adjustLogicNodeDataRequirement((SLogicNode*)pScan, pNode->resultDataOrder);
}
if (TSDB_CODE_SUCCESS == code) {
if (QUERY_NODE_LOGIC_PLAN_AGG == pNode->pParent->type) {
SAggLogicNode* pParent = (SAggLogicNode*)(pNode->pParent);
pParent->hasGroupKeyOptimized = true;
}
NODES_CLEAR_LIST(pNode->pChildren);
nodesDestroyNode((SNode*)pNode);
}
@ -1569,6 +1574,8 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
break;
}
}
pAgg->hasGroupKeyOptimized = true;
NODES_DESTORY_LIST(pAgg->pGroupKeys);
if (TSDB_CODE_SUCCESS == code && start >= 0) {
code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, start, pAgg);
@ -1577,6 +1584,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
if (TSDB_CODE_SUCCESS == code) {
code = partTagsOptRebuildTbanme(pScan->pGroupTags);
}
pCxt->optimized = true;
return code;
}
@ -2380,6 +2388,9 @@ static bool tagScanOptShouldBeOptimized(SLogicNode* pNode) {
if (pScan->hasNormalCols) {
return false;
}
if (pScan->tableType == TSDB_SYSTEM_TABLE) {
return false;
}
if (NULL == pNode->pParent || QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) ||
1 != LIST_LENGTH(pNode->pParent->pChildren)) {
return false;

1
source/libs/planner/src/planPhysiCreater.c
View File

@ -872,6 +872,7 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
}
pAgg->mergeDataBlock = (GROUP_ACTION_KEEP == pAggLogicNode->node.groupAction ? false : true);
pAgg->groupKeyOptimized = pAggLogicNode->hasGroupKeyOptimized;
SNodeList* pPrecalcExprs = NULL;
SNodeList* pGroupKeys = NULL;

16
source/libs/planner/src/planSpliter.c
View File

@ -333,13 +333,23 @@ static bool stbSplHasPartTbname(SNodeList* pPartKeys) {
return false;
}
static bool stbSplIsPartTableAgg(SAggLogicNode* pAgg) {
if (NULL != pAgg->pGroupKeys) {
return stbSplHasPartTbname(pAgg->pGroupKeys);
static bool stbSplNotSystemScan(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
return SCAN_TYPE_SYSTEM_TABLE != ((SScanLogicNode*)pNode)->scanType;
} else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
return stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0));
} else {
return true;
}
}
static bool stbSplIsPartTableAgg(SAggLogicNode* pAgg) {
if (1 != LIST_LENGTH(pAgg->node.pChildren)) {
return false;
}
if (NULL != pAgg->pGroupKeys) {
return stbSplHasPartTbname(pAgg->pGroupKeys) && stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0));
}
return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)));
}

17
source/libs/qworker/inc/qwInt.h
View File

@ -114,14 +114,14 @@ typedef struct SQWTaskStatus {
typedef struct SQWTaskCtx {
SRWLatch lock;
int8_t phase;
int8_t inFetch;
int8_t taskType;
int8_t explain;
int8_t needFetch;
int8_t localExec;
int32_t msgType;
int32_t fetchType;
int32_t execId;
int32_t level;
uint64_t sId;
bool queryGotData;
bool queryRsped;
@ -221,8 +221,16 @@ typedef struct SQWorkerMgmt {
#define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase)
#define QW_SET_PHASE(ctx, _value) \
do { \
if ((_value) != QW_PHASE_PRE_FETCH && (_value) != QW_PHASE_POST_FETCH) { \
atomic_store_8(&(ctx)->phase, _value); \
switch (_value) { \
case QW_PHASE_PRE_FETCH: \
ctx->inFetch = 1; \
break; \
case QW_PHASE_POST_FETCH: \
ctx->inFetch = 0; \
break; \
default: \
atomic_store_8(&(ctx)->phase, _value); \
break; \
} \
} while (0)
@ -230,6 +238,7 @@ typedef struct SQWorkerMgmt {
#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code)
#define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY)
#define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch)
#define QW_SET_QTID(id, qId, tId, eId) \
do { \

6
source/libs/qworker/src/qwDbg.c
View File

@ -124,11 +124,11 @@ void qwDbgDumpTasksInfo(SQWorker *mgmt) {
void *key = taosHashGetKey(pIter, NULL);
QW_GET_QTID(key, qId, tId, eId);
QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, msgType:%d, fetchType:%d, "
"execId:%x, level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, "
QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, msgType:%d, "
"sId:%" PRId64 ", level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, "
"rspCode:%x, affectedRows:%" PRId64 ", taskHandle:%p, sinkHandle:%p, tbFName:%s, sver:%d, tver:%d, events:%d,%d,%d,%d,%d",
ctx, ctx->lock, ctx->phase, ctx->taskType, ctx->explain, ctx->needFetch, ctx->localExec, ctx->msgType,
ctx->fetchType, ctx->execId, ctx->level, ctx->queryGotData, ctx->queryRsped, ctx->queryEnd, ctx->queryContinue,
ctx->sId, ctx->level, ctx->queryGotData, ctx->queryRsped, ctx->queryEnd, ctx->queryContinue,
ctx->queryInQueue, ctx->rspCode, ctx->affectedRows, ctx->taskHandle, ctx->sinkHandle, ctx->tbInfo.tbFName,
ctx->tbInfo.sversion, ctx->tbInfo.tversion, ctx->events[QW_EVENT_CANCEL], ctx->events[QW_EVENT_READY],
ctx->events[QW_EVENT_FETCH], ctx->events[QW_EVENT_DROP], ctx->events[QW_EVENT_CQUERY]);

33
source/libs/qworker/src/qworker.c
View File

@ -508,14 +508,6 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp
}
if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
if (QW_PHASE_POST_FETCH == phase) {
QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase));
QW_ERR_JRET(TSDB_CODE_APP_ERROR);
}
// qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code);
// QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code));
QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
QW_ERR_JRET(ctx->rspCode);
}
@ -580,6 +572,7 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx));
ctx->ctrlConnInfo = qwMsg->connInfo;
ctx->sId = sId;
ctx->phase = -1;
QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT));
@ -670,7 +663,7 @@ _return:
qwMsg->connInfo = ctx->dataConnInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code);
qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
rsp = NULL;
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
@ -688,6 +681,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
void *rsp = NULL;
int32_t dataLen = 0;
bool queryStop = false;
bool qComplete = false;
do {
ctx = NULL;
@ -712,17 +706,18 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
}
if (rsp) {
bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd);
qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd);
qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete);
if (qComplete) {
atomic_store_8((int8_t *)&ctx->queryEnd, true);
atomic_store_8((int8_t *)&ctx->queryContinue, 0);
}
qwMsg->connInfo = ctx->dataConnInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code);
qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
rsp = NULL;
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code,
@ -744,14 +739,13 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
rsp = NULL;
qwMsg->connInfo = ctx->dataConnInfo;
qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, NULL, 0, code);
qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, NULL, 0, code);
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
0);
}
QW_LOCK(QW_WRITE, &ctx->lock);
if ((queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code ||
0 == atomic_load_8((int8_t *)&ctx->queryContinue)) {
if (qComplete || (queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code) {
// Note: query is not running anymore
QW_SET_PHASE(ctx, 0);
QW_UNLOCK(QW_WRITE, &ctx->lock);
@ -1178,8 +1172,9 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) {
QW_DLOG("start to stop all tasks, taskNum:%d", taosHashGetSize(mgmt->ctxHash));
uint64_t qId, tId;
uint64_t qId, tId, sId;
int32_t eId;
int64_t rId = 0;
void *pIter = taosHashIterate(mgmt->ctxHash, NULL);
while (pIter) {
SQWTaskCtx *ctx = (SQWTaskCtx *)pIter;
@ -1188,6 +1183,8 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) {
QW_LOCK(QW_WRITE, &ctx->lock);
sId = ctx->sId;
QW_TASK_DLOG_E("start to force stop task");
if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP) || QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) {
@ -1200,9 +1197,11 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) {
if (QW_QUERY_RUNNING(ctx)) {
qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED);
} else if (!QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) {
} else if (QW_FETCH_RUNNING(ctx)) {
QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_VND_STOPPED);
QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP);
QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP);
} else {
qwDropTask(QW_FPARAMS());
}
QW_UNLOCK(QW_WRITE, &ctx->lock);

3
source/libs/scalar/inc/sclInt.h
View File

@ -53,9 +53,6 @@ typedef struct SScalarCtx {
#define SCL_IS_COMPARISON_OPERATOR(_opType) ((_opType) >= OP_TYPE_GREATER_THAN && (_opType) < OP_TYPE_IS_NOT_UNKNOWN)
#define SCL_DOWNGRADE_DATETYPE(_type) \
((_type) == TSDB_DATA_TYPE_BIGINT || TSDB_DATA_TYPE_DOUBLE == (_type) || (_type) == TSDB_DATA_TYPE_UBIGINT)
#define SCL_NO_NEED_CONVERT_COMPARISION(_ltype, _rtype, _optr) \
(IS_NUMERIC_TYPE(_ltype) && IS_NUMERIC_TYPE(_rtype) && \
((_optr) >= OP_TYPE_GREATER_THAN && (_optr) <= OP_TYPE_NOT_EQUAL))
#define sclFatal(...) qFatal(__VA_ARGS__)
#define sclError(...) qError(__VA_ARGS__)

22
source/libs/scalar/src/filter.c
View File

@ -130,9 +130,9 @@ __compar_fn_t gDataCompare[] = {compareInt32Val,
compareFloatVal,
compareDoubleVal,
compareLenPrefixedStr,
compareStrPatternMatch,
comparestrPatternMatch,
compareChkInString,
compareWStrPatternMatch,
comparewcsPatternMatch,
compareLenPrefixedWStr,
compareUint8Val,
compareUint16Val,
@ -142,15 +142,17 @@ __compar_fn_t gDataCompare[] = {compareInt32Val,
setChkInBytes2,
setChkInBytes4,
setChkInBytes8,
compareStrRegexCompMatch,
compareStrRegexCompNMatch,
comparestrRegexMatch,
comparestrRegexNMatch,
setChkNotInBytes1,
setChkNotInBytes2,
setChkNotInBytes4,
setChkNotInBytes8,
compareChkNotInString,
compareStrPatternNotMatch,
compareWStrPatternNotMatch};
comparestrPatternNMatch,
comparewcsPatternNMatch,
comparewcsRegexMatch,
comparewcsRegexNMatch,};
__compar_fn_t gInt8SignCompare[] = {compareInt8Val, compareInt8Int16, compareInt8Int32,
compareInt8Int64, compareInt8Float, compareInt8Double};
@ -295,9 +297,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_NCHAR: {
if (optr == OP_TYPE_MATCH) {
comparFn = 19;
comparFn = 28;
} else if (optr == OP_TYPE_NMATCH) {
comparFn = 20;
comparFn = 29;
} else if (optr == OP_TYPE_LIKE) {
comparFn = 9;
} else if (optr == OP_TYPE_NOT_LIKE) {
@ -336,7 +338,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
__compar_fn_t filterGetCompFunc(int32_t type, int32_t optr) { return gDataCompare[filterGetCompFuncIdx(type, optr)]; }
__compar_fn_t filterGetCompFuncEx(int32_t lType, int32_t rType, int32_t optr) {
if (TSDB_DATA_TYPE_NULL == rType) {
if (TSDB_DATA_TYPE_NULL == rType || TSDB_DATA_TYPE_JSON == rType) {
return NULL;
}
@ -4085,7 +4087,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SC
SArray *pList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(pList, &pSrc);
int32_t code = scalarCalculate(info->sclCtx.node, pList, &output);
code = scalarCalculate(info->sclCtx.node, pList, &output);
taosArrayDestroy(pList);
FLT_ERR_RET(code);

85
source/libs/scalar/src/sclvector.c
View File

@ -37,6 +37,11 @@
#define IS_HELPER_NULL(col, i) colDataIsNull_s(col, i) || IS_JSON_NULL(col->info.type, colDataGetVarData(col, i))
bool noConvertBeforeCompare(int32_t leftType, int32_t rightType, int32_t optr) {
return IS_NUMERIC_TYPE(leftType) && IS_NUMERIC_TYPE(rightType) &&
(optr >= OP_TYPE_GREATER_THAN && optr <= OP_TYPE_NOT_EQUAL);
}
void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType) {
switch (outType) {
case TSDB_DATA_TYPE_BOOL: {
@ -338,6 +343,7 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam *pOut, int32_t rowInd
colDataAppendInt8(pOut->columnData, rowIndex, (int8_t *)&v);
}
// todo remove this malloc
static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) {
int32_t len = 0;
int32_t inputLen = varDataLen(buf);
@ -399,6 +405,8 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
}
pCtx->pOut->numOfRows = pCtx->pIn->numOfRows;
char* tmp = NULL;
for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) {
if (IS_HELPER_NULL(pCtx->pIn->columnData, i)) {
colDataAppendNULL(pCtx->pOut->columnData, i);
@ -421,12 +429,16 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
continue;
}
}
int32_t bufSize = pCtx->pIn->columnData->info.bytes;
char *tmp = taosMemoryMalloc(varDataTLen(data));
if (!tmp) {
sclError("out of memory in vectorConvertFromVarData");
return TSDB_CODE_OUT_OF_MEMORY;
if (tmp == NULL) {
tmp = taosMemoryMalloc(bufSize);
if (tmp == NULL) {
sclError("out of memory in vectorConvertFromVarData");
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (vton) {
memcpy(tmp, data, varDataTLen(data));
} else {
@ -434,6 +446,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
memcpy(tmp, varDataVal(data), varDataLen(data));
tmp[varDataLen(data)] = 0;
} else if (TSDB_DATA_TYPE_NCHAR == convertType) {
// we need to convert it to native char string, and then perform the string to numeric data
ASSERT(varDataLen(data) <= bufSize);
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp);
@ -448,9 +461,11 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
}
(*func)(tmp, pCtx->pOut, i, overflow);
taosMemoryFreeClear(tmp);
}
if (tmp != NULL) {
taosMemoryFreeClear(tmp);
}
return TSDB_CODE_SUCCESS;
}
@ -901,9 +916,11 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) {
int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex,
int32_t numOfRows) {
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
output->numOfRows = input->numOfRows;
SDataType t = {.type = type};
t.bytes = IS_VAR_DATA_TYPE(t.type)? input->columnData->info.bytes:tDataTypes[type].bytes;
int32_t code = sclCreateColumnInfoData(&t, input->numOfRows, output);
if (code != TSDB_CODE_SUCCESS) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -925,25 +942,43 @@ int32_t vectorConvertCols(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara
return TSDB_CODE_SUCCESS;
}
int8_t type = 0;
int32_t code = 0;
SScalarParam *param1 = NULL, *paramOut1 = NULL;
SScalarParam *param2 = NULL, *paramOut2 = NULL;
int32_t code = 0;
if (leftType < rightType) {
// always convert least data
if (IS_VAR_DATA_TYPE(leftType) && IS_VAR_DATA_TYPE(rightType) && (pLeft->numOfRows != pRight->numOfRows) &&
leftType != TSDB_DATA_TYPE_JSON && rightType != TSDB_DATA_TYPE_JSON) {
param1 = pLeft;
param2 = pRight;
paramOut1 = pLeftOut;
paramOut2 = pRightOut;
} else {
param1 = pRight;
param2 = pLeft;
paramOut1 = pRightOut;
paramOut2 = pLeftOut;
}
int8_t type = vectorGetConvertType(GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2));
if (0 == type) {
return TSDB_CODE_SUCCESS;
if (pLeft->numOfRows > pRight->numOfRows) {
type = leftType;
} else {
type = rightType;
}
} else {
// we only define half value in the convert-matrix, so make sure param1 always less equal than param2
if (leftType < rightType) {
param1 = pLeft;
param2 = pRight;
paramOut1 = pLeftOut;
paramOut2 = pRightOut;
} else {
param1 = pRight;
param2 = pLeft;
paramOut1 = pRightOut;
paramOut2 = pLeftOut;
}
type = vectorGetConvertType(GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2));
if (0 == type) {
return TSDB_CODE_SUCCESS;
}
}
if (type != GET_PARAM_TYPE(param1)) {
@ -1683,23 +1718,13 @@ void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *
SScalarParam *param1 = NULL;
SScalarParam *param2 = NULL;
if (SCL_NO_NEED_CONVERT_COMPARISION(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
if (noConvertBeforeCompare(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
param1 = pLeft;
param2 = pRight;
} else {
vectorConvertCols(pLeft, pRight, &pLeftOut, &pRightOut, startIndex, numOfRows);
if (pLeftOut.columnData != NULL) {
param1 = &pLeftOut;
} else {
param1 = pLeft;
}
if (pRightOut.columnData != NULL) {
param2 = &pRightOut;
} else {
param2 = pRight;
}
param1 = (pLeftOut.columnData != NULL) ? &pLeftOut : pLeft;
param2 = (pRightOut.columnData != NULL) ? &pRightOut : pRight;
}
doVectorCompare(param1, param2, pOut, startIndex, numOfRows, _ord, optr);

10
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -344,7 +344,7 @@ TEST(constantTest, int_or_binary) {
ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE);
SValueNode *v = (SValueNode *)res;
ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT);
ASSERT_EQ(v->datum.b, scltLeftV | scltRightV);
ASSERT_EQ(v->datum.i, scltLeftV | scltRightV);
nodesDestroyNode(res);
}
@ -1101,7 +1101,8 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH ||
opType == OP_TYPE_NMATCH) {
printf("op:%s,3result:%d,except:%f\n", operatorTypeStr(opType), *((bool *)colDataGetData(column, 0)), exceptValue);
ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue);
assert(*(bool *)colDataGetData(column, 0) == exceptValue);
// ASSERT_EQ((int) *((bool *)colDataGetData(column, 0)), (int)exceptValue);
}
taosArrayDestroyEx(blockList, scltFreeDataBlock);
@ -1426,7 +1427,7 @@ TEST(columnTest, json_column_logic_op) {
printf("--------------------json string-- 6.6hello {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
key = "k9";
bool eRes8[len + len1] = {false, false, false, false, false, false, false, true, true, false, true, false, true};
bool eRes8[len + len1] = {false, false, false, false, false, false, false, true, true, false, true, true, true};
for (int i = 0; i < len; i++) {
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes8[i], op[i], false);
}
@ -1437,6 +1438,9 @@ TEST(columnTest, json_column_logic_op) {
for (int i = len; i < len + len1; i++) {
void *rightData = prepareNchar(inputNchar[i - len]);
if (i == 11) {
printf("abc\n");
}
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes8[i], op[i], false);
taosMemoryFree(rightData);
}

29
source/libs/stream/src/stream.c
View File

@ -187,6 +187,23 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq,
return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1;
}
int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) {
if (pTask->outputType == TASK_OUTPUT__TABLE) {
pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
taosFreeQitem(pBlock);
} else if (pTask->outputType == TASK_OUTPUT__SMA) {
pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
taosFreeQitem(pBlock);
} else {
ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH);
taosWriteQitem(pTask->outputQueue->queue, pBlock);
streamDispatch(pTask);
}
return 0;
}
int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) {
qDebug("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId,
pReq->upstreamTaskId);
@ -199,9 +216,9 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S
return -1;
}
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
streamDispatch(pTask);
}
/*if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {*/
/*streamDispatch(pTask);*/
/*}*/
} else {
streamSchedExec(pTask);
}
@ -237,9 +254,9 @@ int32_t streamProcessRunReq(SStreamTask* pTask) {
return -1;
}
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
streamDispatch(pTask);
}
/*if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {*/
/*streamDispatch(pTask);*/
/*}*/
return 0;
}

6
source/libs/stream/src/streamExec.c
View File

@ -110,10 +110,14 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
int32_t batchCnt = 0;
while (1) {
if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
return 0;
}
SSDataBlock* output = NULL;
uint64_t ts = 0;
if (qExecTask(exec, &output, &ts) < 0) {
ASSERT(0);
return -1;
}
if (output == NULL) {
if (qStreamRecoverScanFinished(exec)) {

1
source/libs/stream/src/streamRecover.c
View File

@ -200,7 +200,6 @@ int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req*
int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) {
void* exec = pTask->exec.executor;
if (qStreamSourceRecoverStep2(exec, ver) < 0) {
ASSERT(0);
}
return streamScanExec(pTask, 100);
}

2
source/libs/stream/src/streamState.c
View File

@ -107,7 +107,7 @@ static inline int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2,
}
SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages) {
szPage = szPage < 0 ? 4096 : szPage;
szPage = szPage < 0 ? (16 * 1024) : szPage;
pages = pages < 0 ? 256 : pages;
SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState));
if (pState == NULL) {

4
source/libs/tdb/src/db/tdbBtree.c
View File

@ -1742,6 +1742,10 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
}
memcpy(pVal, cd.pVal, cd.vLen);
if (TDB_CELLDECODER_FREE_VAL(&cd)) {
tdbTrace("tdb/btree-next decoder: %p pVal free: %p", &cd, cd.pVal);
tdbFree(cd.pVal);
}
} else {
pVal = NULL;
}

6
source/os/CMakeLists.txt
View File

@ -41,9 +41,13 @@ target_link_libraries(
)
if(TD_WINDOWS)
target_link_libraries(
os PUBLIC ws2_32 iconv msvcregex wcwidth winmm crashdump
os PUBLIC ws2_32 iconv msvcregex wcwidth winmm crashdump dbghelp
)
elseif(TD_DARWIN_64)
find_library(CORE_FOUNDATION_FRAMEWORK CoreFoundation)
target_link_libraries(os PUBLIC ${CORE_FOUNDATION_FRAMEWORK})
find_library(SYSTEM_CONFIGURATION_FRAMEWORK SystemConfiguration)
target_link_libraries(os PUBLIC ${SYSTEM_CONFIGURATION_FRAMEWORK})
target_link_libraries(
os PUBLIC dl m iconv
)

12
source/os/src/osMath.c
View File

@ -16,6 +16,7 @@
#define ALLOW_FORBID_FUNC
#define _DEFAULT_SOURCE
#include "os.h"
#include <stdlib.h>
#ifdef WINDOWS
void swapStr(char* j, char* J, int width) {
@ -33,16 +34,5 @@ void swapStr(char* j, char* J, int width) {
// todo refactor: 1) move away; 2) use merge sort instead; 3) qsort is not a stable sort actually.
void taosSort(void* arr, int64_t sz, int64_t width, __compar_fn_t compar) {
#ifdef WINDOWS
int64_t i, j;
for (i = 0; i < sz - 1; i++) {
for (j = 0; j < sz - 1 - i; j++) {
if (compar((char*)arr + j * width, (char*)arr + (j + 1) * width) > 0.00) {
swapStr((char*)arr + j * width, (char*)arr + (j + 1) * width, width);
}
}
}
#else
qsort(arr, sz, width, compar);
#endif
}

20
source/os/src/osSocket.c
View File

@ -988,7 +988,7 @@ int32_t taosGetFqdn(char *fqdn) {
#endif
char hostname[1024];
hostname[1023] = '\0';
if (gethostname(hostname, 1023) == -1) {
if (taosGetlocalhostname(hostname, 1023) == -1) {
#ifdef WINDOWS
printf("failed to get hostname, reason:%s\n", strerror(WSAGetLastError()));
#else
@ -998,30 +998,28 @@ int32_t taosGetFqdn(char *fqdn) {
return -1;
}
struct addrinfo hints = {0};
struct addrinfo *result = NULL;
#ifdef __APPLE__
// on macosx, hostname -f has the form of xxx.local
// which will block getaddrinfo for a few seconds if AI_CANONNAME is set
// thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return
// immediately
hints.ai_family = AF_INET;
// hints.ai_family = AF_INET;
strcpy(fqdn, hostname);
strcpy(fqdn+strlen(hostname), ".local");
#else // __APPLE__
struct addrinfo hints = {0};
struct addrinfo *result = NULL;
hints.ai_flags = AI_CANONNAME;
#endif // __APPLE__
int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
if (!result) {
fprintf(stderr, "failed to get fqdn, code:%d, reason:%s\n", ret, gai_strerror(ret));
return -1;
}
#ifdef __APPLE__
// refer to comments above
strcpy(fqdn, hostname);
#else // __APPLE__
strcpy(fqdn, result->ai_canonname);
#endif // __APPLE__
freeaddrinfo(result);
#endif // __APPLE__
return 0;
}

51
source/os/src/osSysinfo.c
View File

@ -98,6 +98,9 @@ LONG WINAPI exceptionHandler(LPEXCEPTION_POINTERS exception);
#include <errno.h>
#include <libproc.h>
#include <sys/sysctl.h>
#include <SystemConfiguration/SCDynamicStoreCopySpecific.h>
#include <CoreFoundation/CFString.h>
#include <stdio.h>
#else
@ -1003,6 +1006,11 @@ SysNameInfo taosGetSysNameInfo() {
tstrncpy(info.machine, uts.machine, sizeof(info.machine));
}
char localHostName[512];
taosGetlocalhostname(localHostName, 512);
TdCmdPtr pCmd = taosOpenCmd("scutil --get LocalHostName");
tstrncpy(info.nodename, localHostName, sizeof(info.nodename));
return info;
#else
SysNameInfo info = {0};
@ -1038,3 +1046,46 @@ bool taosCheckCurrentInDll() {
return false;
#endif
}
#ifdef _TD_DARWIN_64
int taosGetMaclocalhostnameByCommand(char *hostname, size_t maxLen) {
TdCmdPtr pCmd = taosOpenCmd("scutil --get LocalHostName");
if (pCmd != NULL) {
if (taosGetsCmd(pCmd, maxLen - 1, hostname) > 0) {
int len = strlen(hostname);
if (hostname[len - 1] == '\n') {
hostname[len - 1] = '\0';
}
return 0;
}
taosCloseCmd(&pCmd);
}
return -1;
}
int getMacLocalHostNameBySCD(char *hostname, size_t maxLen) {
SCDynamicStoreRef store = SCDynamicStoreCreate(NULL, CFSTR(""), NULL, NULL);
CFStringRef hostname_cfstr = SCDynamicStoreCopyLocalHostName(store);
if (hostname_cfstr != NULL) {
CFStringGetCString(hostname_cfstr, hostname, maxLen - 1, kCFStringEncodingMacRoman);
CFRelease(hostname_cfstr);
} else {
return -1;
}
CFRelease(store);
return 0;
}
#endif
int taosGetlocalhostname(char *hostname, size_t maxLen) {
#ifdef _TD_DARWIN_64
int res = getMacLocalHostNameBySCD(hostname, maxLen);
if (res != 0) {
return taosGetMaclocalhostnameByCommand(hostname, maxLen);
} else {
return 0;
}
#else
return gethostname(hostname, maxLen);
#endif
}

24
source/os/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,24 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
PROJECT(TDengine)
FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest)
FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64)
FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64)
IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR))
MESSAGE(STATUS "gTest library found, build os test")
INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ENDIF()
INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc)
# osTests
add_executable(osTests "osTests.cpp")
target_link_libraries(osTests os util gtest_main)
add_test(
NAME osTests
COMMAND osTests
)

38
source/os/test/osTests.cpp
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <iostream>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wformat"
#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
#pragma GCC diagnostic ignored "-Wpointer-arith"
#include "os.h"
TEST(osTest, osSystem) {
const char *flags = "UTL FATAL ";
ELogLevel level = DEBUG_FATAL;
int32_t dflag = 255; // tsLogEmbedded ? 255 : uDebugFlag
taosPrintTrace(flags, level, dflag);
}
#pragma GCC diagnostic pop

248
source/util/src/tcompare.c
View File

@ -17,6 +17,7 @@
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "tcompare.h"
#include "tutil.h"
#include "regex.h"
#include "tdef.h"
#include "thash.h"
@ -997,6 +998,7 @@ int32_t compareUint64Uint32(const void *pLeft, const void *pRight) {
}
int32_t compareJsonValDesc(const void *pLeft, const void *pRight) { return compareJsonVal(pRight, pLeft); }
/*
* Compare two strings
* TSDB_MATCH: Match
@ -1007,59 +1009,62 @@ int32_t compareJsonValDesc(const void *pLeft, const void *pRight) { return compa
* '_': Matches one character
*
*/
int32_t patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, const SPatternCompareInfo *pInfo) {
char c, c1;
int32_t i = 0;
int32_t j = 0;
int32_t o = 0;
int32_t m = 0;
int32_t nMatchChar = 0;
while ((c = patterStr[i++]) != 0) {
while ((i < psize) && ((c = pattern[i++]) != 0)) {
if (c == pInfo->matchAll) { /* Match "*" */
while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) {
while ((i < psize) && ((c = pattern[i++]) == pInfo->matchAll || c == pInfo->matchOne)) {
if (c == pInfo->matchOne) {
if (j > size || str[j++] == 0) {
// empty string, return not match
if (j >= ssize || str[j++] == 0) { // empty string, return not match
return TSDB_PATTERN_NOWILDCARDMATCH;
} else {
++o;
++nMatchChar;
}
}
}
if (c == 0) {
if (i >= psize && (c == pInfo->umatchOne || c == pInfo->umatchAll)) {
return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */
}
char next[3] = {toupper(c), tolower(c), 0};
m = o;
while (1) {
size_t n = strcspn(str + m, next);
str += m + n;
char rejectList[2] = {toupper(c), tolower(c)};
if (str[0] == 0 || (n >= size)) {
str += nMatchChar;
int32_t remain = ssize - nMatchChar;
while (1) {
size_t n = tstrncspn(str, remain, rejectList, 2);
str += n;
remain -= n;
if ((remain <= 0) || str[0] == 0) {
break;
}
int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
int32_t ret = patternMatch(&pattern[i], psize - i, ++str, --remain, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
m = 0;
}
return TSDB_PATTERN_NOWILDCARDMATCH;
}
c1 = str[j++];
++o;
if (j < ssize) {
c1 = str[j++];
++nMatchChar;
if (j <= size) {
if (c == '\\' && patterStr[i] == '_' && c1 == '_') {
if (c == '\\' && pattern[i] == '_' && c1 == '_') {
i++;
continue;
}
if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) {
continue;
}
@ -1068,39 +1073,48 @@ int32_t patternMatch(const char *patterStr, const char *str, size_t size, const
return TSDB_PATTERN_NOMATCH;
}
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
return (j >= ssize || str[j] == 0) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo) {
int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, const SPatternCompareInfo *pInfo) {
TdUcs4 c, c1;
TdUcs4 matchOne = L'_'; // "_"
TdUcs4 matchAll = L'%'; // "%"
int32_t i = 0;
int32_t j = 0;
int32_t nMatchChar = 0;
while ((c = patterStr[i++]) != 0) {
if (c == matchAll) { /* Match "%" */
while ((i < psize) && ((c = pattern[i++]) != 0)) {
if (c == pInfo->umatchAll) { /* Match "%" */
while ((c = patterStr[i++]) == matchAll || c == matchOne) {
if (c == matchOne && (j >= size || str[j++] == 0)) {
return TSDB_PATTERN_NOWILDCARDMATCH;
while ((i < psize) && ((c = pattern[i++]) == pInfo->umatchAll || c == pInfo->umatchOne)) {
if (c == pInfo->umatchOne) {
if (j >= ssize || str[j++] == 0) {
return TSDB_PATTERN_NOWILDCARDMATCH;
} else {
++nMatchChar;
}
}
}
if (c == 0) {
if (i >= psize && (c == pInfo->umatchOne || c == pInfo->umatchAll)) {
return TSDB_PATTERN_MATCH;
}
TdUcs4 accept[3] = {towupper(c), towlower(c), 0};
TdUcs4 rejectList[2] = {towupper(c), towlower(c)};
str += nMatchChar;
int32_t remain = ssize - nMatchChar;
while (1) {
size_t n = wcscspn(str, accept);
size_t n = twcsncspn(str, remain, rejectList, 2);
str += n;
if (str[0] == 0 || (n >= size)) {
remain -= n;
if ((remain <= 0) || str[0] == 0) {
break;
}
int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
int32_t ret = wcsPatternMatch(&pattern[i], psize - i, ++str, --remain, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
@ -1109,10 +1123,16 @@ int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size,
return TSDB_PATTERN_NOWILDCARDMATCH;
}
c1 = str[j++];
if (j < ssize) {
c1 = str[j++];
nMatchChar++;
if (j <= size) {
if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
if (c == L'\\' && pattern[i] == L'_' && c1 == L'_') {
i++;
continue;
}
if (c == c1 || towlower(c) == towlower(c1) || (c == pInfo->umatchOne && c1 != 0)) {
continue;
}
}
@ -1120,16 +1140,39 @@ int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size,
return TSDB_PATTERN_NOMATCH;
}
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
return (j >= ssize || str[j] == 0) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight) { return compareStrRegexComp(pLeft, pRight); }
int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight) {
return compareStrRegexComp(pLeft, pRight) ? 0 : 1;
int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight) {
return comparestrRegexMatch(pLeft, pRight) ? 0 : 1;
}
int32_t compareStrRegexComp(const void *pLeft, const void *pRight) {
static int32_t doExecRegexMatch(const char *pString, const char *pPattern) {
int32_t ret = 0;
regex_t regex;
char msgbuf[256] = {0};
int32_t cflags = REG_EXTENDED;
if ((ret = regcomp(&regex, pPattern, cflags)) != 0) {
regerror(ret, &regex, msgbuf, tListLen(msgbuf));
uError("Failed to compile regex pattern %s. reason %s", pPattern, msgbuf);
regfree(&regex);
return 1;
}
regmatch_t pmatch[1];
ret = regexec(&regex, pString, 1, pmatch, 0);
if (ret != 0 && ret != REG_NOMATCH) {
regerror(ret, &regex, msgbuf, sizeof(msgbuf));
uDebug("Failed to match %s with pattern %s, reason %s", pString, pPattern, msgbuf)
}
regfree(&regex);
return (ret == 0) ? 0 : 1;
}
int32_t comparestrRegexMatch(const void *pLeft, const void *pRight) {
size_t sz = varDataLen(pRight);
char *pattern = taosMemoryMalloc(sz + 1);
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
@ -1140,30 +1183,48 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight) {
memcpy(str, varDataVal(pLeft), sz);
str[sz] = 0;
int32_t errCode = 0;
regex_t regex;
char msgbuf[256] = {0};
int32_t ret = doExecRegexMatch(str, pattern);
int32_t cflags = REG_EXTENDED;
if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
regfree(&regex);
taosMemoryFree(str);
taosMemoryFree(pattern);
return 1;
}
errCode = regexec(&regex, str, 0, NULL, 0);
if (errCode != 0 && errCode != REG_NOMATCH) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
}
int32_t result = (errCode == 0) ? 0 : 1;
regfree(&regex);
taosMemoryFree(str);
taosMemoryFree(pattern);
return result;
return (ret == 0) ? 0 : 1;;
}
int32_t comparewcsRegexMatch(const void* pString, const void* pPattern) {
size_t len = varDataLen(pPattern);
char *pattern = taosMemoryMalloc(len + 1);
int convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pPattern), len, pattern);
if (convertLen < 0) {
taosMemoryFree(pattern);
return TSDB_CODE_APP_ERROR;
}
pattern[convertLen] = 0;
len = varDataLen(pString);
char *str = taosMemoryMalloc(len + 1);
convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pString), len, str);
if (convertLen < 0) {
taosMemoryFree(str);
taosMemoryFree(pattern);
return TSDB_CODE_APP_ERROR;
}
str[convertLen] = 0;
int32_t ret = doExecRegexMatch(str, pattern);
taosMemoryFree(str);
taosMemoryFree(pattern);
return (ret == 0) ? 0 : 1;
}
int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight) {
return comparewcsRegexMatch(pLeft, pRight) ? 0 : 1;
}
int32_t taosArrayCompareString(const void *a, const void *b) {
@ -1173,46 +1234,35 @@ int32_t taosArrayCompareString(const void *a, const void *b) {
return compareLenPrefixedStr(x, y);
}
int32_t compareStrPatternMatch(const void *pLeft, const void *pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN);
char *pattern = taosMemoryCalloc(varDataLen(pRight) + 1, sizeof(char));
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
int32_t comparestrPatternMatch(const void *pLeft, const void *pRight) {
SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER;
ASSERT(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN);
size_t pLen = varDataLen(pRight);
size_t sz = varDataLen(pLeft);
char *buf = taosMemoryMalloc(sz + 1);
memcpy(buf, varDataVal(pLeft), sz);
buf[sz] = 0;
int32_t ret = patternMatch(pattern, buf, sz, &pInfo);
taosMemoryFree(buf);
taosMemoryFree(pattern);
int32_t ret = patternMatch(varDataVal(pRight), pLen, varDataVal(pLeft), sz, &pInfo);
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
}
int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight) {
return compareStrPatternMatch(pLeft, pRight) ? 0 : 1;
int32_t comparestrPatternNMatch(const void *pLeft, const void *pRight) {
return comparestrPatternMatch(pLeft, pRight) ? 0 : 1;
}
int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
int32_t comparewcsPatternMatch(const void *pLeft, const void *pRight) {
SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER;
assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
char *pattern = taosMemoryCalloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1);
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
int32_t ret =
WCSPatternMatch((TdUcs4 *)pattern, (TdUcs4 *)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo);
taosMemoryFree(pattern);
size_t psize = varDataLen(pRight);
int32_t ret = wcsPatternMatch((TdUcs4 *)varDataVal(pRight), psize / TSDB_NCHAR_SIZE, (TdUcs4 *)varDataVal(pLeft),
varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo);
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
}
int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight) {
return compareWStrPatternMatch(pLeft, pRight) ? 0 : 1;
int32_t comparewcsPatternNMatch(const void *pLeft, const void *pRight) {
return comparewcsPatternMatch(pLeft, pRight) ? 0 : 1;
}
__compar_fn_t getComparFunc(int32_t type, int32_t optr) {
__compar_fn_t comparFn = NULL;
@ -1285,13 +1335,13 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
break;
case TSDB_DATA_TYPE_BINARY: {
if (optr == OP_TYPE_MATCH) {
comparFn = compareStrRegexCompMatch;
comparFn = comparestrRegexMatch;
} else if (optr == OP_TYPE_NMATCH) {
comparFn = compareStrRegexCompNMatch;
comparFn = comparestrRegexNMatch;
} else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */
comparFn = compareStrPatternMatch;
comparFn = comparestrPatternMatch;
} else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */
comparFn = compareStrPatternNotMatch;
comparFn = comparestrPatternNMatch;
} else if (optr == OP_TYPE_IN) {
comparFn = compareChkInString;
} else if (optr == OP_TYPE_NOT_IN) {
@ -1305,13 +1355,13 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_NCHAR: {
if (optr == OP_TYPE_MATCH) {
comparFn = compareStrRegexCompMatch;
comparFn = comparewcsRegexMatch;
} else if (optr == OP_TYPE_NMATCH) {
comparFn = compareStrRegexCompNMatch;
comparFn = comparewcsRegexNMatch;
} else if (optr == OP_TYPE_LIKE) {
comparFn = compareWStrPatternMatch;
comparFn = comparewcsPatternMatch;
} else if (optr == OP_TYPE_NOT_LIKE) {
comparFn = compareWStrPatternNotMatch;
comparFn = comparewcsPatternNMatch;
} else if (optr == OP_TYPE_IN) {
comparFn = compareChkInString;
} else if (optr == OP_TYPE_NOT_IN) {

24
source/util/src/tlog.c
View File

@ -790,7 +790,7 @@ cmp_end:
return ret;
}
bool taosAssert(bool condition, const char *file, int32_t line, const char *format, ...) {
bool taosAssertDebug(bool condition, const char *file, int32_t line, const char *format, ...) {
if (condition) return false;
const char *flags = "UTL FATAL ";
@ -822,4 +822,24 @@ bool taosAssert(bool condition, const char *file, int32_t line, const char *form
}
return true;
}
}
#ifdef NDEBUG
bool taosAssertRelease(bool condition) {
if (condition) return false;
const char *flags = "UTL FATAL ";
ELogLevel level = DEBUG_FATAL;
int32_t dflag = 255; // tsLogEmbedded ? 255 : uDebugFlag
taosPrintLog(flags, level, dflag, "tAssert called in release mode, exit:%d", tsAssert);
taosPrintTrace(flags, level, dflag);
if (tsAssert) {
taosMsleep(300);
abort();
}
return true;
}
#endif

80
source/util/src/tutil.c
View File

@ -144,6 +144,16 @@ char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote) {
return NULL;
}
TdUcs4* wcsnchr(const TdUcs4* haystack, TdUcs4 needle, size_t len) {
for(int32_t i = 0; i < len; ++i) {
if (haystack[i] == needle) {
return (TdUcs4*) &haystack[i];
}
}
return NULL;
}
char *strtolower(char *dst, const char *src) {
int32_t esc = 0;
char quote = 0, *p = dst, c;
@ -376,3 +386,73 @@ void taosIp2String(uint32_t ip, char *str) {
void taosIpPort2String(uint32_t ip, uint16_t port, char *str) {
sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port);
}
size_t tstrncspn(const char *str, size_t size, const char *reject, size_t rsize) {
if (rsize == 0 || rsize == 1) {
char* p = strnchr(str, reject[0], size, false);
return (p == NULL)? size:(p-str);
}
/* Use multiple small memsets to enable inlining on most targets. */
unsigned char table[256];
unsigned char *p = memset(table, 0, 64);
memset(p + 64, 0, 64);
memset(p + 128, 0, 64);
memset(p + 192, 0, 64);
unsigned char *s = (unsigned char *)reject;
int32_t index = 0;
do {
p[s[index++]] = 1;
} while (index < rsize);
s = (unsigned char*) str;
int32_t times = size >> 2;
if (times == 0) {
for(int32_t i = 0; i < size; ++i) {
if (p[s[i]]) {
return i;
}
}
return size;
}
index = 0;
uint32_t c0, c1, c2, c3;
for(int32_t i = 0; i < times; ++i, index += 4) {
int32_t j = index;
c0 = p[s[j]];
c1 = p[s[j + 1]];
c2 = p[s[j + 2]];
c3 = p[s[j + 3]];
if ((c0 | c1 | c2 | c3) != 0) {
size_t count = ((i + 1) >> 2);
return (c0 | c1) != 0 ? count - c0 + 1 : count - c2 + 3;
}
}
int32_t offset = times * 4;
for(int32_t i = offset; i < size; ++i) {
if (p[s[i]]) {
return i;
}
}
return size;
}
size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize) {
if (rsize == 0 || rsize == 1) {
TdUcs4* p = wcsnchr(wcs, reject[0], size);
return (p == NULL)? size:(p-wcs);
}
size_t index = 0;
while ((index < size) && (wcsnchr(reject, wcs[index], rsize) == NULL)) {
++index;
}
return index;
}

297
source/util/test/utilTests.cpp
View File

@ -0,0 +1,297 @@
#include <gtest/gtest.h>
#include <stdlib.h>
#include <tutil.h>
#include <random>
#include "tarray.h"
#include "tcompare.h"
namespace {
} // namespace
TEST(utilTest, wchar_pattern_match_test) {
const TdWchar* pattern = L"%1";
int32_t ret = 0;
SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER;
const TdWchar* str0 = L"14";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern), 2, reinterpret_cast<const TdUcs4*>(str0), wcslen(str0), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const TdWchar* str1 = L"11";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern), 2, reinterpret_cast<const TdUcs4*>(str1), wcslen(str1), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* str2 = L"41";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern), 2, reinterpret_cast<const TdUcs4*>(str2), wcslen(str2), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* pattern3 = L"%_";
const TdWchar* str3 = L"88";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern3), 2, reinterpret_cast<const TdUcs4*>(str3), wcslen(str3), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* pattern4 = L"%___";
const TdWchar* str4 = L"88";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern4), 4, reinterpret_cast<const TdUcs4*>(str4), wcslen(str4), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const TdWchar* pattern5 = L"%___";
const TdWchar* str5 = L"883391";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern5), 4, reinterpret_cast<const TdUcs4*>(str5), wcslen(str5), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* pattern6 = L"%___66";
const TdWchar* str6 = L"88339166";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern6), 6, reinterpret_cast<const TdUcs4*>(str6), wcslen(str6), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* pattern7 = L"%____66";
const TdWchar* str7 = L"66166";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern7), 7, reinterpret_cast<const TdUcs4*>(str7), wcslen(str7), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const TdWchar* pattern8 = L"6%____66";
const TdWchar* str8 = L"666166";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern8), 8, reinterpret_cast<const TdUcs4*>(str8), wcslen(str8), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const TdWchar* pattern9 = L"6\\__6";
const TdWchar* str9 = L"6_66";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern9), 6, reinterpret_cast<const TdUcs4*>(str9), wcslen(str9), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* pattern10 = L"%";
const TdWchar* str10 = L"";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern10), 1, reinterpret_cast<const TdUcs4*>(str10), 0, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* pattern11 = L"china%";
const TdWchar* str11 = L"CHI ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern11), 6, reinterpret_cast<const TdUcs4*>(str11), 3, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH);
const TdWchar* pattern12 = L"abc%";
const TdWchar* str12 = L"";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern12), 4, reinterpret_cast<const TdUcs4*>(str12), 0, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH);
}
TEST(utilTest, wchar_pattern_match_no_terminated) {
const TdWchar* pattern = L"%1 ";
int32_t ret = 0;
SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER;
const TdWchar* str0 = L"14 ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern), 2, reinterpret_cast<const TdUcs4*>(str0), 2, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const TdWchar* str1 = L"11 ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern), 2, reinterpret_cast<const TdUcs4*>(str1), 2, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* str2 = L"41 ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern), 2, reinterpret_cast<const TdUcs4*>(str2), 2, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* pattern3 = L"%_ ";
const TdWchar* str3 = L"88 ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern3), 2, reinterpret_cast<const TdUcs4*>(str3), 2, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* pattern4 = L"%___ ";
const TdWchar* str4 = L"88 ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern4), 4, reinterpret_cast<const TdUcs4*>(str4), 2, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const TdWchar* pattern5 = L"%___ ";
const TdWchar* str5 = L"883391 ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern5), 4, reinterpret_cast<const TdUcs4*>(str5), 6, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* pattern6 = L"%___66 ";
const TdWchar* str6 = L"88339166 ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern6), 6, reinterpret_cast<const TdUcs4*>(str6), 8, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* pattern7 = L"%____66 ";
const TdWchar* str7 = L"66166 ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern7), 7, reinterpret_cast<const TdUcs4*>(str7), 5, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const TdWchar* pattern8 = L"6%____66 ";
const TdWchar* str8 = L"666166 ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern8), 8, reinterpret_cast<const TdUcs4*>(str8), 6, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const TdWchar* pattern9 = L"6\\_6 ";
const TdWchar* str9 = L"6_6 ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern9), 4, reinterpret_cast<const TdUcs4*>(str9), 3, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const TdWchar* pattern10 = L"% ";
const TdWchar* str10 = L"6_6 ";
ret = wcsPatternMatch(reinterpret_cast<const TdUcs4*>(pattern10), 1, reinterpret_cast<const TdUcs4*>(str10), 3, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
}
TEST(utilTest, char_pattern_match_test) {
const char* pattern = "%1";
int32_t ret = 0;
SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER;
const char* str0 = "14";
ret = patternMatch(pattern, 2, str0, strlen(str0), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const char* str1 = "11";
ret = patternMatch(pattern, 2, str1, strlen(str1), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* str2 = "41";
ret = patternMatch(pattern, 2, str2, strlen(str2), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* pattern3 = "%_";
const char* str3 = "88";
ret = patternMatch(pattern3, 2, str3, strlen(str3), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* pattern4 = "%___";
const char* str4 = "88";
ret = patternMatch(pattern4, 4, str4, strlen(str4), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const char* pattern5 = "%___";
const char* str5 = "883391";
ret = patternMatch(pattern5, 4, str5, strlen(str5), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* pattern6 = "%___66";
const char* str6 = "88339166";
ret = patternMatch(pattern6, 6, str6, strlen(str6), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* pattern7 = "%____66";
const char* str7 = "66166";
ret = patternMatch(pattern7, 7, str7, strlen(str7), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const char* pattern8 = "6%____66";
const char* str8 = "666166";
ret = patternMatch(pattern8, 8, str8, strlen(str8), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const char* pattern9 = "6\\_6";
const char* str9 = "6_6";
ret = patternMatch(pattern9, 5, str9, strlen(str9), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* pattern10 = "%";
const char* str10 = " ";
ret = patternMatch(pattern10, 1, str10, 0, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* pattern11 = "china%";
const char* str11 = "abc ";
ret = patternMatch(pattern11, 6, str11, 3, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH);
const char* pattern12 = "abc%";
const char* str12 = NULL;
ret = patternMatch(pattern12, 4, str12, 0, &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH);
}
TEST(utilTest, char_pattern_match_no_terminated) {
const char* pattern = "%1 ";
int32_t ret = 0;
SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER;
const char* str0 = "14";
ret = patternMatch(pattern, 2, str0, strlen(str0), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const char* str1 = "11";
ret = patternMatch(pattern, 2, str1, strlen(str1), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* str2 = "41";
ret = patternMatch(pattern, 2, str2, strlen(str2), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* pattern3 = "%_ ";
const char* str3 = "88";
ret = patternMatch(pattern3, 2, str3, strlen(str3), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* pattern4 = "%___ ";
const char* str4 = "88";
ret = patternMatch(pattern4, 4, str4, strlen(str4), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const char* pattern5 = "%___ ";
const char* str5 = "883391";
ret = patternMatch(pattern5, 4, str5, strlen(str5), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* pattern6 = "%___66 ";
const char* str6 = "88339166";
ret = patternMatch(pattern6, 6, str6, strlen(str6), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* pattern7 = "%____66 ";
const char* str7 = "66166";
ret = patternMatch(pattern7, 7, str7, strlen(str7), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const char* pattern8 = "6%____66 ";
const char* str8 = "666166";
ret = patternMatch(pattern8, 8, str8, strlen(str8), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
const char* pattern9 = "6\\_6 ";
const char* str9 = "6_6";
ret = patternMatch(pattern9, 4, str9, strlen(str9), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
const char* pattern10 = "% ";
const char* str10 = "6_6";
ret = patternMatch(pattern10, 1, str10, strlen(str10), &pInfo);
ASSERT_EQ(ret, TSDB_PATTERN_MATCH);
}
TEST(utilTest, tstrncspn) {
const char* p1 = "abc";
const char* reject = "d";
size_t v = tstrncspn(p1, strlen(p1), reject, 1);
ASSERT_EQ(v, 3);
const char* reject1 = "a";
v = tstrncspn(p1, strlen(p1), reject1, 1);
ASSERT_EQ(v, 0);
const char* reject2 = "de";
v = tstrncspn(p1, strlen(p1), reject2, 2);
ASSERT_EQ(v, 3);
const char* p2 = "abcdefghijklmn";
v = tstrncspn(p2, strlen(p2), reject2, 2);
ASSERT_EQ(v, 3);
const char* reject3 = "12345n";
v = tstrncspn(p2, strlen(p2), reject3, 6);
ASSERT_EQ(v, 13);
const char* reject4 = "";
v = tstrncspn(p2, strlen(p2), reject4, 0);
ASSERT_EQ(v, 14);
const char* reject5 = "911";
v = tstrncspn(p2, strlen(p2), reject5, 0);
ASSERT_EQ(v, 14);
}

5
tests/parallel_test/cases.task
View File

@ -408,6 +408,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellError.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellNetChk.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py
,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3
,,n,system-test,python3 ./test.py -f 0-others/udfTest.py
@ -732,6 +733,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3
@ -830,6 +832,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 3
@ -925,6 +928,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 4
@ -1030,6 +1034,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py
#develop test

87
tests/system-test/0-others/backquote_check.py Normal file
View File

@ -0,0 +1,87 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import *
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.dbname = 'db'
self.setsql = TDSetSql()
self.stbname = 'stb'
self.streamname = 'stm'
self.streamtb = 'stm_stb'
def topic_name_check(self):
tdSql.execute(f'create database if not exists {self.dbname}')
tdSql.execute(f'use {self.dbname}')
tdSql.execute(f'create stable {self.stbname} (ts timestamp,c0 int) tags(t0 int)')
for name in [self.dbname,self.stbname]:
type = ''
if name == self.dbname:
type = 'database'
elif name == self.stbname:
type = 'stable'
tdSql.execute(f'create topic if not exists {name} as {type} {name}')
tdSql.query('show topics')
tdSql.checkEqual(tdSql.queryResult[0][0],name)
tdSql.execute(f'drop topic {name}')
tdSql.execute(f'create topic if not exists `{name}` as {type} {name}')
tdSql.query('show topics')
tdSql.checkEqual(tdSql.queryResult[0][0],name)
tdSql.execute(f'drop topic {name}')
tdSql.execute(f'create topic if not exists `{name}` as {type} `{name}`')
tdSql.query('show topics')
tdSql.checkEqual(tdSql.queryResult[0][0],name)
tdSql.execute(f'drop topic {name}')
tdSql.execute(f'create topic if not exists `{name}` as {type} `{name}`')
tdSql.query('show topics')
tdSql.checkEqual(tdSql.queryResult[0][0],name)
tdSql.execute(f'drop topic `{name}`')
def db_name_check(self):
tdSql.execute(f'create database if not exists `{self.dbname}`')
tdSql.execute(f'use `{self.dbname}`')
tdSql.execute(f'drop database {self.dbname}')
def stream_name_check(self):
tdSql.execute(f'create database if not exists {self.dbname}')
tdSql.execute(f'use {self.dbname}')
tdSql.execute(f'create stable {self.stbname} (ts timestamp,c0 int) tags(t0 int)')
tdSql.execute(f'create stream `{self.streamname}` into `{self.streamtb}` as select count(*) from {self.stbname} interval(10s);')
tdSql.query('show streams')
tdSql.checkEqual(tdSql.queryResult[0][0],self.streamname)
tdSql.execute(f'drop stream {self.streamname}')
tdSql.execute(f'drop stable {self.streamtb}')
tdSql.execute(f'create stream {self.streamname} into `{self.streamtb}` as select count(*) from {self.stbname} interval(10s);')
tdSql.query('show streams')
tdSql.checkEqual(tdSql.queryResult[0][0],self.streamname)
tdSql.execute(f'drop stream `{self.streamname}`')
tdSql.execute(f'drop database {self.dbname}')
def run(self):
self.topic_name_check()
self.db_name_check()
self.stream_name_check()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

148
tests/system-test/99-TDcase/TD-21561.py Normal file
View File

@ -0,0 +1,148 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import random
import os
import time
import taos
import subprocess
from faker import Faker
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
from util.dnodes import tdDnodes
from util.dnodes import *
class TDTestCase:
updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1}
def init(self, conn, logSql, replicaVar):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.testcasePath = os.path.split(__file__)[0]
self.testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
self.db = "hyp"
def dropandcreateDB_random(self,database,n):
ts = 1630000000000
num_random = 10
fake = Faker('zh_CN')
tdSql.execute('''drop database if exists %s ;''' %database)
tdSql.execute('''create database %s keep 36500 ;'''%(database))
tdSql.execute('''use %s;'''%database)
tdSql.execute('''create stable %s.stable_1 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \
q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \
tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);'''%database)
tdSql.execute('''create stable %s.stable_2 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \
q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \
tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);'''%database)
for i in range(num_random):
tdSql.execute('''create table %s.table_%d \
(ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%(database,i))
tdSql.execute('''create table %s.stable_1_%d using %s.stable_1 tags('stable_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;'''
%(database,i,database,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1),
fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) ,
fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1)))
# insert data
for i in range(num_random):
for j in range(n):
tdSql.execute('''insert into %s.stable_1_%d (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts)\
values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d) ;'''
% (database,i,ts + i*1000 + j, fake.random_int(min=-2147483647, max=2147483647, step=1),
fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1),
fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) ,
fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i))
tdSql.execute('''insert into %s.table_%d (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) \
values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d) ;'''
% (database,i,ts + i*1000 + j, fake.random_int(min=-2147483647, max=2147483647, step=1),
fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1),
fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) ,
fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i ))
tdSql.query("select count(*) from %s.stable_1;" %database)
tdSql.checkData(0,0,num_random*n)
tdSql.query("select count(*) from %s.table_0;"%database)
tdSql.checkData(0,0,n)
def sqls(self,database,function):
sql0 = f"select {function}(q_tinyint) from {database}.stable_1 where tbname in ('stable_1_1') group by tbname ;"
sql1 = f"select {function}(q_tinyint) from {database}.stable_1 where tbname in ('stable_1_1') group by tbname;"
sql2 = f"select {function}(q_tinyint) from {database}.stable_1 where tbname in ('stable_1_1') and _C0 is not null and _C0 between 1600000000000 and now +1h and (q_nchar like 'nchar%' or q_binary = '0' or q_nchar = 'nchar_' ) and q_bool in (0 , 1) group by tbname ;"
sql3 = f"select * from (select {function}(q_tinyint) from {database}.stable_1 where tbname in ('stable_1_1') and _C0 is not null and _C0 between 1600000000000 and now +1h and (q_nchar like 'nchar%' or q_binary = '0' or q_nchar = 'nchar_' ) and q_bool in (0 , 1) group by tbname) ;"
self.constant_check(sql1,sql0)
self.constant_check(sql1,sql2)
self.constant_check(sql2,sql3)
def check_flushdb(self,database):
functions = ['COUNT', 'HYPERLOGLOG']
for f in functions:
for i in range(20):
self.sqls(database, f);
tdSql.execute(" flush database %s;" %database)
tdLog.info("flush database success")
for i in range(20):
self.sqls(database, f);
def constant_check(self,sql1,sql2):
tdLog.info("\n=============sql1:(%s)___sql2:(%s) ====================\n" %(sql1,sql2))
tdSql.query(sql1)
sql1_value = tdSql.getData(0,0)
tdSql.query("reset query cache")
tdSql.query(sql2)
sql2_value = tdSql.getData(0,0)
self.value_check(sql1_value,sql2_value)
def value_check(self,base_value,check_value):
if base_value==check_value:
tdLog.info(f"checkEqual success, base_value={base_value},check_value={check_value}")
else :
tdLog.exit(f"checkEqual error, base_value=={base_value},check_value={check_value}")
def run(self):
startTime = time.time()
os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
self.dropandcreateDB_random("%s" %self.db, 100)
self.check_flushdb("%s" %self.db)
endTime = time.time()
print("total time %ds" % (endTime - startTime))
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())