xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

other:merge 3.0

This commit is contained in:
Haojun Liao 2022-10-11 22:56:54 +08:00
parent c71810c887 58b0ca6efe
commit d3b37ef294
75 changed files with 1549 additions and 788 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -15,7 +15,7 @@
[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
English | [简体中文](README-CN.md) | [Learn more about TSDB](https://tdengine.com/tsdb/)
English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine.com) | [Learn more about TSDB](https://tdengine.com/tsdb/)
# What is TDengine？
@ -33,7 +33,7 @@ TDengine is an open source, high-performance, cloud native [time-series database
- **[Open Source](https://tdengine.com/tdengine/open-source-time-series-database/)**: TDengine’s core modules, including cluster feature, are all available under open source licenses. It has gathered 18.8k stars on GitHub. There is an active developer community, and over 139k running instances worldwide.
For a full list of TDengine competitive advantages, please [check here](https://tdengine.com/tdengine/)
For a full list of TDengine competitive advantages, please [check here](https://tdengine.com/tdengine/). The easiest way to experience TDengine is through [TDengine Cloud](https://cloud.tdengine.com).
# Documentation

2
cmake/taosadapter_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taosadapter
ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
GIT_TAG be729ab
GIT_TAG cc43ef0
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG 85179e9
GIT_TAG d58230c
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

4
docs/en/14-reference/05-taosbenchmark.md
View File

@ -23,7 +23,7 @@ There are two ways to install taosBenchmark:
TaosBenchmark needs to be executed on the terminal of the operating system, it supports two configuration methods: [Command-line arguments](#command-line-arguments-in-detail) and [JSON configuration file](#configuration-file-parameters-in-detail). These two methods are mutually exclusive. Users can use `-f <json file>` to specify a configuration file. When running taosBenchmark with command-line arguments to control its behavior, users should use other parameters for configuration, but not the `-f` parameter. In addition, taosBenchmark offers a special way of running without parameters.
taosBenchmark supports the complete performance testing of TDengine by providing functionally to write, query, and subscribe. These three functions are mutually exclusive, users can only select one of them each time taosBenchmark runs. The query and subscribe functionalities are only configurable using a json configuration file by specifying the parameter `filetype`, while write can be performed through both the command-line and a configuration file. If you want to test the performance of queries or data subscriptionm configure taosBenchmark with the configuration file. You can modify the value of the `filetype` parameter to specify the function that you want to test.
taosBenchmark supports the complete performance testing of TDengine by providing functionally to write, query, and subscribe. These three functions are mutually exclusive, users can only select one of them each time taosBenchmark runs. The query and subscribe functionalities are only configurable using a json configuration file by specifying the parameter `filetype`, while write can be performed through both the command-line and a configuration file. If you want to test the performance of queries or data subscription configure taosBenchmark with the configuration file. You can modify the value of the `filetype` parameter to specify the function that you want to test.
**Make sure that the TDengine cluster is running correctly before running taosBenchmark. **
@ -340,7 +340,7 @@ The configuration parameters for specifying super table tag columns and data col
- **values**: The value field of the nchar/binary column/label, which will be chosen randomly from the values.
- **sma**: Insert the column into the BSMA. Enter `yes` or `no`. The default is `no`.
- **sma**: Insert the column into the SMA. Enter `yes` or `no`. The default is `no`.
#### insertion behavior configuration parameters

48
docs/examples/c/tmq_example.c
View File

@ -184,22 +184,54 @@ void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) {
tmq_t* build_consumer() {
tmq_conf_res_t code;
tmq_conf_t* conf = tmq_conf_new();
code = tmq_conf_set(conf, "enable.auto.commit", "true");
if (TMQ_CONF_OK != code) return NULL;
if (TMQ_CONF_OK != code) {
tmq_conf_destroy(conf);
return NULL;
}
code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
if (TMQ_CONF_OK != code) return NULL;
if (TMQ_CONF_OK != code) {
tmq_conf_destroy(conf);
return NULL;
}
code = tmq_conf_set(conf, "group.id", "cgrpName");
if (TMQ_CONF_OK != code) return NULL;
if (TMQ_CONF_OK != code) {
tmq_conf_destroy(conf);
return NULL;
}
code = tmq_conf_set(conf, "client.id", "user defined name");
if (TMQ_CONF_OK != code) return NULL;
if (TMQ_CONF_OK != code) {
tmq_conf_destroy(conf);
return NULL;
}
code = tmq_conf_set(conf, "td.connect.user", "root");
if (TMQ_CONF_OK != code) return NULL;
if (TMQ_CONF_OK != code) {
tmq_conf_destroy(conf);
return NULL;
}
code = tmq_conf_set(conf, "td.connect.pass", "taosdata");
if (TMQ_CONF_OK != code) return NULL;
if (TMQ_CONF_OK != code) {
tmq_conf_destroy(conf);
return NULL;
}
code = tmq_conf_set(conf, "auto.offset.reset", "earliest");
if (TMQ_CONF_OK != code) return NULL;
if (TMQ_CONF_OK != code) {
tmq_conf_destroy(conf);
return NULL;
}
code = tmq_conf_set(conf, "experimental.snapshot.enable", "false");
if (TMQ_CONF_OK != code) return NULL;
if (TMQ_CONF_OK != code) {
tmq_conf_destroy(conf);
return NULL;
}
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);

2
docs/zh/14-reference/05-taosbenchmark.md
View File

@ -340,7 +340,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
- **values** : nchar/binary 列/标签的值域，将从值中随机选择。
- **sma**: 将该列加入bsma中，值为 "yes" 或者 "no"，默认为 "no"。
- **sma**: 将该列加入 SMA 中，值为 "yes" 或者 "no"，默认为 "no"。
#### 插入行为配置参数

3
include/common/tcommon.h
View File

@ -171,7 +171,8 @@ typedef struct SDataBlockInfo {
STimeWindow calWin; // used for stream, do not serialize
TSKEY watermark; // used for stream
char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream
char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition
STag* pTag; // used for stream partition
} SDataBlockInfo;
typedef struct SSDataBlock {

1
include/common/tglobal.h
View File

@ -103,6 +103,7 @@ extern bool tsKeepColumnName;
// client
extern int32_t tsMinSlidingTime;
extern int32_t tsMinIntervalTime;
extern int32_t tsMaxMemUsedByInsert;
// build info
extern char version[];

4
include/libs/catalog/catalog.h
View File

@ -303,6 +303,10 @@ int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t re
int32_t catalogClearCache(void);
SMetaData* catalogCloneMetaData(SMetaData* pData);
void catalogFreeMetaData(SMetaData* pData);
/**
* Destroy catalog and relase all resources
*/

2
include/libs/function/function.h
View File

@ -54,7 +54,7 @@ typedef struct SFuncExecFuncs {
FExecCombine combine;
} SFuncExecFuncs;
#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results
#define MAX_INTERVAL_TIME_WINDOW 10000000 // maximum allowed time windows in final results
#define TOP_BOTTOM_QUERY_LIMIT 100
#define FUNCTIONS_NAME_MAX_LENGTH 16

1
include/libs/nodes/querynodes.h
View File

@ -385,7 +385,6 @@ typedef struct SCmdMsgInfo {
SEpSet epSet;
void* pMsg;
int32_t msgLen;
void* pExtension; // todo remove it soon
} SCmdMsgInfo;
typedef enum EQueryExecMode {

11
include/libs/parser/parser.h
View File

@ -33,6 +33,13 @@ typedef struct SStmtCallback {
int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**);
} SStmtCallback;
typedef struct SParseCsvCxt {
TdFilePtr fp; // last parsed file
int32_t tableNo; // last parsed table
SName tableName; // last parsed table
const char* pLastSqlPos; // the location of the last parsed sql
} SParseCsvCxt;
typedef struct SParseContext {
uint64_t requestId;
int64_t requestRid;
@ -57,6 +64,8 @@ typedef struct SParseContext {
SArray* pTableMetaPos; // sql table pos => catalog data pos
SArray* pTableVgroupPos; // sql table pos => catalog data pos
int64_t allocatorId;
bool needMultiParse;
SParseCsvCxt csvCxt;
} SParseContext;
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
@ -67,6 +76,8 @@ int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq
int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCatalogReq,
const struct SMetaData* pMetaData, SQuery* pQuery);
void qDestroyParseContext(SParseContext* pCxt);
void qDestroyQuery(SQuery* pQueryNode);
int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema);

3
include/os/os.h
View File

@ -51,6 +51,9 @@ extern "C" {
#endif
#else
#ifndef __func__
#define __func__ __FUNCTION__
#endif
#include <malloc.h>
#include <time.h>
#ifndef TD_USE_WINSOCK

4
include/util/taoserror.h
View File

@ -632,6 +632,10 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001)
#define TSDB_CODE_TMQ_CONSUMER_CLOSED TAOS_DEF_ERROR_CODE(0, 0x4002)
// TDLite
#define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x4100)
#define TSDB_CODE_TDLITE_IVLD_OPEN_DIR TAOS_DEF_ERROR_CODE(0, 0x4101)
#ifdef __cplusplus
}
#endif

4
packaging/tools/make_install.sh
View File

@ -317,10 +317,6 @@ function install_lib() {
${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so ||:
${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
if [ -d "${lib64_link_dir}" ]; then
${csudo}ln -sf ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
fi
fi
else
${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \

2
packaging/tools/makepkg.sh
View File

@ -47,7 +47,7 @@ if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then
cd ${top_dir}/tools/taos-tools/packaging/deb
[ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
taostools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}')
taostools_ver=$(git tag |grep -v taos | sort | tail -1)
taostools_install_dir="${release_dir}/${clientName}Tools-${taostools_ver}"
cd ${curr_dir}

25
source/client/inc/clientInt.h
View File

@ -380,16 +380,25 @@ void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey);
// --- mq
void hbMgrInitMqHbRspHandle();
typedef struct SSqlCallbackWrapper {
SParseContext* pParseCtx;
SCatalogReq* pCatalogReq;
SMetaData* pResultMeta;
SRequestObj* pRequest;
} SSqlCallbackWrapper;
SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res);
int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList);
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta);
int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList);
void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta);
int32_t removeMeta(STscObj* pTscObj, SArray* tbList);
int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);
int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog);
bool qnodeRequired(SRequestObj* pRequest);
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, SSqlCallbackWrapper* pWrapper);
int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList);
void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta);
int32_t removeMeta(STscObj* pTscObj, SArray* tbList);
int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);
int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog);
bool qnodeRequired(SRequestObj* pRequest);
int32_t continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest);
void destorySqlCallbackWrapper(SSqlCallbackWrapper* pWrapper);
#ifdef __cplusplus
}

132
source/client/src/clientImpl.c
View File

@ -868,10 +868,11 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
return code;
}
//todo refacto the error code mgmt
// todo refacto the error code mgmt
void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
SRequestObj* pRequest = (SRequestObj*)param;
STscObj* pTscObj = pRequest->pTscObj;
SSqlCallbackWrapper* pWrapper = param;
SRequestObj* pRequest = pWrapper->pRequest;
STscObj* pTscObj = pRequest->pTscObj;
pRequest->code = code;
if (pResult) {
@ -882,7 +883,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
int32_t type = pRequest->type;
if (TDMT_VND_SUBMIT == type || TDMT_VND_DELETE == type || TDMT_VND_CREATE_TABLE == type) {
if (pResult) {
pRequest->body.resInfo.numOfRows = pResult->numOfRows;
pRequest->body.resInfo.numOfRows += pResult->numOfRows;
// record the insert rows
if (TDMT_VND_SUBMIT == type) {
@ -899,12 +900,13 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
pRequest->requestId);
if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64,
pRequest->self, tstrerror(code), pRequest->retry, pRequest->requestId);
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self,
tstrerror(code), pRequest->retry, pRequest->requestId);
pRequest->prevCode = code;
schedulerFreeJob(&pRequest->body.queryJob, 0);
qDestroyQuery(pRequest->pQuery);
pRequest->pQuery = NULL;
destorySqlCallbackWrapper(pWrapper);
doAsyncQuery(pRequest, true);
return;
}
@ -920,6 +922,15 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
pRequest->code = code1;
}
if (pRequest->code == TSDB_CODE_SUCCESS && NULL != pWrapper->pParseCtx && pWrapper->pParseCtx->needMultiParse) {
code = continueInsertFromCsv(pWrapper, pRequest);
if (TSDB_CODE_SUCCESS == code) {
return;
}
}
destorySqlCallbackWrapper(pWrapper);
// return to client
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
}
@ -1020,24 +1031,13 @@ SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool vali
return launchQueryImpl(pRequest, pQuery, false, NULL);
}
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta) {
int32_t code = 0;
static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta,
SSqlCallbackWrapper* pWrapper) {
pRequest->type = pQuery->msgType;
pRequest->body.execMode = pQuery->execMode;
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
switch (pQuery->execMode) {
case QUERY_EXEC_MODE_LOCAL:
asyncExecLocalCmd(pRequest, pQuery);
return;
case QUERY_EXEC_MODE_RPC:
code = asyncExecDdlQuery(pRequest, pQuery);
break;
case QUERY_EXEC_MODE_SCHEDULE: {
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
pRequest->type = pQuery->msgType;
SPlanContext cxt = {.queryId = pRequest->requestId,
SPlanContext cxt = {.queryId = pRequest->requestId,
.acctId = pRequest->pTscObj->acctId,
.mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
.pAstRoot = pQuery->pRoot,
@ -1051,8 +1051,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
SAppInstInfo* pAppInfo = getAppInfo(pRequest);
SQueryPlan* pDag = NULL;
int64_t st = taosGetTimestampUs();
code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
int64_t st = taosGetTimestampUs();
int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
if (code) {
tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
@ -1061,42 +1061,64 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
}
pRequest->metric.planEnd = taosGetTimestampUs();
if (code == TSDB_CODE_SUCCESS) {
tscDebug("0x%" PRIx64 " create query plan success, elapsed time:%.2f ms, 0x%" PRIx64, pRequest->self,
(pRequest->metric.planEnd - st)/1000.0, pRequest->requestId);
}
if (code == TSDB_CODE_SUCCESS) {
tscDebug("0x%" PRIx64 " create query plan success, elapsed time:%.2f ms, 0x%" PRIx64, pRequest->self,
(pRequest->metric.planEnd - st)/1000.0, pRequest->requestId);
}
if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
SArray* pNodeList = NULL;
buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);
SRequestConnInfo conn = {
.pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
SSchedulerReq req = {
.syncReq = false,
.localReq = (tsQueryPolicy == QUERY_POLICY_CLIENT),
.pConn = &conn,
.pNodeList = pNodeList,
.pDag = pDag,
.allocatorRefId = pRequest->allocatorRefId,
.sql = pRequest->sqlstr,
.startTs = pRequest->metric.start,
.execFp = schedulerExecCb,
.cbParam = pRequest,
.chkKillFp = chkRequestKilled,
.chkKillParam = (void*)pRequest->self,
.pExecRes = NULL,
};
code = schedulerExecJob(&req, &pRequest->body.queryJob);
taosArrayDestroy(pNodeList);
} else {
tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
}
SRequestConnInfo conn = {.pTrans = getAppInfo(pRequest)->pTransporter,
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self};
SSchedulerReq req = {
.syncReq = false,
.localReq = (tsQueryPolicy == QUERY_POLICY_CLIENT),
.pConn = &conn,
.pNodeList = pNodeList,
.pDag = pDag,
.allocatorRefId = pRequest->allocatorRefId,
.sql = pRequest->sqlstr,
.startTs = pRequest->metric.start,
.execFp = schedulerExecCb,
.cbParam = pWrapper,
.chkKillFp = chkRequestKilled,
.chkKillParam = (void*)pRequest->self,
.pExecRes = NULL,
};
code = schedulerExecJob(&req, &pRequest->body.queryJob);
taosArrayDestroy(pNodeList);
} else {
tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
}
// todo not to be released here
taosArrayDestroy(pMnodeList);
// todo not to be released here
taosArrayDestroy(pMnodeList);
return code;
}
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, SSqlCallbackWrapper* pWrapper) {
int32_t code = 0;
pRequest->body.execMode = pQuery->execMode;
if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) {
destorySqlCallbackWrapper(pWrapper);
}
switch (pQuery->execMode) {
case QUERY_EXEC_MODE_LOCAL:
asyncExecLocalCmd(pRequest, pQuery);
break;
case QUERY_EXEC_MODE_RPC:
code = asyncExecDdlQuery(pRequest, pQuery);
break;
case QUERY_EXEC_MODE_SCHEDULE: {
code = asyncExecSchQuery(pRequest, pQuery, pResultMeta, pWrapper);
break;
}
case QUERY_EXEC_MODE_EMPTY_RESULT:

170
source/client/src/clientMain.c
View File

@ -667,43 +667,47 @@ const char *taos_get_server_info(TAOS *taos) {
return pTscObj->sDetailVer;
}
typedef struct SqlParseWrapper {
SParseContext *pCtx;
SCatalogReq catalogReq;
SRequestObj *pRequest;
} SqlParseWrapper;
static void destoryTablesReq(void *p) {
STablesReq *pRes = (STablesReq *)p;
taosArrayDestroy(pRes->pTables);
}
static void destorySqlParseWrapper(SqlParseWrapper *pWrapper) {
taosArrayDestroy(pWrapper->catalogReq.pDbVgroup);
taosArrayDestroy(pWrapper->catalogReq.pDbCfg);
taosArrayDestroy(pWrapper->catalogReq.pDbInfo);
taosArrayDestroyEx(pWrapper->catalogReq.pTableMeta, destoryTablesReq);
taosArrayDestroyEx(pWrapper->catalogReq.pTableHash, destoryTablesReq);
taosArrayDestroy(pWrapper->catalogReq.pUdf);
taosArrayDestroy(pWrapper->catalogReq.pIndex);
taosArrayDestroy(pWrapper->catalogReq.pUser);
taosArrayDestroy(pWrapper->catalogReq.pTableIndex);
taosArrayDestroy(pWrapper->pCtx->pTableMetaPos);
taosArrayDestroy(pWrapper->pCtx->pTableVgroupPos);
taosMemoryFree(pWrapper->pCtx);
static void destoryCatalogReq(SCatalogReq *pCatalogReq) {
if (NULL == pCatalogReq) {
return;
}
taosArrayDestroy(pCatalogReq->pDbVgroup);
taosArrayDestroy(pCatalogReq->pDbCfg);
taosArrayDestroy(pCatalogReq->pDbInfo);
taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq);
taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq);
taosArrayDestroy(pCatalogReq->pUdf);
taosArrayDestroy(pCatalogReq->pIndex);
taosArrayDestroy(pCatalogReq->pUser);
taosArrayDestroy(pCatalogReq->pTableIndex);
taosMemoryFree(pCatalogReq);
}
void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) {
if (NULL == pWrapper) {
return;
}
destoryCatalogReq(pWrapper->pCatalogReq);
qDestroyParseContext(pWrapper->pParseCtx);
catalogFreeMetaData(pWrapper->pResultMeta);
taosMemoryFree(pWrapper);
}
void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
SqlParseWrapper *pWrapper = (SqlParseWrapper *)param;
SRequestObj *pRequest = pWrapper->pRequest;
SQuery *pQuery = pRequest->pQuery;
SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param;
SRequestObj *pRequest = pWrapper->pRequest;
SQuery *pQuery = pRequest->pQuery;
pRequest->metric.ctgEnd = taosGetTimestampUs();
qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId);
if (code == TSDB_CODE_SUCCESS) {
code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery);
code = qAnalyseSqlSemantic(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery);
pRequest->stableQuery = pQuery->stableQuery;
if (pQuery->pRoot) {
pRequest->stmtType = pQuery->pRoot->type;
@ -712,6 +716,13 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
pRequest->metric.semanticEnd = taosGetTimestampUs();
if (code == TSDB_CODE_SUCCESS && pWrapper->pParseCtx->needMultiParse) {
pWrapper->pResultMeta = catalogCloneMetaData(pResultMeta);
if (NULL == pWrapper->pResultMeta) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (code == TSDB_CODE_SUCCESS) {
if (pQuery->haveResultSet) {
setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols);
@ -722,15 +733,13 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
TSWAP(pRequest->tableList, (pQuery)->pTableList);
TSWAP(pRequest->targetTableList, (pQuery)->pTargetTableList);
destorySqlParseWrapper(pWrapper);
double el = (pRequest->metric.semanticEnd - pRequest->metric.ctgEnd)/1000.0;
double el = (pRequest->metric.semanticEnd - pRequest->metric.ctgEnd) / 1000.0;
tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, elapsed time:%.2f ms, reqId:0x%" PRIx64,
pRequest->self, el, pRequest->requestId);
launchAsyncQuery(pRequest, pQuery, pResultMeta);
launchAsyncQuery(pRequest, pQuery, pResultMeta, pWrapper);
} else {
destorySqlParseWrapper(pWrapper);
destorySqlCallbackWrapper(pWrapper);
qDestroyQuery(pRequest->pQuery);
pRequest->pQuery = NULL;
@ -750,6 +759,16 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
}
}
int32_t continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) {
qDestroyQuery(pRequest->pQuery);
pRequest->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == pRequest->pQuery) {
return TSDB_CODE_OUT_OF_MEMORY;
}
retrieveMetaCallback(pWrapper->pResultMeta, pWrapper, TSDB_CODE_SUCCESS);
return TSDB_CODE_SUCCESS;
}
void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
int64_t connId = *(int64_t *)taos;
taosAsyncQueryImpl(connId, sql, fp, param, false);
@ -786,37 +805,48 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
}
void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
SParseContext *pCxt = NULL;
STscObj *pTscObj = pRequest->pTscObj;
int32_t code = 0;
STscObj *pTscObj = pRequest->pTscObj;
SSqlCallbackWrapper *pWrapper = NULL;
int32_t code = TSDB_CODE_SUCCESS;
if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) {
code = pRequest->prevCode;
goto _error;
}
code = createParseContext(pRequest, &pCxt);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
if (TSDB_CODE_SUCCESS == code) {
pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper));
if (pWrapper == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
pWrapper->pRequest = pRequest;
}
}
pCxt->mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCxt->pCatalog);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
if (TSDB_CODE_SUCCESS == code) {
code = createParseContext(pRequest, &pWrapper->pParseCtx);
}
pRequest->metric.syntaxStart = taosGetTimestampUs();
SCatalogReq catalogReq = {.forceUpdate = updateMetaForce, .qNodeRequired = qnodeRequired(pRequest)};
code = qParseSqlSyntax(pCxt, &pRequest->pQuery, &catalogReq);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
if (TSDB_CODE_SUCCESS == code) {
pWrapper->pParseCtx->mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pWrapper->pParseCtx->pCatalog);
}
pRequest->metric.syntaxEnd = taosGetTimestampUs();
if (TSDB_CODE_SUCCESS == code) {
pRequest->metric.syntaxStart = taosGetTimestampUs();
if (!updateMetaForce) {
pWrapper->pCatalogReq = taosMemoryCalloc(1, sizeof(SCatalogReq));
if (pWrapper->pCatalogReq == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
pWrapper->pCatalogReq->forceUpdate = updateMetaForce;
pWrapper->pCatalogReq->qNodeRequired = qnodeRequired(pRequest);
code = qParseSqlSyntax(pWrapper->pParseCtx, &pRequest->pQuery, pWrapper->pCatalogReq);
}
pRequest->metric.syntaxEnd = taosGetTimestampUs();
}
if (TSDB_CODE_SUCCESS == code && !updateMetaForce) {
SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary;
if (NULL == pRequest->pQuery->pRoot) {
atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);
@ -825,38 +855,26 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
}
}
SqlParseWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SqlParseWrapper));
if (pWrapper == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
if (TSDB_CODE_SUCCESS == code) {
SRequestConnInfo conn = {.pTrans = pWrapper->pParseCtx->pTransporter,
.requestId = pWrapper->pParseCtx->requestId,
.requestObjRefId = pWrapper->pParseCtx->requestRid,
.mgmtEps = pWrapper->pParseCtx->mgmtEpSet};
pRequest->metric.ctgStart = taosGetTimestampUs();
code = catalogAsyncGetAllMeta(pWrapper->pParseCtx->pCatalog, &conn, pWrapper->pCatalogReq, retrieveMetaCallback,
pWrapper, &pRequest->body.queryJob);
}
pWrapper->pCtx = pCxt;
pWrapper->pRequest = pRequest;
pWrapper->catalogReq = catalogReq;
SRequestConnInfo conn = {.pTrans = pCxt->pTransporter,
.requestId = pCxt->requestId,
.requestObjRefId = pCxt->requestRid,
.mgmtEps = pCxt->mgmtEpSet};
pRequest->metric.ctgStart = taosGetTimestampUs();
code = catalogAsyncGetAllMeta(pCxt->pCatalog, &conn, &catalogReq, retrieveMetaCallback, pWrapper,
&pRequest->body.queryJob);
pCxt = NULL;
if (code == TSDB_CODE_SUCCESS) {
return;
if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code),
pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
terrno = code;
pRequest->code = code;
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
}
_error:
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code),
pRequest->requestId);
taosMemoryFree(pCxt);
terrno = code;
pRequest->code = code;
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
}
static void fetchCallback(void *pResult, void *param, int32_t code) {

245
source/client/src/clientSml.c
View File

@ -209,7 +209,7 @@ static inline bool smlCheckDuplicateKey(const char *key, int32_t keyLen, SHashOb
}
static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) {
if(pBuf->buf){
if (pBuf->buf) {
memset(pBuf->buf, 0, pBuf->len);
if (msg1) strncat(pBuf->buf, msg1, pBuf->len);
int32_t left = pBuf->len - strlen(pBuf->buf);
@ -256,15 +256,15 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) {
while (result <= length) {
result *= 2;
}
if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){
if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE;
} else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
} else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
}
if (type == TSDB_DATA_TYPE_NCHAR){
if (type == TSDB_DATA_TYPE_NCHAR) {
result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
}else if (type == TSDB_DATA_TYPE_BINARY){
} else if (type == TSDB_DATA_TYPE_BINARY) {
result = result + VARSTR_HEADER_SIZE;
}
return result;
@ -274,7 +274,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
ESchemaAction *action, bool isTag) {
int32_t code = TSDB_CODE_SUCCESS;
for (int j = 0; j < taosArrayGetSize(cols); ++j) {
if(j == 0 && !isTag) continue;
if (j == 0 && !isTag) continue;
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j);
code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info);
if (code != TSDB_CODE_SUCCESS) {
@ -286,12 +286,12 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) {
SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
int32_t i = 0;
for ( ;i < length; i++) {
int32_t i = 0;
for (; i < length; i++) {
taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES);
}
if (isTag){
if (isTag) {
i = 0;
} else {
i = 1;
@ -306,7 +306,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool
return 0;
}
static int32_t getBytes(uint8_t type, int32_t length){
static int32_t getBytes(uint8_t type, int32_t length) {
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
return smlFindNearestPowerOf2(length, type);
} else {
@ -314,21 +314,22 @@ static int32_t getBytes(uint8_t type, int32_t length){
}
}
static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray* results, int32_t numOfCols, bool isTag) {
static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols,
SArray *results, int32_t numOfCols, bool isTag) {
for (int j = 0; j < taosArrayGetSize(cols); ++j) {
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j);
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j);
ESchemaAction action = SCHEMA_ACTION_NULL;
smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info);
if(action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG){
if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) {
SField field = {0};
field.type = kv->type;
field.bytes = getBytes(kv->type, kv->length);
memcpy(field.name, kv->key, kv->keyLen);
taosArrayPush(results, &field);
}else if(action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE){
} else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) {
uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen);
uint16_t newIndex = *index;
if(isTag) newIndex -= numOfCols;
uint16_t newIndex = *index;
if (isTag) newIndex -= numOfCols;
SField *field = (SField *)taosArrayGet(results, newIndex);
field->bytes = getBytes(kv->type, kv->length);
}
@ -336,12 +337,11 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
return TSDB_CODE_SUCCESS;
}
//static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData,
// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){
static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns, SArray* pTags,
STableMeta *pTableMeta, ESchemaAction action){
SRequestObj* pRequest = NULL;
// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData,
// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){
static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta,
ESchemaAction action) {
SRequestObj *pRequest = NULL;
SMCreateStbReq pReq = {0};
int32_t code = TSDB_CODE_SUCCESS;
SCmdMsgInfo pCmdMsg = {0};
@ -363,24 +363,24 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns,
goto end;
}
if (action == SCHEMA_ACTION_CREATE_STABLE){
if (action == SCHEMA_ACTION_CREATE_STABLE) {
pReq.colVer = 1;
pReq.tagVer = 1;
pReq.suid = 0;
pReq.source = TD_REQ_FROM_APP;
} else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE){
} else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) {
pReq.colVer = pTableMeta->sversion;
pReq.tagVer = pTableMeta->tversion + 1;
pReq.suid = pTableMeta->uid;
pReq.source = TD_REQ_FROM_TAOX;
} else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE){
} else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) {
pReq.colVer = pTableMeta->sversion + 1;
pReq.tagVer = pTableMeta->tversion;
pReq.suid = pTableMeta->uid;
pReq.source = TD_REQ_FROM_TAOX;
}
if (pReq.numOfTags == 0){
if (pReq.numOfTags == 0) {
pReq.numOfTags = 1;
SField field = {0};
field.type = TSDB_DATA_TYPE_NCHAR;
@ -412,7 +412,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns,
launchQueryImpl(pRequest, &pQuery, true, NULL);
if(pRequest->code == TSDB_CODE_SUCCESS){
if (pRequest->code == TSDB_CODE_SUCCESS) {
catalogRemoveTableMeta(info->pCatalog, pName);
}
code = pRequest->code;
@ -425,11 +425,11 @@ end:
}
static int32_t smlModifyDBSchemas(SSmlHandle *info) {
int32_t code = 0;
SHashObj *hashTmp = NULL;
STableMeta *pTableMeta = NULL;
int32_t code = 0;
SHashObj *hashTmp = NULL;
STableMeta *pTableMeta = NULL;
SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
strcpy(pName.dbname, info->pRequest->pDb);
SRequestConnInfo conn = {0};
@ -451,8 +451,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) {
SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField));
SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField));
SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField));
SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField));
smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true);
smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false);
@ -463,8 +463,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
}
info->cost.numOfCreateSTables++;
} else if (code == TSDB_CODE_SUCCESS) {
hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags,
taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true,
HASH_NO_LOCK);
for (uint16_t i = pTableMeta->tableInfo.numOfColumns;
i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES);
@ -475,22 +475,25 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
if (action != SCHEMA_ACTION_NULL){
SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
if (action != SCHEMA_ACTION_NULL) {
SArray *pColumns =
taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
SArray *pTags =
taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
SField field = {0};
field.type = pTableMeta->schema[i].type;
field.bytes = pTableMeta->schema[i].bytes;
strcpy(field.name, pTableMeta->schema[i].name);
if(i < pTableMeta->tableInfo.numOfColumns){
if (i < pTableMeta->tableInfo.numOfColumns) {
taosArrayPush(pColumns, &field);
}else{
} else {
taosArrayPush(pTags, &field);
}
}
smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, pTableMeta->tableInfo.numOfColumns, true);
smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags,
pTableMeta->tableInfo.numOfColumns, true);
code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
if (code != TSDB_CODE_SUCCESS) {
@ -518,23 +521,26 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
if (action != SCHEMA_ACTION_NULL){
SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
if (action != SCHEMA_ACTION_NULL) {
SArray *pColumns =
taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
SArray *pTags =
taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
SField field = {0};
field.type = pTableMeta->schema[i].type;
field.bytes = pTableMeta->schema[i].bytes;
strcpy(field.name, pTableMeta->schema[i].name);
if(i < pTableMeta->tableInfo.numOfColumns){
if (i < pTableMeta->tableInfo.numOfColumns) {
taosArrayPush(pColumns, &field);
}else{
} else {
taosArrayPush(pTags, &field);
}
}
smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, pTableMeta->tableInfo.numOfColumns, false);
smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns,
pTableMeta->tableInfo.numOfColumns, false);
code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
if (code != TSDB_CODE_SUCCESS) {
@ -847,7 +853,7 @@ static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t
static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) {
int64_t ts = 0;
if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
// uError("SML:data:%s,len:%d", data, len);
// uError("SML:data:%s,len:%d", data, len);
ts = smlParseInfluxTime(info, data, len);
} else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
ts = smlParseOpenTsdbTime(info, data, len);
@ -877,7 +883,7 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
if (smlIsBinary(pVal->value, pVal->length)) {
pVal->type = TSDB_DATA_TYPE_BINARY;
pVal->length -= BINARY_ADD_LEN;
if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){
if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
pVal->value += (BINARY_ADD_LEN - 1);
@ -887,7 +893,7 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
if (smlIsNchar(pVal->value, pVal->length)) {
pVal->type = TSDB_DATA_TYPE_NCHAR;
pVal->length -= NCHAR_ADD_LEN;
if(pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
pVal->value += (NCHAR_ADD_LEN - 1);
@ -1063,7 +1069,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab
continue;
}
if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
@ -1224,7 +1230,7 @@ static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *c
kv->value = value;
kv->length = valueLen;
if (isTag) {
if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
kv->type = TSDB_DATA_TYPE_NCHAR;
@ -1362,8 +1368,8 @@ static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) {
static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) {
SHashObj *s1 = *(SHashObj **)key1;
SHashObj *s2 = *(SHashObj **)key2;
SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN);
SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN);
SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN);
SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN);
ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP);
ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP);
if (kv1->i < kv2->i) {
@ -1375,12 +1381,12 @@ static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) {
}
}
static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){
if(dataFormat){
static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *cols) {
if (dataFormat) {
void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT);
if(p == NULL){
if (p == NULL) {
taosArrayPush(oneTable->cols, &cols);
}else{
} else {
taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols);
}
return TSDB_CODE_SUCCESS;
@ -1397,9 +1403,9 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col
}
void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT);
if(p == NULL){
if (p == NULL) {
taosArrayPush(oneTable->cols, &kvHash);
}else{
} else {
taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash);
}
return TSDB_CODE_SUCCESS;
@ -1488,15 +1494,15 @@ static void smlDestroyInfo(SSmlHandle *info) {
taosMemoryFreeClear(info);
}
static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLProtocolType protocol, int8_t precision){
int32_t code = TSDB_CODE_SUCCESS;
SSmlHandle* info = (SSmlHandle*)taosMemoryCalloc(1, sizeof(SSmlHandle));
static SSmlHandle *smlBuildSmlInfo(STscObj *pTscObj, SRequestObj *request, SMLProtocolType protocol, int8_t precision) {
int32_t code = TSDB_CODE_SUCCESS;
SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle));
if (NULL == info) {
return NULL;
}
info->id = smlGenId();
info->pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
info->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == info->pQuery) {
uError("SML:0x%" PRIx64 " create info->pQuery error", info->id);
goto cleanup;
@ -1511,8 +1517,8 @@ static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLPr
}
((SVnodeModifOpStmt *)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV;
if (pTscObj){
info->taos = pTscObj;
if (pTscObj) {
info->taos = pTscObj;
code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog);
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code);
@ -1528,7 +1534,7 @@ static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLPr
info->dataFormat = true;
}
if(request){
if (request) {
info->pRequest = request;
info->msgBuf.buf = info->pRequest->msgBuf;
info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE;
@ -1827,10 +1833,11 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) {
}
pVal->length = (int16_t)strlen(value->valuestring);
if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){
if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
if (pVal->type == TSDB_DATA_TYPE_NCHAR && pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
if (pVal->type == TSDB_DATA_TYPE_NCHAR &&
pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
@ -2058,7 +2065,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) {
SSmlLineInfo elements = {0};
uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql);
int ret = smlParseInfluxString(sql, &elements, &info->msgBuf);
int ret = smlParseInfluxString(sql, &elements, &info->msgBuf);
if (ret != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id);
return ret;
@ -2314,7 +2321,8 @@ static int32_t smlInsertData(SSmlHandle *info) {
(*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid
code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat,
(*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->msgBuf.buf, info->msgBuf.len);
(*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen,
info->msgBuf.buf, info->msgBuf.len);
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
return code;
@ -2336,7 +2344,12 @@ static int32_t smlInsertData(SSmlHandle *info) {
SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary;
atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);
launchAsyncQuery(info->pRequest, info->pQuery, NULL);
SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper));
if (pWrapper == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pWrapper->pRequest = info->pRequest;
launchAsyncQuery(info->pRequest, info->pQuery, NULL, pWrapper);
return TSDB_CODE_SUCCESS;
}
@ -2417,41 +2430,41 @@ static int smlProcess(SSmlHandle *info, char *lines[], int numLines) {
}
static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) {
// SCatalog *catalog = NULL;
// int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog);
// if (code != TSDB_CODE_SUCCESS) {
// uError("SML get catalog error %d", code);
// return code;
// }
//
// SName name;
// tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db));
// char dbFname[TSDB_DB_FNAME_LEN] = {0};
// tNameGetFullDbName(&name, dbFname);
// SDbCfgInfo pInfo = {0};
//
// SRequestConnInfo conn = {0};
// conn.pTrans = taos->pAppInfo->pTransporter;
// conn.requestId = request->requestId;
// conn.requestObjRefId = request->self;
// conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp);
//
// code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo);
// if (code != TSDB_CODE_SUCCESS) {
// return code;
// }
// taosArrayDestroy(pInfo.pRetensions);
//
// if (!pInfo.schemaless) {
// return TSDB_CODE_SML_INVALID_DB_CONF;
// }
// SCatalog *catalog = NULL;
// int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog);
// if (code != TSDB_CODE_SUCCESS) {
// uError("SML get catalog error %d", code);
// return code;
// }
//
// SName name;
// tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db));
// char dbFname[TSDB_DB_FNAME_LEN] = {0};
// tNameGetFullDbName(&name, dbFname);
// SDbCfgInfo pInfo = {0};
//
// SRequestConnInfo conn = {0};
// conn.pTrans = taos->pAppInfo->pTransporter;
// conn.requestId = request->requestId;
// conn.requestObjRefId = request->self;
// conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp);
//
// code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo);
// if (code != TSDB_CODE_SUCCESS) {
// return code;
// }
// taosArrayDestroy(pInfo.pRetensions);
//
// if (!pInfo.schemaless) {
// return TSDB_CODE_SML_INVALID_DB_CONF;
// }
return TSDB_CODE_SUCCESS;
}
static void smlInsertCallback(void *param, void *res, int32_t code) {
SRequestObj *pRequest = (SRequestObj *)res;
SSmlHandle *info = (SSmlHandle *)param;
int32_t rows = taos_affected_rows(pRequest);
int32_t rows = taos_affected_rows(pRequest);
uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf);
Params *pParam = info->params;
@ -2461,7 +2474,7 @@ static void smlInsertCallback(void *param, void *res, int32_t code) {
if (code != TSDB_CODE_SUCCESS) {
pParam->request->code = code;
pParam->request->body.resInfo.numOfRows += rows;
}else{
} else {
pParam->request->body.resInfo.numOfRows += info->affectedRows;
}
if (pParam->cnt == pParam->total) {
@ -2497,20 +2510,20 @@ static void smlInsertCallback(void *param, void *res, int32_t code) {
*
*/
TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) {
TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
if (NULL == taos) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL;
}
SRequestObj* request = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT);
if(!request){
SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT);
if (!request) {
uError("SML:taos_schemaless_insert error request is null");
return NULL;
}
int batchs = 0;
STscObj* pTscObj = request->pTscObj;
int batchs = 0;
STscObj *pTscObj = request->pTscObj;
pTscObj->schemalessType = 1;
SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf};
@ -2526,7 +2539,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
goto end;
}
if(isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS){
if (isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS) {
request->code = TSDB_CODE_SML_INVALID_DB_CONF;
smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL);
goto end;
@ -2551,9 +2564,9 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
goto end;
}
if(protocol == TSDB_SML_JSON_PROTOCOL){
if (protocol == TSDB_SML_JSON_PROTOCOL) {
numLines = 1;
}else if(numLines <= 0){
} else if (numLines <= 0) {
request->code = TSDB_CODE_SML_INVALID_DATA;
smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL);
goto end;
@ -2562,14 +2575,14 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
batchs = ceil(((double)numLines) / LINE_BATCH);
params.total = batchs;
for (int i = 0; i < batchs; ++i) {
SRequestObj* req = (SRequestObj*)createRequest(pTscObj->id, TSDB_SQL_INSERT);
if(!req){
SRequestObj *req = (SRequestObj *)createRequest(pTscObj->id, TSDB_SQL_INSERT);
if (!req) {
request->code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML:taos_schemaless_insert error request is null");
goto end;
}
SSmlHandle* info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision);
if(!info){
SSmlHandle *info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision);
if (!info) {
request->code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML:taos_schemaless_insert error SSmlHandle is null");
goto end;
@ -2599,8 +2612,8 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
end:
taosThreadSpinDestroy(&params.lock);
tsem_destroy(&params.sem);
// ((STscObj *)taos)->schemalessType = 0;
// ((STscObj *)taos)->schemalessType = 0;
pTscObj->schemalessType = 1;
uDebug("resultend:%s", request->msgBuf);
return (TAOS_RES*)request;
return (TAOS_RES *)request;
}

5
source/common/src/tdatablock.c
View File

@ -1235,6 +1235,7 @@ void blockDataFreeRes(SSDataBlock* pBlock) {
taosArrayDestroy(pBlock->pDataBlock);
pBlock->pDataBlock = NULL;
taosMemoryFreeClear(pBlock->pBlockAgg);
taosMemoryFree(pBlock->info.pTag);
memset(&pBlock->info, 0, sizeof(SDataBlockInfo));
}
@ -1317,7 +1318,7 @@ SSDataBlock* createSpecialDataBlock(EStreamType type) {
pBlock->info.rows = 0;
pBlock->info.type = type;
pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(TSKEY) +
sizeof(TSKEY) + TSDB_TABLE_NAME_LEN;
sizeof(TSKEY) + VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN;
pBlock->info.watermark = INT64_MIN;
pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData));
@ -1345,7 +1346,7 @@ SSDataBlock* createSpecialDataBlock(EStreamType type) {
// table name
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = TSDB_TABLE_NAME_LEN;
infoData.info.bytes = VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
return pBlock;

8
source/common/src/tdataformat.c
View File

@ -411,7 +411,7 @@ _exit:
int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow) {
int32_t code = 0;
int32_t rLen;
int32_t rLen = 0;
TSROW_LEN(pRow, rLen);
(*ppRow) = (STSRow2 *)taosMemoryMalloc(rLen);
@ -1678,8 +1678,8 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
int32_t size;
ASSERT(pColDataSrc->nVal > 0);
ASSERT(pColDataDest->cid = pColDataSrc->cid);
ASSERT(pColDataDest->type = pColDataSrc->type);
ASSERT(pColDataDest->cid == pColDataSrc->cid);
ASSERT(pColDataDest->type == pColDataSrc->type);
pColDataDest->smaOn = pColDataSrc->smaOn;
pColDataDest->nVal = pColDataSrc->nVal;
@ -1711,4 +1711,4 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
_exit:
return code;
}
}

43
source/common/src/tglobal.c
View File

@ -85,7 +85,8 @@ uint16_t tsTelemPort = 80;
char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null";
char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value.
// If set to empty system will generate table name using MD5 hash.
bool tsSmlDataFormat = false; // true means that the name and order of cols in each line are the same(only for influx protocol)
// true means that the name and order of cols in each line are the same(only for influx protocol)
bool tsSmlDataFormat = false;
// query
int32_t tsQueryPolicy = 1;
@ -125,6 +126,9 @@ int32_t tsMaxNumOfDistinctResults = 1000 * 10000;
// 1 database precision unit for interval time range, changed accordingly
int32_t tsMinIntervalTime = 1;
// maximum memory allowed to be allocated for a single csv load (in MB)
int32_t tsMaxMemUsedByInsert = 1024;
// the maximum allowed query buffer size during query processing for each data node.
// -1 no limit (default)
// 0 no query allowed, queries are disabled
@ -296,6 +300,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1;
tsNumOfTaskQueueThreads = tsNumOfCores / 2;
tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4);
@ -374,8 +379,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4);
if (cfgAddInt32(pCfg, "numOfVnodeStreamThreads", tsNumOfVnodeStreamThreads, 4, 1024, 0) != 0) return -1;
// tsNumOfVnodeFetchThreads = 1;
// if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1, 0) != 0) return -1;
// tsNumOfVnodeFetchThreads = 1;
// if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1, 0) != 0) return -1;
tsNumOfVnodeWriteThreads = tsNumOfCores;
tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1);
@ -497,15 +502,15 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
pItem->stype = stype;
}
/*
pItem = cfgGetItem(tsCfg, "numOfVnodeFetchThreads");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsNumOfVnodeFetchThreads = numOfCores / 4;
tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
pItem->i32 = tsNumOfVnodeFetchThreads;
pItem->stype = stype;
}
*/
/*
pItem = cfgGetItem(tsCfg, "numOfVnodeFetchThreads");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsNumOfVnodeFetchThreads = numOfCores / 4;
tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
pItem->i32 = tsNumOfVnodeFetchThreads;
pItem->stype = stype;
}
*/
pItem = cfgGetItem(tsCfg, "numOfVnodeWriteThreads");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
@ -648,6 +653,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32;
tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32;
tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32;
@ -705,7 +712,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32;
tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
tsNumOfVnodeStreamThreads = cfgGetItem(pCfg, "numOfVnodeStreamThreads")->i32;
// tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
// tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
@ -877,6 +884,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32;
} else if (strcasecmp("maxNumOfDistinctRes", name) == 0) {
tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32;
} else if (strcasecmp("maxMemUsedByInsert", name) == 0) {
tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32;
}
break;
}
@ -955,10 +964,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32;
} else if (strcasecmp("numOfVnodeQueryThreads", name) == 0) {
tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
/*
} else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) {
tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
*/
/*
} else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) {
tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
*/
} else if (strcasecmp("numOfVnodeWriteThreads", name) == 0) {
tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
} else if (strcasecmp("numOfVnodeSyncThreads", name) == 0) {

9
source/common/src/ttszip.c
View File

@ -35,7 +35,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
// tscError("tmp file created failed since %s", terrstr());
return NULL;
}
STSBuf* pTSBuf = taosMemoryCalloc(1, sizeof(STSBuf));
if (pTSBuf == NULL) {
return NULL;
@ -52,10 +52,13 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
}
if (!autoDelete) {
taosRemoveFile(pTSBuf->path);
if (taosRemoveFile(pTSBuf->path) != 0) {
taosMemoryFree(pTSBuf);
return NULL;
}
}
if (NULL == allocResForTSBuf(pTSBuf)) {
if (allocResForTSBuf(pTSBuf) == NULL) {
return NULL;
}

3
source/dnode/mgmt/CMakeLists.txt
View File

@ -14,7 +14,4 @@ target_include_directories(
taosd
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc"
)
IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
add_dependencies(taosd jemalloc)
ENDIF ()
target_link_libraries(taosd dnode)

13
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -201,14 +201,13 @@ _OVER:
taosCloseFile(&pFile);
taosMemoryFree(content);
for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = ppVnodes[i];
if (pVnode != NULL) {
vmReleaseVnode(pMgmt, pVnode);
}
}
if (ppVnodes != NULL) {
for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = ppVnodes[i];
if (pVnode != NULL) {
vmReleaseVnode(pMgmt, pVnode);
}
}
taosMemoryFree(ppVnodes);
}

72
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE
#include "vmInt.h"
void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) {
pInfo->pVloads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad));
if (pInfo->pVloads == NULL) return;
@ -30,6 +30,7 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
SVnodeObj *pVnode = *ppVnode;
SVnodeLoad vload = {0};
vnodeGetLoad(pVnode->pImpl, &vload);
if (isReset) vnodeResetLoad(pVnode->pImpl, &vload);
taosArrayPush(pInfo->pVloads, &vload);
pIter = taosHashIterate(pMgmt->hash, pIter);
}
@ -39,7 +40,7 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) {
SMonVloadInfo vloads = {0};
vmGetVnodeLoads(pMgmt, &vloads);
vmGetVnodeLoads(pMgmt, &vloads, true);
SArray *pVloads = vloads.pVloads;
if (pVloads == NULL) return;
@ -66,10 +67,10 @@ void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) {
pInfo->vstat.totalVnodes = totalVnodes;
pInfo->vstat.masterNum = masterNum;
pInfo->vstat.numOfSelectReqs = numOfSelectReqs - pMgmt->state.numOfSelectReqs;
pInfo->vstat.numOfInsertReqs = numOfInsertReqs - pMgmt->state.numOfInsertReqs;
pInfo->vstat.numOfInsertSuccessReqs = numOfInsertSuccessReqs - pMgmt->state.numOfInsertSuccessReqs;
pInfo->vstat.numOfBatchInsertReqs = numOfBatchInsertReqs - pMgmt->state.numOfBatchInsertReqs;
pInfo->vstat.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs - pMgmt->state.numOfBatchInsertSuccessReqs;
pInfo->vstat.numOfInsertReqs = numOfInsertReqs; // delta
pInfo->vstat.numOfInsertSuccessReqs = numOfInsertSuccessReqs; // delta
pInfo->vstat.numOfBatchInsertReqs = numOfBatchInsertReqs; // delta
pInfo->vstat.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs; // delta
pMgmt->state.totalVnodes = totalVnodes;
pMgmt->state.masterNum = masterNum;
pMgmt->state.numOfSelectReqs = numOfSelectReqs;
@ -109,7 +110,7 @@ int32_t vmProcessGetMonitorInfoReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t vmProcessGetLoadsReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SMonVloadInfo vloads = {0};
vmGetVnodeLoads(pMgmt, &vloads);
vmGetVnodeLoads(pMgmt, &vloads, false);
int32_t rspLen = tSerializeSMonVloadInfo(NULL, 0, &vloads);
if (rspLen < 0) {
@ -212,38 +213,47 @@ static int32_t vmTsmaProcessCreate(SVnode *pVnode, SCreateVnodeReq *pReq) {
}
int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SCreateVnodeReq createReq = {0};
SCreateVnodeReq req = {0};
SVnodeCfg vnodeCfg = {0};
SWrapperCfg wrapperCfg = {0};
int32_t code = -1;
char path[TSDB_FILENAME_LEN] = {0};
if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
dInfo(
"vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d "
"tsdbPageSize:%d",
createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize,
createReq.sstTrigger, createReq.tsdbPageSize);
dInfo("vgId:%d, hashMethod:%d begin:%u end:%u prefix:%d surfix:%d", createReq.vgId, createReq.hashMethod,
createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix);
vmGenerateVnodeCfg(&createReq, &vnodeCfg);
"vgId:%d, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64
" cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64
"days:%d keep0:%d keep1:%d keep2:%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d, wal "
"fsync:%d level:%d retentionPeriod:%d retentionSize:%d rollPeriod:%d segSize:%d, hash method:%d begin:%u end:%u "
"prefix:%d surfix:%d replica:%d selfIndex:%d strict:%d",
req.vgId, req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024,
req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize, req.tsdbPageSize * 1024, req.db, req.dbUid,
req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, req.isTsma, req.precision, req.compression,
req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix,
req.replica, req.selfIndex, req.strict);
for (int32_t i = 0; i < req.replica; ++i) {
dInfo("vgId:%d, replica:%d fqdn:%s port:%u", req.vgId, req.replicas[i].id, req.replicas[i].fqdn,
req.replicas[i].port);
}
vmGenerateVnodeCfg(&req, &vnodeCfg);
if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) {
dError("vgId:%d, failed to adjust tsma days since %s", createReq.vgId, terrstr());
if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) {
dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, terrstr());
code = terrno;
goto _OVER;
}
vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg);
vmGenerateWrapperCfg(pMgmt, &req, &wrapperCfg);
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId);
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
if (pVnode != NULL) {
dDebug("vgId:%d, already exist", createReq.vgId);
tFreeSCreateVnodeReq(&createReq);
dDebug("vgId:%d, already exist", req.vgId);
tFreeSCreateVnodeReq(&req);
vmReleaseVnode(pMgmt, pVnode);
terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED;
code = terrno;
@ -252,36 +262,36 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId);
if (vnodeCreate(path, &vnodeCfg, pMgmt->pTfs) < 0) {
tFreeSCreateVnodeReq(&createReq);
dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
tFreeSCreateVnodeReq(&req);
dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr());
code = terrno;
goto _OVER;
}
SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb);
if (pImpl == NULL) {
dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr());
dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr());
code = terrno;
goto _OVER;
}
code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl);
if (code != 0) {
dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr());
dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr());
code = terrno;
goto _OVER;
}
code = vmTsmaProcessCreate(pImpl, &createReq);
code = vmTsmaProcessCreate(pImpl, &req);
if (code != 0) {
dError("vgId:%d, failed to create tsma since %s", createReq.vgId, terrstr());
dError("vgId:%d, failed to create tsma since %s", req.vgId, terrstr());
code = terrno;
goto _OVER;
}
code = vnodeStart(pImpl);
if (code != 0) {
dError("vgId:%d, failed to start sync since %s", createReq.vgId, terrstr());
dError("vgId:%d, failed to start sync since %s", req.vgId, terrstr());
goto _OVER;
}
@ -296,10 +306,10 @@ _OVER:
vnodeClose(pImpl);
vnodeDestroy(path, pMgmt->pTfs);
} else {
dInfo("vgId:%d, vnode is created", createReq.vgId);
dInfo("vgId:%d, vnode is created", req.vgId);
}
tFreeSCreateVnodeReq(&createReq);
tFreeSCreateVnodeReq(&req);
terrno = code;
return code;
}

2
source/dnode/mgmt/node_mgmt/inc/dmNodes.h
View File

@ -35,7 +35,7 @@ void qmGetMonitorInfo(void *pMgmt, SMonQmInfo *pInfo);
void smGetMonitorInfo(void *pMgmt, SMonSmInfo *pInfo);
void bmGetMonitorInfo(void *pMgmt, SMonBmInfo *pInfo);
void vmGetVnodeLoads(void *pMgmt, SMonVloadInfo *pInfo);
void vmGetVnodeLoads(void *pMgmt, SMonVloadInfo *pInfo, bool isReset);
void mmGetMnodeLoads(void *pMgmt, SMonMloadInfo *pInfo);
void qmGetQnodeLoads(void *pMgmt, SQnodeLoad *pInfo);

2
source/dnode/mgmt/node_mgmt/src/dmMonitor.c
View File

@ -152,7 +152,7 @@ void dmGetVnodeLoads(SMonVloadInfo *pInfo) {
if (tsMultiProcess) {
dmSendLocalRecv(pDnode, TDMT_MON_VM_LOAD, tDeserializeSMonVloadInfo, pInfo);
} else if (pWrapper->pMgmt != NULL) {
vmGetVnodeLoads(pWrapper->pMgmt, pInfo);
vmGetVnodeLoads(pWrapper->pMgmt, pInfo, false);
}
dmReleaseWrapper(pWrapper);
}

3
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -44,7 +44,8 @@ int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pV
int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo);
int32_t mndSetMoveVgroupInfoToTrans(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vn, SArray *pArray);
int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId);
int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray);
int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
SArray *pArray);
void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *cntlen, bool standby);
void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

6
source/dnode/mnode/impl/src/mndDb.c
View File

@ -746,7 +746,7 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
return 0;
}
static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SArray *pArray = mndBuildDnodesArray(pMnode, 0);
@ -756,8 +756,8 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (mndVgroupInDb(pVgroup, pNew->uid)) {
if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup, pArray) != 0) {
if (mndVgroupInDb(pVgroup, pNewDb->uid)) {
if (mndBuildAlterVgroupAction(pMnode, pTrans, pOldDb, pNewDb, pVgroup, pArray) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
taosArrayDestroy(pArray);

6
source/dnode/mnode/impl/src/mndDef.c
View File

@ -117,9 +117,13 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
SArray *pArray = taosArrayInit(innerSz, sizeof(void *));
for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
if (pTask == NULL) return -1;
if (pTask == NULL) {
taosArrayDestroy(pArray);
return -1;
}
if (tDecodeSStreamTask(pDecoder, pTask) < 0) {
taosMemoryFree(pTask);
taosArrayDestroy(pArray);
return -1;
}
taosArrayPush(pArray, &pTask);

2
source/dnode/mnode/impl/src/mndMain.c
View File

@ -814,9 +814,11 @@ int32_t mndAcquireRpcRef(SMnode *pMnode) {
int32_t code = 0;
taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) {
mTrace("mnode not running");
terrno = TSDB_CODE_APP_NOT_READY;
code = -1;
} else if (!mndIsMaster(pMnode)) {
mTrace("mnode not ready, role:%s restored:%d", syncGetMyRoleStr(pMnode->syncMgmt.sync), pMnode->restored);
code = -1;
} else {
int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1);

2
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -87,7 +87,7 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj);
if (pRaw == NULL) return -1;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("mnode:%d, will be created when deploying, raw:%p", mnodeObj.id, pRaw);

1
source/dnode/mnode/impl/src/mndStream.c
View File

@ -677,7 +677,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
// build stream obj from request
if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createStreamReq) < 0) {
/*ASSERT(0);*/
mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
goto _OVER;
}

157
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -405,7 +405,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2
pDnode->memUsed = mndGetVnodesMemory(pMnode, pDnode->id);
mInfo("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d online:%d memory avail:%" PRId64 " used:%" PRId64, pDnode->id,
pDnode->numOfVnodes, pDnode->numOfSupportVnodes, isMnode, online, pDnode->memAvail, pDnode->memUsed);
pDnode->numOfVnodes, pDnode->numOfSupportVnodes, isMnode, online, pDnode->memAvail, pDnode->memUsed);
if (isMnode) {
pDnode->numOfVnodes++;
@ -1293,7 +1293,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d avail:%" PRId64 " used:%" PRId64,
pVgroup->dbName, pVgroup->vgId, vgMem, pNew3->id, pNew3->memAvail, pNew3->memUsed);
terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE;
goto _OVER;
goto _OVER;
} else {
pNew3->memUsed += vgMem;
}
@ -1530,67 +1530,102 @@ _OVER:
#endif
}
int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) {
if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) {
return -1;
static int32_t mndCheckDnodeMemory(SMnode *pMnode, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pOldVgroup,
SVgObj *pNewVgroup, SArray *pArray) {
for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) {
SDnodeObj *pDnode = taosArrayGet(pArray, i);
bool inVgroup = false;
for (int32_t j = 0; j < pOldVgroup->replica; ++j) {
SVnodeGid *pVgId = &pOldVgroup->vnodeGid[i];
if (pDnode->id == pVgId->dnodeId) {
pDnode->memUsed -= mndGetVgroupMemory(pMnode, pOldDb, pOldVgroup);
inVgroup = true;
}
}
} else {
SVgObj newVgroup = {0};
memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
mndTransSetSerial(pTrans);
if (newVgroup.replica < pDb->cfg.replications) {
mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId,
pVgroup->vnodeGid[0].dnodeId);
if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1;
if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1;
if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
} else if (newVgroup.replica > pDb->cfg.replications) {
mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId);
SVnodeGid del1 = {0};
if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1;
if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del1, true) != 0) return -1;
if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
SVnodeGid del2 = {0};
if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1;
if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del2, true) != 0) return -1;
if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
for (int32_t j = 0; j < pNewVgroup->replica; ++j) {
SVnodeGid *pVgId = &pNewVgroup->vnodeGid[i];
if (pDnode->id == pVgId->dnodeId) {
pDnode->memUsed += mndGetVgroupMemory(pMnode, pNewDb, pNewVgroup);
inVgroup = true;
}
}
if (pDnode->memAvail - pDnode->memUsed <= 0) {
mError("db:%s, vgId:%d, no enough memory in dnode:%d, avail:%" PRId64 " used:%" PRId64, pNewVgroup->dbName,
pNewVgroup->vgId, pDnode->id, pDnode->memAvail, pDnode->memUsed);
terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE;
return -1;
} else if (inVgroup) {
mInfo("db:%s, vgId:%d, memory in dnode:%d, avail:%" PRId64 " used:%" PRId64, pNewVgroup->dbName,
pNewVgroup->vgId, pDnode->id, pDnode->memAvail, pDnode->memUsed);
} else {
}
}
return 0;
}
{
SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
}
int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
SArray *pArray) {
SVgObj newVgroup = {0};
memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
{
SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
if (pVgroup->replica <= 0 || pVgroup->replica == pNewDb->cfg.replications) {
if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) return -1;
if (mndCheckDnodeMemory(pMnode, pOldDb, pNewDb, &newVgroup, pVgroup, pArray) != 0) return -1;
return 0;
}
mndTransSetSerial(pTrans);
if (newVgroup.replica < pNewDb->cfg.replications) {
mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId,
pVgroup->vnodeGid[0].dnodeId);
if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1;
if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1;
if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
} else if (newVgroup.replica > pNewDb->cfg.replications) {
mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId);
SVnodeGid del1 = {0};
if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1;
if (mndAddSetVnodeStandByAction(pMnode, pTrans, pNewDb, pVgroup, &del1, true) != 0) return -1;
if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del1, true) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
SVnodeGid del2 = {0};
if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1;
if (mndAddSetVnodeStandByAction(pMnode, pTrans, pNewDb, pVgroup, &del2, true) != 0) return -1;
if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del2, true) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
} else {
}
{
SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
}
{
SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
}
return 0;
@ -1650,8 +1685,8 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg2, TDMT_VND_ALTER_HASHRANGE) != 0) goto _OVER;
// adjust vgroup
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg1, pArray) != 0) goto _OVER;
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg2, pArray) != 0) goto _OVER;
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER;
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER;
_OVER:
mndTransDrop(pTrans);
@ -1784,7 +1819,7 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
SDnodeObj *pDnode = taosArrayGet(pArray, i);
mInfo("dnode:%d, equivalent vnodes:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes,
pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes);
pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes);
}
SDnodeObj *pSrc = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1);
@ -1793,7 +1828,7 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes;
float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes;
mInfo("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, srcScore,
pDst->id, dstScore);
pDst->id, dstScore);
if (srcScore > dstScore - 0.000001) {
code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst, pBalancedVgroups);

3
source/dnode/vnode/inc/vnode.h
View File

@ -73,6 +73,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num);
int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num);
int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num);
void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName);
@ -91,6 +92,8 @@ typedef struct SMeta SMeta; // todo: remove
typedef struct SMetaReader SMetaReader;
typedef struct SMetaEntry SMetaEntry;
#define META_READER_NOLOCK 0x1
void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
void metaReaderClear(SMetaReader *pReader);
int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);

10
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -56,6 +56,7 @@ typedef struct SSma SSma;
typedef struct STsdb STsdb;
typedef struct STQ STQ;
typedef struct SVState SVState;
typedef struct SVStatis SVStatis;
typedef struct SVBufPool SVBufPool;
typedef struct SQWorker SQHandle;
typedef struct STsdbKeepCfg STsdbKeepCfg;
@ -284,9 +285,17 @@ struct SVState {
int64_t commitTerm;
};
struct SVStatis {
int64_t nInsert; // delta
int64_t nInsertSuccess; // delta
int64_t nBatchInsert; // delta
int64_t nBatchInsertSuccess; // delta
};
struct SVnodeInfo {
SVnodeCfg config;
SVState state;
SVStatis statis;
};
typedef enum {
@ -309,6 +318,7 @@ struct SVnode {
char* path;
SVnodeCfg config;
SVState state;
SVStatis statis;
STfs* pTfs;
SMsgCb msgCb;
TdThreadMutex mutex;

6
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -19,11 +19,13 @@ void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) {
memset(pReader, 0, sizeof(*pReader));
pReader->flags = flags;
pReader->pMeta = pMeta;
metaRLock(pMeta);
if (!(flags & META_READER_NOLOCK)) {
metaRLock(pMeta);
}
}
void metaReaderClear(SMetaReader *pReader) {
if (pReader->pMeta) {
if (pReader->pMeta && !(pReader->flags & META_READER_NOLOCK)) {
metaULock(pReader->pMeta);
}
tDecoderClear(&pReader->coder);

1
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -180,6 +180,7 @@ int32_t tqMetaRestoreCheckInfo(STQ* pTq) {
return -1;
}
}
tdbFree(pKey);
tdbTbcClose(pCur);
return 0;
}

2
source/dnode/vnode/src/tq/tqOffsetSnapshot.c
View File

@ -54,7 +54,7 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) {
char* fname = tqOffsetBuildFName(pReader->pTq->path, 0);
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile != NULL) {
if (pFile == NULL) {
taosMemoryFree(fname);
return 0;
}

78
source/dnode/vnode/src/tq/tqSink.c
View File

@ -84,26 +84,6 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
continue;
}
STagVal tagVal = {
.cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
.type = TSDB_DATA_TYPE_UBIGINT,
.i64 = (int64_t)pDataBlock->info.groupId,
};
STag* pTag = NULL;
taosArrayClear(tagArray);
taosArrayPush(tagArray, &tagVal);
tTagNew(tagArray, 1, false, &pTag);
if (pTag == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(tagArray);
return NULL;
}
SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
char tagNameStr[TSDB_COL_NAME_LEN] = {0};
strcpy(tagNameStr, "group_id");
taosArrayPush(tagName, tagNameStr);
// STag* pTag = NULL;
// taosArrayClear(tagArray);
// SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
@ -126,42 +106,76 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
// }
SVCreateTbReq createTbReq = {0};
// set const
createTbReq.flags = 0;
createTbReq.type = TSDB_CHILD_TABLE;
createTbReq.ctb.suid = suid;
// set super table name
SName name = {0};
tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName);
// set tag content
taosArrayClear(tagArray);
STagVal tagVal = {
.cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
.type = TSDB_DATA_TYPE_UBIGINT,
.i64 = (int64_t)pDataBlock->info.groupId,
};
taosArrayPush(tagArray, &tagVal);
createTbReq.ctb.tagNum = taosArrayGetSize(tagArray);
STag* pTag = NULL;
tTagNew(tagArray, 1, false, &pTag);
if (pTag == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(tagArray);
taosArrayDestroyP(schemaReqs, taosMemoryFree);
taosArrayDestroy(schemaReqSz);
return NULL;
}
createTbReq.ctb.pTag = (uint8_t*)pTag;
// set tag name
SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
char tagNameStr[TSDB_COL_NAME_LEN] = {0};
strcpy(tagNameStr, "group_id");
taosArrayPush(tagName, tagNameStr);
createTbReq.ctb.tagName = tagName;
// set table name
if (pDataBlock->info.parTbName[0]) {
createTbReq.name = strdup(pDataBlock->info.parTbName);
} else {
createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
}
createTbReq.ctb.suid = suid;
createTbReq.ctb.pTag = (uint8_t*)pTag;
createTbReq.ctb.tagNum = taosArrayGetSize(tagArray);
createTbReq.ctb.tagName = tagName;
// save schema len
int32_t code;
int32_t schemaLen;
tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
if (code < 0) {
tdDestroySVCreateTbReq(&createTbReq);
taosArrayDestroy(tagArray);
taosMemoryFreeClear(ret);
taosArrayDestroyP(schemaReqs, taosMemoryFree);
taosArrayDestroy(schemaReqSz);
return NULL;
}
taosArrayPush(schemaReqSz, &schemaLen);
// save schema str
void* schemaStr = taosMemoryMalloc(schemaLen);
if (schemaStr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tdDestroySVCreateTbReq(&createTbReq);
taosArrayDestroy(tagArray);
taosArrayDestroyP(schemaReqs, taosMemoryFree);
taosArrayDestroy(schemaReqSz);
return NULL;
}
taosArrayPush(schemaReqs, &schemaStr);
taosArrayPush(schemaReqSz, &schemaLen);
SEncoder encoder = {0};
tEncoderInit(&encoder, schemaStr, schemaLen);
@ -169,6 +183,10 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tdDestroySVCreateTbReq(&createTbReq);
taosArrayDestroy(tagArray);
taosArrayDestroyP(schemaReqs, taosMemoryFree);
taosArrayDestroy(schemaReqSz);
tEncoderClear(&encoder);
return NULL;
}
tEncoderClear(&encoder);
@ -221,10 +239,8 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
tqDebug("tq sink, convert block %d, rows: %d", i, rows);
int32_t dataLen = 0;
void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk));
int32_t schemaLen = 0;
void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk));
if (createTb) {
schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i);
void* schemaStr = taosArrayGetP(schemaReqs, i);
@ -262,7 +278,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
ret->length = htonl(ret->length);
if (schemaReqs) taosArrayDestroyP(schemaReqs, taosMemoryFree);
taosArrayDestroyP(schemaReqs, taosMemoryFree);
taosArrayDestroy(schemaReqSz);
return ret;

47
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -265,24 +265,43 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
// only apply to the child tables, ordinary tables will not incur this filter procedure.
size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
SArray *pTmp = taosArrayInit(size, sizeof(SSttBlk));
for (int32_t i = 0; i < size; ++i) {
SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
uint64_t s = p->suid;
if (s < suid) {
continue;
}
if (s == suid) {
taosArrayPush(pTmp, p);
} else if (s > suid) {
break;
if (size > 1) {
SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0);
SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1);
// all identical
if (pStart->suid == pEnd->suid) {
if (pStart->suid == suid) {
// do nothing
} else if (pStart->suid != suid) {
// no qualified stt block existed
(*pIter)->iSttBlk = -1;
double el = (taosGetTimestampUs() - st)/1000.0;
tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
return code;
}
} else {
SArray *pTmp = taosArrayInit(size, sizeof(SSttBlk));
for (int32_t i = 0; i < size; ++i) {
SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
uint64_t s = p->suid;
if (s < suid) {
continue;
}
if (s == suid) {
taosArrayPush(pTmp, p);
} else if (s > suid) {
break;
}
}
taosArrayDestroy(pBlockLoadInfo->aSttBlk);
pBlockLoadInfo->aSttBlk = pTmp;
}
}
taosArrayDestroy(pBlockLoadInfo->aSttBlk);
pBlockLoadInfo->aSttBlk = pTmp;
double el = (taosGetTimestampUs() - st)/1000.0;
tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
}

29
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -28,13 +28,24 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
}
// create vnode env
if (tfsMkdirAt(pTfs, path, (SDiskID){0}) < 0) {
vError("vgId:%d, failed to create vnode since:%s", pCfg->vgId, tstrerror(terrno));
return -1;
if (pTfs) {
if (tfsMkdirAt(pTfs, path, (SDiskID){0}) < 0) {
vError("vgId:%d, failed to create vnode since:%s", pCfg->vgId, tstrerror(terrno));
return -1;
}
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
} else {
if (taosMkDir(path)) {
return TAOS_SYSTEM_ERROR(errno);
}
strcpy(dir, path);
}
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
info.config = *pCfg;
if (pCfg) {
info.config = *pCfg;
} else {
info.config = vnodeCfgDefault;
}
info.state.committed = -1;
info.state.applied = -1;
info.state.commitID = 0;
@ -44,7 +55,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
return -1;
}
vInfo("vgId:%d, vnode is created", pCfg->vgId);
vInfo("vgId:%d, vnode is created", info.config.vgId);
return 0;
}
@ -58,7 +69,11 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
char tdir[TSDB_FILENAME_LEN * 2];
int ret;
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
if (pTfs) {
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
} else {
snprintf(dir, TSDB_FILENAME_LEN, "%s", path);
}
info.config = vnodeCfgDefault;

29
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -15,6 +15,12 @@
#include "vnd.h"
#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType) \
do { \
int##vType##_t newVal = atomic_sub_fetch_##vType(&(pVar), (oVal)); \
ASSERT(newVal >= 0); \
} while (0)
int vnodeQueryOpen(SVnode *pVnode) {
return qWorkerInit(NODE_TYPE_VNODE, TD_VID(pVnode), (void **)&pVnode->pQuery, &pVnode->msgCb);
}
@ -375,13 +381,28 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
pLoad->compStorage = (int64_t)2 * 1073741824;
pLoad->pointsWritten = 100;
pLoad->numOfSelectReqs = 1;
pLoad->numOfInsertReqs = 3;
pLoad->numOfInsertSuccessReqs = 2;
pLoad->numOfBatchInsertReqs = 5;
pLoad->numOfBatchInsertSuccessReqs = 4;
pLoad->numOfInsertReqs = atomic_load_64(&pVnode->statis.nInsert);
pLoad->numOfInsertSuccessReqs = atomic_load_64(&pVnode->statis.nInsertSuccess);
pLoad->numOfBatchInsertReqs = atomic_load_64(&pVnode->statis.nBatchInsert);
pLoad->numOfBatchInsertSuccessReqs = atomic_load_64(&pVnode->statis.nBatchInsertSuccess);
return 0;
}
/**
* @brief Reset the statistics value by monitor interval
*
* @param pVnode
* @param pLoad
*/
void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsert, pLoad->numOfInsertReqs, 64);
VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsertSuccess, pLoad->numOfInsertSuccessReqs, 64);
VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsert, pLoad->numOfBatchInsertReqs, 64);
VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsertSuccess, pLoad->numOfBatchInsertSuccessReqs, 64);
}
void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) {
if (dbname) {
*dbname = pVnode->config.dbname;

65
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -812,10 +812,10 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
SSubmitReq *pSubmitReq = (SSubmitReq *)pReq;
SSubmitRsp submitRsp = {0};
SSubmitMsgIter msgIter = {0};
SSubmitBlk *pBlock;
SSubmitBlk *pBlock = NULL;
SVCreateTbReq createTbReq = {0};
SDecoder decoder = {0};
int32_t nRows;
int32_t nRows = 0;
int32_t tsize, ret;
SEncoder encoder = {0};
SArray *newTbUids = NULL;
@ -823,6 +823,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
pRsp->code = 0;
pSubmitReq->version = version;
atomic_fetch_add_64(&pVnode->statis.nBatchInsert, 1);
#ifdef TD_DEBUG_PRINT_ROW
vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__);
@ -942,11 +943,15 @@ _exit:
taosArrayDestroyEx(submitRsp.pArray, tFreeSSubmitBlkRsp);
atomic_fetch_add_64(&pVnode->statis.nInsert, submitRsp.numOfRows);
atomic_fetch_add_64(&pVnode->statis.nInsertSuccess, submitRsp.affectedRows);
// TODO: the partial success scenario and the error case
// => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level
// 1/level 2.
// TODO: refactor
if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
atomic_fetch_add_64(&pVnode->statis.nBatchInsertSuccess, 1);
tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);
}
@ -1024,71 +1029,75 @@ static int32_t vnodeProcessAlterHashRangeReq(SVnode *pVnode, int64_t version, vo
}
static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
SAlterVnodeReq alterReq = {0};
SAlterVnodeReq req = {0};
bool walChanged = false;
bool tsdbChanged = false;
if (tDeserializeSAlterVnodeReq(pReq, len, &alterReq) != 0) {
if (tDeserializeSAlterVnodeReq(pReq, len, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return TSDB_CODE_INVALID_MSG;
}
vInfo("vgId:%d, start to alter vnode config, cacheLast:%d cacheLastSize:%d", TD_VID(pVnode), alterReq.cacheLast,
alterReq.cacheLastSize);
if (pVnode->config.cacheLastSize != alterReq.cacheLastSize) {
pVnode->config.cacheLastSize = alterReq.cacheLastSize;
vInfo("vgId:%d, start to alter vnode config, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64
" cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d fsync:%d level:%d strict:%d",
TD_VID(pVnode), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024,
req.cacheLast, req.cacheLastSize, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
req.walFsyncPeriod, req.walLevel, req.strict);
if (pVnode->config.cacheLastSize != req.cacheLastSize) {
pVnode->config.cacheLastSize = req.cacheLastSize;
tsdbCacheSetCapacity(pVnode, (size_t)pVnode->config.cacheLastSize * 1024 * 1024);
}
if (pVnode->config.szBuf != alterReq.buffer * 1024LL * 1024LL) {
if (pVnode->config.szBuf != req.buffer * 1024LL * 1024LL) {
vInfo("vgId:%d vnode buffer is changed from %" PRId64 " to %" PRId64, TD_VID(pVnode), pVnode->config.szBuf,
alterReq.buffer * 1024LL * 1024LL);
pVnode->config.szBuf = alterReq.buffer * 1024LL * 1024LL;
req.buffer * 1024LL * 1024LL);
pVnode->config.szBuf = req.buffer * 1024LL * 1024LL;
}
if (pVnode->config.szCache != alterReq.pages) {
if (metaAlterCache(pVnode->pMeta, alterReq.pages) < 0) {
if (pVnode->config.szCache != req.pages) {
if (metaAlterCache(pVnode->pMeta, req.pages) < 0) {
vError("vgId:%d failed to change vnode pages from %d to %d failed since %s", TD_VID(pVnode),
pVnode->config.szCache, alterReq.pages, tstrerror(errno));
pVnode->config.szCache, req.pages, tstrerror(errno));
return errno;
} else {
vInfo("vgId:%d vnode pages is changed from %d to %d", TD_VID(pVnode), pVnode->config.szCache, alterReq.pages);
pVnode->config.szCache = alterReq.pages;
vInfo("vgId:%d vnode pages is changed from %d to %d", TD_VID(pVnode), pVnode->config.szCache, req.pages);
pVnode->config.szCache = req.pages;
}
}
if (pVnode->config.cacheLast != alterReq.cacheLast) {
pVnode->config.cacheLast = alterReq.cacheLast;
if (pVnode->config.cacheLast != req.cacheLast) {
pVnode->config.cacheLast = req.cacheLast;
}
if (pVnode->config.walCfg.fsyncPeriod != alterReq.walFsyncPeriod) {
pVnode->config.walCfg.fsyncPeriod = alterReq.walFsyncPeriod;
if (pVnode->config.walCfg.fsyncPeriod != req.walFsyncPeriod) {
pVnode->config.walCfg.fsyncPeriod = req.walFsyncPeriod;
walChanged = true;
}
if (pVnode->config.walCfg.level != alterReq.walLevel) {
pVnode->config.walCfg.level = alterReq.walLevel;
if (pVnode->config.walCfg.level != req.walLevel) {
pVnode->config.walCfg.level = req.walLevel;
walChanged = true;
}
if (pVnode->config.tsdbCfg.keep0 != alterReq.daysToKeep0) {
pVnode->config.tsdbCfg.keep0 = alterReq.daysToKeep0;
if (pVnode->config.tsdbCfg.keep0 != req.daysToKeep0) {
pVnode->config.tsdbCfg.keep0 = req.daysToKeep0;
if (!VND_IS_RSMA(pVnode)) {
tsdbChanged = true;
}
}
if (pVnode->config.tsdbCfg.keep1 != alterReq.daysToKeep1) {
pVnode->config.tsdbCfg.keep1 = alterReq.daysToKeep1;
if (pVnode->config.tsdbCfg.keep1 != req.daysToKeep1) {
pVnode->config.tsdbCfg.keep1 = req.daysToKeep1;
if (!VND_IS_RSMA(pVnode)) {
tsdbChanged = true;
}
}
if (pVnode->config.tsdbCfg.keep2 != alterReq.daysToKeep2) {
pVnode->config.tsdbCfg.keep2 = alterReq.daysToKeep2;
if (pVnode->config.tsdbCfg.keep2 != req.daysToKeep2) {
pVnode->config.tsdbCfg.keep2 = req.daysToKeep2;
if (!VND_IS_RSMA(pVnode)) {
tsdbChanged = true;
}

228
source/libs/catalog/src/ctgUtil.c
View File

@ -1193,4 +1193,232 @@ SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) {
return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx);
}
static void* ctgCloneDbVgroup(void* pSrc) {
return taosArrayDup((const SArray*)pSrc);
}
static void ctgFreeDbVgroup(void* p) {
taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
}
static void* ctgCloneDbCfgInfo(void* pSrc) {
SDbCfgInfo* pDst = taosMemoryMalloc(sizeof(SDbCfgInfo));
if (NULL == pDst) {
return NULL;
}
memcpy(pDst, pSrc, sizeof(SDbCfgInfo));
return pDst;
}
static void ctgFreeDbCfgInfo(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void* ctgCloneDbInfo(void* pSrc) {
SDbInfo* pDst = taosMemoryMalloc(sizeof(SDbInfo));
if (NULL == pDst) {
return NULL;
}
memcpy(pDst, pSrc, sizeof(SDbInfo));
return pDst;
}
static void ctgFreeDbInfo(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void* ctgCloneTableMeta(void* pSrc) {
STableMeta* pMeta = pSrc;
int32_t size = sizeof(STableMeta) + (pMeta->tableInfo.numOfColumns + pMeta->tableInfo.numOfTags) * sizeof(SSchema);
STableMeta* pDst = taosMemoryMalloc(size);
if (NULL == pDst) {
return NULL;
}
memcpy(pDst, pSrc, size);
return pDst;
}
static void ctgFreeTableMeta(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void* ctgCloneVgroupInfo(void* pSrc) {
SVgroupInfo* pDst = taosMemoryMalloc(sizeof(SVgroupInfo));
if (NULL == pDst) {
return NULL;
}
memcpy(pDst, pSrc, sizeof(SVgroupInfo));
return pDst;
}
static void ctgFreeVgroupInfo(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void* ctgCloneTableIndices(void* pSrc) {
return taosArrayDup((const SArray*)pSrc);
}
static void ctgFreeTableIndices(void* p) {
taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
}
static void* ctgCloneFuncInfo(void* pSrc) {
SFuncInfo* pDst = taosMemoryMalloc(sizeof(SFuncInfo));
if (NULL == pDst) {
return NULL;
}
memcpy(pDst, pSrc, sizeof(SFuncInfo));
return pDst;
}
static void ctgFreeFuncInfo(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void* ctgCloneIndexInfo(void* pSrc) {
SIndexInfo* pDst = taosMemoryMalloc(sizeof(SIndexInfo));
if (NULL == pDst) {
return NULL;
}
memcpy(pDst, pSrc, sizeof(SIndexInfo));
return pDst;
}
static void ctgFreeIndexInfo(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void* ctgCloneUserAuth(void* pSrc) {
bool* pDst = taosMemoryMalloc(sizeof(bool));
if (NULL == pDst) {
return NULL;
}
*pDst = *(bool*)pSrc;
return pDst;
}
static void ctgFreeUserAuth(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void* ctgCloneQnodeList(void* pSrc) {
return taosArrayDup((const SArray*)pSrc);
}
static void ctgFreeQnodeList(void* p) {
taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
}
static void* ctgCloneTableCfg(void* pSrc) {
STableCfg* pDst = taosMemoryMalloc(sizeof(STableCfg));
if (NULL == pDst) {
return NULL;
}
memcpy(pDst, pSrc, sizeof(STableCfg));
return pDst;
}
static void ctgFreeTableCfg(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void* ctgCloneDnodeList(void* pSrc) {
return taosArrayDup((const SArray*)pSrc);
}
static void ctgFreeDnodeList(void* p) {
taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
}
static int32_t ctgCloneMetaDataArray(SArray* pSrc, FCopy copyFunc, SArray** pDst) {
if (NULL == pSrc) {
return TSDB_CODE_SUCCESS;
}
int32_t size = taosArrayGetSize(pSrc);
*pDst = taosArrayInit(size, sizeof(SMetaRes));
if (NULL == *pDst) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < size; ++i) {
SMetaRes* pRes = taosArrayGet(pSrc, i);
SMetaRes res = {.code = pRes->code, .pRes = copyFunc(pRes->pRes)};
if (NULL == res.pRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(*pDst, &res);
}
return TSDB_CODE_SUCCESS;
}
SMetaData* catalogCloneMetaData(SMetaData* pData) {
SMetaData* pRes = taosMemoryCalloc(1, sizeof(SMetaData));
if (NULL == pRes) {
return NULL;
}
int32_t code = ctgCloneMetaDataArray(pData->pDbVgroup, ctgCloneDbVgroup, &pRes->pDbVgroup);
if (TSDB_CODE_SUCCESS == code) {
code = ctgCloneMetaDataArray(pData->pDbCfg, ctgCloneDbCfgInfo, &pRes->pDbCfg);
}
if (TSDB_CODE_SUCCESS == code) {
code = ctgCloneMetaDataArray(pData->pDbInfo, ctgCloneDbInfo, &pRes->pDbInfo);
}
if (TSDB_CODE_SUCCESS == code) {
code = ctgCloneMetaDataArray(pData->pTableMeta, ctgCloneTableMeta, &pRes->pTableMeta);
}
if (TSDB_CODE_SUCCESS == code) {
code = ctgCloneMetaDataArray(pData->pTableHash, ctgCloneVgroupInfo, &pRes->pTableHash);
}
if (TSDB_CODE_SUCCESS == code) {
code = ctgCloneMetaDataArray(pData->pTableIndex, ctgCloneTableIndices, &pRes->pTableIndex);
}
if (TSDB_CODE_SUCCESS == code) {
code = ctgCloneMetaDataArray(pData->pUdfList, ctgCloneFuncInfo, &pRes->pUdfList);
}
if (TSDB_CODE_SUCCESS == code) {
code = ctgCloneMetaDataArray(pData->pIndex, ctgCloneIndexInfo, &pRes->pIndex);
}
if (TSDB_CODE_SUCCESS == code) {
code = ctgCloneMetaDataArray(pData->pUser, ctgCloneUserAuth, &pRes->pUser);
}
if (TSDB_CODE_SUCCESS == code) {
code = ctgCloneMetaDataArray(pData->pQnodeList, ctgCloneQnodeList, &pRes->pQnodeList);
}
if (TSDB_CODE_SUCCESS == code) {
code = ctgCloneMetaDataArray(pData->pTableCfg, ctgCloneTableCfg, &pRes->pTableCfg);
}
if (TSDB_CODE_SUCCESS == code) {
code = ctgCloneMetaDataArray(pData->pDnodeList, ctgCloneDnodeList, &pRes->pDnodeList);
}
if (TSDB_CODE_SUCCESS != code) {
catalogFreeMetaData(pRes);
return NULL;
}
return pRes;
}
void catalogFreeMetaData(SMetaData* pData) {
if (NULL == pData) {
return;
}
taosArrayDestroyEx(pData->pDbVgroup, ctgFreeDbVgroup);
taosArrayDestroyEx(pData->pDbCfg, ctgFreeDbCfgInfo);
taosArrayDestroyEx(pData->pDbInfo, ctgFreeDbInfo);
taosArrayDestroyEx(pData->pTableMeta, ctgFreeTableMeta);
taosArrayDestroyEx(pData->pTableHash, ctgFreeVgroupInfo);
taosArrayDestroyEx(pData->pTableIndex, ctgFreeTableIndices);
taosArrayDestroyEx(pData->pUdfList, ctgFreeFuncInfo);
taosArrayDestroyEx(pData->pIndex, ctgFreeIndexInfo);
taosArrayDestroyEx(pData->pUser, ctgFreeUserAuth);
taosArrayDestroyEx(pData->pQnodeList, ctgFreeQnodeList);
taosArrayDestroyEx(pData->pTableCfg, ctgFreeTableCfg);
taosArrayDestroyEx(pData->pDnodeList, ctgFreeDnodeList);
taosMemoryFreeClear(pData->pSvrVer);
taosMemoryFree(pData);
}

6
source/libs/executor/inc/executorimpl.h
View File

@ -768,6 +768,7 @@ typedef struct SStreamSessionAggOperatorInfo {
SPhysiNode* pPhyNode; // create new child
bool isFinal;
bool ignoreExpiredData;
SHashObj* pGroupIdTbNameMap;
} SStreamSessionAggOperatorInfo;
typedef struct SStreamPartitionOperatorInfo {
@ -846,6 +847,7 @@ typedef struct SStreamStateAggOperatorInfo {
void* pDelIterator;
SArray* pChildren; // cache for children's result;
bool ignoreExpiredData;
SHashObj* pGroupIdTbNameMap;
} SStreamStateAggOperatorInfo;
typedef struct SSortOperatorInfo {
@ -900,8 +902,12 @@ void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs);
int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey);
void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows);
void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
SDiskbasedBuf* pBuf);
void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
SDiskbasedBuf* pBuf);
int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf);
bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);

40
source/libs/executor/src/executorimpl.c
View File

@ -1401,6 +1401,46 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
return 0;
}
void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
SDiskbasedBuf* pBuf) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SSDataBlock* pBlock = pbInfo->pRes;
// set output datablock version
pBlock->info.version = pTaskInfo->version;
blockDataCleanup(pBlock);
if (!hasRemainResults(pGroupResInfo)) {
return;
}
// clear the existed group id
pBlock->info.groupId = 0;
ASSERT(!pbInfo->mergeResultBlock);
doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo);
if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) {
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t));
if (tbname != NULL) {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} else {
pBlock->info.parTbName[0] = 0;
}
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t));
if (tbname != NULL) {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} else {
pBlock->info.parTbName[0] = 0;
}
}
}
void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
SDiskbasedBuf* pBuf) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

76
source/libs/executor/src/scanoperator.c
View File

@ -1305,6 +1305,56 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock,
return code;
}
static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* pResBlock) {
if (pTagCalSup == NULL || pTagCalSup->numOfExprs == 0) return;
if (pBlock == NULL || pBlock->info.rows == 0) return;
SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0);
ASSERT(pSrcBlock->info.rows == 1);
blockDataEnsureCapacity(pResBlock, 1);
projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, 1, NULL);
ASSERT(pResBlock->info.rows == 1);
// build tagArray
// build STag
// set STag
blockDataDestroy(pSrcBlock);
}
static void calBlockTbName(SExprSupp* pTbNameCalSup, SSDataBlock* pBlock) {
if (pTbNameCalSup == NULL || pTbNameCalSup->numOfExprs == 0) return;
if (pBlock == NULL || pBlock->info.rows == 0) return;
SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0);
ASSERT(pSrcBlock->info.rows == 1);
SSDataBlock* pResBlock = createDataBlock();
pResBlock->info.rowSize = VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN;
SColumnInfoData data = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_TABLE_NAME_LEN, 0);
taosArrayPush(pResBlock->pDataBlock, &data);
blockDataEnsureCapacity(pResBlock, 1);
projectApplyFunctions(pTbNameCalSup->pExprInfo, pResBlock, pSrcBlock, pTbNameCalSup->pCtx, 1, NULL);
ASSERT(pResBlock->info.rows == 1);
ASSERT(taosArrayGetSize(pResBlock->pDataBlock) == 1);
SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, 0);
ASSERT(pCol->info.type == TSDB_DATA_TYPE_VARCHAR);
void* pData = colDataGetData(pCol, 0);
// TODO check tbname validation
if (pData != (void*)-1 && pData != NULL) {
memcpy(pBlock->info.parTbName, varDataVal(pData), varDataLen(pData));
} else {
pBlock->info.parTbName[0] = 0;
}
blockDataDestroy(pSrcBlock);
blockDataDestroy(pResBlock);
}
void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
uint64_t* pGp, void* pTbName) {
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
@ -1424,28 +1474,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
blockDataFreeRes((SSDataBlock*)pBlock);
if (pInfo->tbnameCalSup.numOfExprs > 0 && pInfo->pRes->info.rows > 0) {
SSDataBlock* pTmpBlock = blockCopyOneRow(pInfo->pRes, 0);
SSDataBlock* pResBlock = createDataBlock();
pResBlock->info.rowSize = TSDB_TABLE_NAME_LEN;
SColumnInfoData data = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_TABLE_NAME_LEN, 0);
taosArrayPush(pResBlock->pDataBlock, &data);
blockDataEnsureCapacity(pResBlock, 1);
projectApplyFunctions(pInfo->tbnameCalSup.pExprInfo, pResBlock, pTmpBlock, pInfo->tbnameCalSup.pCtx, 1, NULL);
ASSERT(pResBlock->info.rows == 1);
ASSERT(taosArrayGetSize(pResBlock->pDataBlock) == 1);
SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, 0);
ASSERT(pCol->info.type == TSDB_DATA_TYPE_VARCHAR);
void* pData = colDataGetData(pCol, 0);
// TODO check tbname validation
if (pData != (void*)-1) {
memcpy(pInfo->pRes->info.parTbName, varDataVal(pData), varDataLen(pData));
} else {
pInfo->pRes->info.parTbName[0] = 0;
}
blockDataDestroy(pTmpBlock);
blockDataDestroy(pResBlock);
}
calBlockTbName(&pInfo->tbnameCalSup, pInfo->pRes);
return 0;
}
@ -1783,6 +1812,7 @@ FETCH_NEXT_BLOCK:
pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA;
checkUpdateData(pInfo, true, pSDB, false);
// printDataBlock(pSDB, "stream scan update");
calBlockTbName(&pInfo->tbnameCalSup, pSDB);
return pSDB;
}
blockDataCleanup(pInfo->pUpdateDataRes);
@ -2760,7 +2790,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
colDataAppend(pColInfoData, numOfRows, (char*)&ts, false);
SMetaReader mr = {0};
metaReaderInit(&mr, pInfo->readHandle.meta, 0);
metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK);
uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
int32_t code = metaGetTableEntryByUid(&mr, suid);

8
source/libs/executor/src/tfill.c
View File

@ -105,9 +105,8 @@ static bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, S
} else if (pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_END) {
// TODO: include endpoint
SInterval* pInterval = &pFillInfo->interval;
int32_t step = (pFillInfo->order == TSDB_ORDER_ASC) ? 1 : -1;
int64_t windowEnd =
taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision);
taosTimeAdd(pFillInfo->currentKey, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
colDataAppend(pDstColInfoData, rowIndex, (const char*)&windowEnd, false);
return true;
} else if (pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_DURATION) {
@ -562,10 +561,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma
int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
TSKEY ekey1 = ekey;
if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->interval.precision);
}
int64_t numOfRes = -1;
if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set.
TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];

34
source/libs/executor/src/timewindowoperator.c
View File

@ -3728,6 +3728,9 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
pInfo->pPhyNode = pPhyNode;
pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
pInfo->pGroupIdTbNameMap =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
pOperator->name = "StreamSessionWindowAggOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
pOperator->blocking = true;
@ -4351,7 +4354,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
}
@ -4370,6 +4373,12 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
}
printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv");
if (pBlock->info.parTbName[0]) {
taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName,
TSDB_TABLE_NAME_LEN);
/*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/
}
if (pBlock->info.type == STREAM_CLEAR) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, START_TS_COLUMN_INDEX,
@ -4451,7 +4460,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
doBuildStreamResBlock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}
@ -4482,7 +4491,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
}
{
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, "semi session");
return pBInfo->pRes;
@ -4523,6 +4532,12 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
}
printDataBlock(pBlock, "semi session recv");
if (pBlock->info.parTbName[0]) {
taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName,
TSDB_TABLE_NAME_LEN);
/*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/
}
if (pBlock->info.type == STREAM_CLEAR) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, START_TS_COLUMN_INDEX, pSup->numOfExprs, 0, pWins);
@ -4566,7 +4581,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, "semi session");
return pBInfo->pRes;
@ -4610,6 +4625,10 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL,
destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
}
pInfo->pGroupIdTbNameMap =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
pOperator->operatorType = pPhyNode->type;
if (numOfChild > 0) {
pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
@ -4910,7 +4929,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
printDataBlock(pInfo->pDelRes, "single state");
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
}
@ -4972,7 +4991,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
printDataBlock(pInfo->pDelRes, "single state");
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
doBuildStreamResBlock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
printDataBlock(pBInfo->pRes, "single state");
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}
@ -5045,6 +5064,9 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->pChildren = NULL;
pInfo->ignoreExpiredData = pStateNode->window.igExpired;
pInfo->pGroupIdTbNameMap =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
pOperator->name = "StreamStateAggOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
pOperator->blocking = true;

2
source/libs/function/src/builtins.c
View File

@ -662,7 +662,7 @@ static int32_t translateTopBot(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
if (pValue->datum.i < 1 || pValue->datum.i > 100) {
if (pValue->datum.i < 1 || pValue->datum.i > TOP_BOTTOM_QUERY_LIMIT) {
return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
}

108
source/libs/parser/src/parInsert.c
View File

@ -266,7 +266,7 @@ static int32_t getTableVgroup(SInsertParseContext* pCxt, int32_t tbNo, SName* pT
return catalogGetTableHashVgroup(pBasicCtx->pCatalog, &conn, pTbName, pVg);
}
static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* dbFname, bool isStb) {
static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName* name, bool isStb) {
CHECK_CODE(getTableSchema(pCxt, tbNo, name, isStb, &pCxt->pTableMeta));
if (!isStb) {
SVgroupInfo vg;
@ -276,12 +276,12 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName*
return TSDB_CODE_SUCCESS;
}
static int32_t getTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* dbFname) {
return getTableMetaImpl(pCxt, tbNo, name, dbFname, false);
static int32_t getTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) {
return getTableMetaImpl(pCxt, tbNo, name, false);
}
static int32_t getSTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* dbFname) {
return getTableMetaImpl(pCxt, tbNo, name, dbFname, true);
static int32_t getSTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) {
return getTableMetaImpl(pCxt, tbNo, name, true);
}
static int32_t getDBCfg(SInsertParseContext* pCxt, const char* pDbFName, SDbCfgInfo* pInfo) {
@ -1178,7 +1178,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName*
tNameGetFullDbName(&sname, dbFName);
strcpy(pCxt->sTableName, sname.tname);
CHECK_CODE(getSTableMeta(pCxt, tbNo, &sname, dbFName));
CHECK_CODE(getSTableMeta(pCxt, tbNo, &sname));
if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) {
return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed");
}
@ -1385,6 +1385,10 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataB
(*numOfRows)++;
}
pCxt->pSql = pRawSql;
if (pDataBlock->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) {
break;
}
}
if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
@ -1393,23 +1397,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataB
return TSDB_CODE_SUCCESS;
}
static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STableDataBlocks* dataBuf) {
char filePathStr[TSDB_FILENAME_LEN] = {0};
if (TK_NK_STRING == filePath.type) {
trimString(filePath.z, filePath.n, filePathStr, sizeof(filePathStr));
} else {
strncpy(filePathStr, filePath.z, filePath.n);
}
TdFilePtr fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM);
if (NULL == fp) {
return TAOS_SYSTEM_ERROR(errno);
}
static int32_t parseDataFromFileAgain(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName,
STableDataBlocks* dataBuf) {
int32_t maxNumOfRows;
CHECK_CODE(allocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows));
int32_t numOfRows = 0;
CHECK_CODE(parseCsvFile(pCxt, fp, dataBuf, maxNumOfRows, &numOfRows));
CHECK_CODE(parseCsvFile(pCxt, pCxt->pComCxt->csvCxt.fp, dataBuf, maxNumOfRows, &numOfRows));
SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData);
if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) {
@ -1417,12 +1411,38 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STa
"too many rows in sql, total number of rows should be less than INT32_MAX");
}
if (!taosEOFFile(pCxt->pComCxt->csvCxt.fp)) {
pCxt->pComCxt->needMultiParse = true;
pCxt->pComCxt->csvCxt.tableNo = tableNo;
memcpy(&pCxt->pComCxt->csvCxt.tableName, pTableName, sizeof(SName));
pCxt->pComCxt->csvCxt.pLastSqlPos = pCxt->pSql;
}
dataBuf->numOfTables = 1;
pCxt->totalNum += numOfRows;
return TSDB_CODE_SUCCESS;
}
static int32_t parseDataFromFile(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName, SToken filePath,
STableDataBlocks* dataBuf) {
char filePathStr[TSDB_FILENAME_LEN] = {0};
if (TK_NK_STRING == filePath.type) {
trimString(filePath.z, filePath.n, filePathStr, sizeof(filePathStr));
} else {
strncpy(filePathStr, filePath.z, filePath.n);
}
pCxt->pComCxt->csvCxt.fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM);
if (NULL == pCxt->pComCxt->csvCxt.fp) {
return TAOS_SYSTEM_ERROR(errno);
}
return parseDataFromFileAgain(pCxt, tableNo, pTableName, dataBuf);
}
static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) {
if (!pCxt->pComCxt->needMultiParse) {
taosCloseFile(&pCxt->pComCxt->csvCxt.fp);
}
taosMemoryFreeClear(pCxt->pTableMeta);
destroyBoundColumnInfo(&pCxt->tags);
tdDestroySVCreateTbReq(&pCxt->createTblReq);
@ -1481,7 +1501,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", sToken.z);
}
if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) &&
!pCxt->pComCxt->needMultiParse) {
return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");
}
break;
@ -1536,7 +1557,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
NEXT_TOKEN(pCxt->pSql, sToken);
autoCreateTbl = true;
} else if (!existedUsing) {
CHECK_CODE(getTableMeta(pCxt, tbNum, &name, dbFName));
CHECK_CODE(getTableMeta(pCxt, tbNum, &name));
if (TSDB_SUPER_TABLE == pCxt->pTableMeta->tableType) {
return buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported");
}
@ -1577,17 +1598,22 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
if (0 == sToken.n || (TK_NK_STRING != sToken.type && TK_NK_ID != sToken.type)) {
return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z);
}
CHECK_CODE(parseDataFromFile(pCxt, sToken, dataBuf));
CHECK_CODE(parseDataFromFile(pCxt, tbNum, &name, sToken, dataBuf));
pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT;
tbNum++;
continue;
if (!pCxt->pComCxt->needMultiParse) {
continue;
} else {
parserInfo("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId);
break;
}
}
return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z);
}
qDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
parserInfo("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) {
SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags));
@ -1612,6 +1638,26 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
return buildOutput(pCxt);
}
static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) {
STableDataBlocks* dataBuf = NULL;
CHECK_CODE(getTableMeta(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName));
CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, &pCxt->pTableMeta->uid, sizeof(pCxt->pTableMeta->uid),
TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize,
pCxt->pTableMeta, &dataBuf, NULL, &pCxt->createTblReq));
CHECK_CODE(parseDataFromFileAgain(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName, dataBuf));
if (taosEOFFile(pCxt->pComCxt->csvCxt.fp)) {
CHECK_CODE(parseInsertBody(pCxt));
pCxt->pComCxt->needMultiParse = false;
return TSDB_CODE_SUCCESS;
}
parserInfo("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
// merge according to vgId
if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) {
CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks));
}
return buildOutput(pCxt);
}
// INSERT INTO
// tb_name
// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)]
@ -1621,7 +1667,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache) {
SInsertParseContext context = {
.pComCxt = pContext,
.pSql = (char*)pContext->pSql,
.pSql = pContext->needMultiParse ? (char*)pContext->csvCxt.pLastSqlPos : (char*)pContext->pSql,
.msg = {.buf = pContext->pMsg, .len = pContext->msgLen},
.pTableMeta = NULL,
.createTblReq = {0},
@ -1691,10 +1737,16 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache
context.pOutput->payloadType = PAYLOAD_TYPE_KV;
int32_t code = skipInsertInto(&context.pSql, &context.msg);
if (TSDB_CODE_SUCCESS == code) {
code = parseInsertBody(&context);
int32_t code = TSDB_CODE_SUCCESS;
if (!context.pComCxt->needMultiParse) {
code = skipInsertInto(&context.pSql, &context.msg);
if (TSDB_CODE_SUCCESS == code) {
code = parseInsertBody(&context);
}
} else {
code = parseInsertBodyAgain(&context);
}
if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) {
SName* pTable = taosHashIterate(context.pTableNameHashObj, NULL);
while (NULL != pTable) {

10
source/libs/parser/src/parser.c
View File

@ -214,6 +214,16 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata
return code;
}
void qDestroyParseContext(SParseContext* pCxt) {
if (NULL == pCxt) {
return;
}
taosArrayDestroy(pCxt->pTableMetaPos);
taosArrayDestroy(pCxt->pTableVgroupPos);
taosMemoryFree(pCxt);
}
void qDestroyQuery(SQuery* pQueryNode) { nodesDestroyNode((SNode*)pQueryNode); }
int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) {

20
source/libs/parser/test/mockCatalog.cpp
View File

@ -130,16 +130,16 @@ void generatePerformanceSchema(MockCatalogService* mcs) {
* c5 | column | DOUBLE | 8 |
*/
void generateTestTables(MockCatalogService* mcs, const std::string& db) {
ITableBuilder& builder = mcs->createTableBuilder(db, "t1", TSDB_NORMAL_TABLE, 6)
.setPrecision(TSDB_TIME_PRECISION_MILLI)
.setVgid(1)
.addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP)
.addColumn("c1", TSDB_DATA_TYPE_INT)
.addColumn("c2", TSDB_DATA_TYPE_BINARY, 20)
.addColumn("c3", TSDB_DATA_TYPE_BIGINT)
.addColumn("c4", TSDB_DATA_TYPE_DOUBLE)
.addColumn("c5", TSDB_DATA_TYPE_DOUBLE);
builder.done();
mcs->createTableBuilder(db, "t1", TSDB_NORMAL_TABLE, 6)
.setPrecision(TSDB_TIME_PRECISION_MILLI)
.setVgid(1)
.addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP)
.addColumn("c1", TSDB_DATA_TYPE_INT)
.addColumn("c2", TSDB_DATA_TYPE_BINARY, 20)
.addColumn("c3", TSDB_DATA_TYPE_BIGINT)
.addColumn("c4", TSDB_DATA_TYPE_DOUBLE)
.addColumn("c5", TSDB_DATA_TYPE_DOUBLE)
.done();
}
/*

1
source/libs/parser/test/parTestUtil.cpp
View File

@ -343,7 +343,6 @@ class ParserTestBaseImpl {
unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
doParseSql(&cxt, query.get());
SQuery* pQuery = *(query.get());
if (g_dump) {
dump();

2
source/libs/scalar/src/filter.c
View File

@ -1939,7 +1939,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
fi->data = taosMemoryCalloc(1, bytes);
} else{
} else {
if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE
/*
fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes);

6
source/libs/scalar/src/scalar.c
View File

@ -363,7 +363,11 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
}
SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index);
if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) {
if (NULL == block) {
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
if (ref->slotId >= taosArrayGetSize(block->pDataBlock)) {
sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}

248
source/libs/scalar/src/sclfunc.c
View File

@ -709,8 +709,15 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
int16_t outputType = GET_PARAM_TYPE(&pOutput[0]);
int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]);
char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows + 1, 1);
char *output = outputBuf;
int32_t code = TSDB_CODE_SUCCESS;
char * convBuf = taosMemoryMalloc(inputLen);
char * output = taosMemoryCalloc(1, outputLen + TSDB_NCHAR_SIZE);
char buf[400] = {0};
if (convBuf == NULL || output == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
if (colDataIsNull_s(pInput[0].columnData, i)) {
@ -723,17 +730,18 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
switch(outputType) {
case TSDB_DATA_TYPE_TINYINT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(int8_t *)output = taosStr2Int8(varDataVal(input), NULL, 10);
memcpy(buf, varDataVal(input), varDataLen(input));
buf[varDataLen(input)] = 0;
*(int8_t *)output = taosStr2Int8(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
newBuf[len] = 0;
*(int8_t *)output = taosStr2Int8(newBuf, NULL, 10);
taosMemoryFree(newBuf);
convBuf[len] = 0;
*(int8_t *)output = taosStr2Int8(convBuf, NULL, 10);
} else {
GET_TYPED_DATA(*(int8_t *)output, int8_t, inputType, input);
}
@ -741,17 +749,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
case TSDB_DATA_TYPE_SMALLINT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(int16_t *)output = taosStr2Int16(varDataVal(input), NULL, 10);
memcpy(buf, varDataVal(input), varDataLen(input));
buf[varDataLen(input)] = 0;
*(int16_t *)output = taosStr2Int16(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
newBuf[len] = 0;
*(int16_t *)output = taosStr2Int16(newBuf, NULL, 10);
taosMemoryFree(newBuf);
convBuf[len] = 0;
*(int16_t *)output = taosStr2Int16(convBuf, NULL, 10);
} else {
GET_TYPED_DATA(*(int16_t *)output, int16_t, inputType, input);
}
@ -759,17 +767,18 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
case TSDB_DATA_TYPE_INT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(int32_t *)output = taosStr2Int32(varDataVal(input), NULL, 10);
memcpy(buf, varDataVal(input), varDataLen(input));
buf[varDataLen(input)] = 0;
*(int32_t *)output = taosStr2Int32(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
newBuf[len] = 0;
*(int32_t *)output = taosStr2Int32(newBuf, NULL, 10);
taosMemoryFree(newBuf);
convBuf[len] = 0;
*(int32_t *)output = taosStr2Int32(convBuf, NULL, 10);
} else {
GET_TYPED_DATA(*(int32_t *)output, int32_t, inputType, input);
}
@ -777,17 +786,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
case TSDB_DATA_TYPE_BIGINT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(int64_t *)output = taosStr2Int64(varDataVal(input), NULL, 10);
memcpy(buf, varDataVal(input), varDataLen(input));
buf[varDataLen(input)] = 0;
*(int64_t *)output = taosStr2Int64(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
newBuf[len] = 0;
*(int64_t *)output = taosStr2Int64(newBuf, NULL, 10);
taosMemoryFree(newBuf);
convBuf[len] = 0;
*(int64_t *)output = taosStr2Int64(convBuf, NULL, 10);
} else {
GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input);
}
@ -795,17 +804,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
case TSDB_DATA_TYPE_UTINYINT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(uint8_t *)output = taosStr2UInt8(varDataVal(input), NULL, 10);
memcpy(buf, varDataVal(input), varDataLen(input));
buf[varDataLen(input)] = 0;
*(uint8_t *)output = taosStr2UInt8(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
newBuf[len] = 0;
*(uint8_t *)output = taosStr2UInt8(newBuf, NULL, 10);
taosMemoryFree(newBuf);
convBuf[len] = 0;
*(uint8_t *)output = taosStr2UInt8(convBuf, NULL, 10);
} else {
GET_TYPED_DATA(*(uint8_t *)output, uint8_t, inputType, input);
}
@ -813,17 +822,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
case TSDB_DATA_TYPE_USMALLINT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(uint16_t *)output = taosStr2UInt16(varDataVal(input), NULL, 10);
memcpy(buf, varDataVal(input), varDataLen(input));
buf[varDataLen(input)] = 0;
*(uint16_t *)output = taosStr2UInt16(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
newBuf[len] = 0;
*(uint16_t *)output = taosStr2UInt16(newBuf, NULL, 10);
taosMemoryFree(newBuf);
convBuf[len] = 0;
*(uint16_t *)output = taosStr2UInt16(convBuf, NULL, 10);
} else {
GET_TYPED_DATA(*(uint16_t *)output, uint16_t, inputType, input);
}
@ -831,17 +840,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
case TSDB_DATA_TYPE_UINT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(uint32_t *)output = taosStr2UInt32(varDataVal(input), NULL, 10);
memcpy(buf, varDataVal(input), varDataLen(input));
buf[varDataLen(input)] = 0;
*(uint32_t *)output = taosStr2UInt32(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
newBuf[len] = 0;
*(uint32_t *)output = taosStr2UInt32(newBuf, NULL, 10);
taosMemoryFree(newBuf);
convBuf[len] = 0;
*(uint32_t *)output = taosStr2UInt32(convBuf, NULL, 10);
} else {
GET_TYPED_DATA(*(uint32_t *)output, uint32_t, inputType, input);
}
@ -849,17 +858,18 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
case TSDB_DATA_TYPE_UBIGINT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(uint64_t *)output = taosStr2UInt64(varDataVal(input), NULL, 10);
memcpy(buf, varDataVal(input), varDataLen(input));
buf[varDataLen(input)] = 0;
*(uint64_t *)output = taosStr2UInt64(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
newBuf[len] = 0;
*(uint64_t *)output = taosStr2UInt64(newBuf, NULL, 10);
taosMemoryFree(newBuf);
convBuf[len] = 0;
*(uint64_t *)output = taosStr2UInt64(convBuf, NULL, 10);
} else {
GET_TYPED_DATA(*(uint64_t *)output, uint64_t, inputType, input);
}
@ -867,17 +877,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
case TSDB_DATA_TYPE_FLOAT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(float *)output = taosStr2Float(varDataVal(input), NULL);
memcpy(buf, varDataVal(input), varDataLen(input));
buf[varDataLen(input)] = 0;
*(float *)output = taosStr2Float(buf, NULL);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
newBuf[len] = 0;
*(float *)output = taosStr2Float(newBuf, NULL);
taosMemoryFree(newBuf);
convBuf[len] = 0;
*(float *)output = taosStr2Float(convBuf, NULL);
} else {
GET_TYPED_DATA(*(float *)output, float, inputType, input);
}
@ -885,17 +895,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
case TSDB_DATA_TYPE_DOUBLE: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(double *)output = taosStr2Double(varDataVal(input), NULL);
memcpy(buf, varDataVal(input), varDataLen(input));
buf[varDataLen(input)] = 0;
*(double *)output = taosStr2Double(buf, NULL);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
newBuf[len] = 0;
*(double *)output = taosStr2Double(newBuf, NULL);
taosMemoryFree(newBuf);
convBuf[len] = 0;
*(double *)output = taosStr2Double(convBuf, NULL);
} else {
GET_TYPED_DATA(*(double *)output, double, inputType, input);
}
@ -903,17 +913,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
case TSDB_DATA_TYPE_BOOL: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(bool *)output = taosStr2Int8(varDataVal(input), NULL, 10);
memcpy(buf, varDataVal(input), varDataLen(input));
buf[varDataLen(input)] = 0;
*(bool *)output = taosStr2Int8(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
newBuf[len] = 0;
*(bool *)output = taosStr2Int8(newBuf, NULL, 10);
taosMemoryFree(newBuf);
convBuf[len] = 0;
*(bool *)output = taosStr2Int8(convBuf, NULL, 10);
} else {
GET_TYPED_DATA(*(bool *)output, bool, inputType, input);
}
@ -937,29 +947,27 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
case TSDB_DATA_TYPE_BINARY: {
if (inputType == TSDB_DATA_TYPE_BOOL) {
// NOTE: sprintf will append '\0' at the end of string
int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), *(int8_t *)input ? "true" : "false");
varDataSetLen(output, len);
} else if (inputType == TSDB_DATA_TYPE_BINARY) {
int32_t len = TMIN(varDataLen(input), outputLen - VARSTR_HEADER_SIZE);
len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input));
memcpy(varDataVal(output), varDataVal(input), len);
varDataSetLen(output, len);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
char *newBuf = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) {
taosMemoryFree(newBuf);
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
len = TMIN(len, outputLen - VARSTR_HEADER_SIZE);
memcpy(varDataVal(output), newBuf, len);
memcpy(varDataVal(output), convBuf, len);
varDataSetLen(output, len);
taosMemoryFree(newBuf);
} else {
char tmp[400] = {0};
NUM_TO_STRING(inputType, input, sizeof(tmp), tmp);
int32_t len = (int32_t)strlen(tmp);
NUM_TO_STRING(inputType, input, sizeof(buf), buf);
int32_t len = (int32_t)strlen(buf);
len = (outputLen - VARSTR_HEADER_SIZE) > len ? len : (outputLen - VARSTR_HEADER_SIZE);
memcpy(varDataVal(output), tmp, len);
memcpy(varDataVal(output), buf, len);
varDataSetLen(output, len);
}
break;
@ -972,14 +980,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false" );
bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
if (!ret) {
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
varDataSetLen(output, len);
} else if (inputType == TSDB_DATA_TYPE_BINARY) {
len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen;
bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
if (!ret) {
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
varDataSetLen(output, len);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
@ -987,38 +998,39 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
memcpy(output, input, len + VARSTR_HEADER_SIZE);
varDataSetLen(output, len);
} else {
char tmp[400] = {0};
NUM_TO_STRING(inputType, input, sizeof(tmp), tmp);
len = (int32_t)strlen(tmp);
NUM_TO_STRING(inputType, input, sizeof(buf), buf);
len = (int32_t)strlen(buf);
len = outputCharLen > len ? len : outputCharLen;
bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
if (!ret) {
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
varDataSetLen(output, len);
}
//for constant conversion, need to set proper length of pOutput description
if (len < outputLen) {
pOutput->columnData->info.bytes = len + VARSTR_HEADER_SIZE;
}
break;
}
default: {
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
goto _end;
}
}
colDataAppend(pOutput->columnData, i, output, false);
if (IS_VAR_DATA_TYPE(outputType)) {
output += varDataTLen(output);
} else {
output += tDataTypes[outputType].bytes;
}
}
pOutput->numOfRows = pInput->numOfRows;
taosMemoryFree(outputBuf);
return TSDB_CODE_SUCCESS;
_end:
taosMemoryFree(output);
taosMemoryFree(convBuf);
return code;
}
int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
@ -1400,8 +1412,6 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
} else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { /* unix timestamp or ts column*/
GET_TYPED_DATA(timeVal[k], int64_t, type, input[k]);
if (type == TSDB_DATA_TYPE_TIMESTAMP) {
int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 :
(timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000);
int64_t timeValSec = timeVal[k] / factor;
if (timeValSec < 1000000000) {
timeVal[k] = timeValSec;
@ -1992,7 +2002,7 @@ int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *
}
}
if (hasNull) {
if (hasNull || (count == 0)) {
colDataAppendNULL(pOutputData, 0);
} else {
if (IS_SIGNED_NUMERIC_TYPE(type)) {

136
source/libs/scalar/src/sclvector.c
View File

@ -281,7 +281,7 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t r
if (taosParseTime(buf, &value, strlen(buf), pOut->columnData->info.precision, tsDaylight) != TSDB_CODE_SUCCESS) {
value = 0;
}
colDataAppendInt64(pOut->columnData, rowIndex, &value);
}
@ -300,29 +300,29 @@ static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowI
*overflow = 0;
}
}
switch (pOut->columnData->info.type) {
case TSDB_DATA_TYPE_TINYINT: {
int8_t value = (int8_t)taosStr2Int8(buf, NULL, 10);
colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value);
break;
}
}
case TSDB_DATA_TYPE_SMALLINT: {
int16_t value = (int16_t)taosStr2Int16(buf, NULL, 10);
colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value);
break;
}
}
case TSDB_DATA_TYPE_INT: {
int32_t value = (int32_t)taosStr2Int32(buf, NULL, 10);
colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value);
break;
}
}
case TSDB_DATA_TYPE_BIGINT: {
int64_t value = (int64_t)taosStr2Int64(buf, NULL, 10);
colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value);
break;
}
}
}
}
@ -347,22 +347,22 @@ static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t ro
uint8_t value = (uint8_t)taosStr2UInt8(buf, NULL, 10);
colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value);
break;
}
}
case TSDB_DATA_TYPE_USMALLINT: {
uint16_t value = (uint16_t)taosStr2UInt16(buf, NULL, 10);
colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value);
break;
}
}
case TSDB_DATA_TYPE_UINT: {
uint32_t value = (uint32_t)taosStr2UInt32(buf, NULL, 10);
colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value);
break;
}
}
case TSDB_DATA_TYPE_UBIGINT: {
uint64_t value = (uint64_t)taosStr2UInt64(buf, NULL, 10);
colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value);
break;
}
}
}
}
@ -372,7 +372,7 @@ static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIn
colDataAppendFloat(pOut->columnData, rowIndex, &value);
return;
}
double value = taosStr2Double(buf, NULL);
colDataAppendDouble(pOut->columnData, rowIndex, &value);
}
@ -488,11 +488,11 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
tmp[len] = 0;
}
}
(*func)(tmp, pOut, i, overflow);
taosMemoryFreeClear(tmp);
}
return TSDB_CODE_SUCCESS;
}
@ -529,24 +529,24 @@ void* ncharTobinary(void *buf){ // todo need to remove , if tobinary
bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData,
void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft, bool *freeRight){
if(optr == OP_TYPE_JSON_CONTAINS) {
if (optr == OP_TYPE_JSON_CONTAINS) {
return true;
}
if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){
if (typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON) {
return true;
}
if(typeLeft == TSDB_DATA_TYPE_JSON){
if(tTagIsJson(*pLeftData)){
if (typeLeft == TSDB_DATA_TYPE_JSON) {
if (tTagIsJson(*pLeftData)) {
terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
return false;
}
typeLeft = **pLeftData;
(*pLeftData) ++;
}
if(typeRight == TSDB_DATA_TYPE_JSON){
if(tTagIsJson(*pLeftData)){
if (typeRight == TSDB_DATA_TYPE_JSON) {
if (tTagIsJson(*pLeftData)) {
terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
return false;
}
@ -554,71 +554,71 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
(*pRightData) ++;
}
if(optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH){
if(typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY){
if (optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH) {
if (typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY) {
return false;
}
}
// if types can not comparable
if((IS_NUMERIC_TYPE(typeLeft) && !IS_NUMERIC_TYPE(typeRight)) ||
(IS_NUMERIC_TYPE(typeRight) && !IS_NUMERIC_TYPE(typeLeft)) ||
(IS_VAR_DATA_TYPE(typeLeft) && !IS_VAR_DATA_TYPE(typeRight)) ||
(IS_VAR_DATA_TYPE(typeRight) && !IS_VAR_DATA_TYPE(typeLeft)) ||
((typeLeft == TSDB_DATA_TYPE_BOOL) && (typeRight != TSDB_DATA_TYPE_BOOL)) ||
((typeRight == TSDB_DATA_TYPE_BOOL) && (typeLeft != TSDB_DATA_TYPE_BOOL)))
if ((IS_NUMERIC_TYPE(typeLeft) && !IS_NUMERIC_TYPE(typeRight)) ||
(IS_NUMERIC_TYPE(typeRight) && !IS_NUMERIC_TYPE(typeLeft)) ||
(IS_VAR_DATA_TYPE(typeLeft) && !IS_VAR_DATA_TYPE(typeRight)) ||
(IS_VAR_DATA_TYPE(typeRight) && !IS_VAR_DATA_TYPE(typeLeft)) ||
((typeLeft == TSDB_DATA_TYPE_BOOL) && (typeRight != TSDB_DATA_TYPE_BOOL)) ||
((typeRight == TSDB_DATA_TYPE_BOOL) && (typeLeft != TSDB_DATA_TYPE_BOOL)))
return false;
if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){
if (typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL) {
*isNull = true;
return true;
}
int8_t type = vectorGetConvertType(typeLeft, typeRight);
if(type == 0) {
if (type == 0) {
*fp = filterGetCompFunc(typeLeft, optr);
return true;
}
*fp = filterGetCompFunc(type, optr);
if(IS_NUMERIC_TYPE(type)){
if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
if (IS_NUMERIC_TYPE(type)) {
if (typeLeft == TSDB_DATA_TYPE_NCHAR) {
ASSERT(0);
// convertNcharToDouble(*pLeftData, pLeftOut);
// *pLeftData = pLeftOut;
} else if(typeLeft == TSDB_DATA_TYPE_BINARY) {
} else if (typeLeft == TSDB_DATA_TYPE_BINARY) {
ASSERT(0);
// convertBinaryToDouble(*pLeftData, pLeftOut);
// *pLeftData = pLeftOut;
} else if(typeLeft != type) {
} else if (typeLeft != type) {
convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
*pLeftData = pLeftOut;
}
if(typeRight == TSDB_DATA_TYPE_NCHAR) {
if (typeRight == TSDB_DATA_TYPE_NCHAR) {
ASSERT(0);
// convertNcharToDouble(*pRightData, pRightOut);
// *pRightData = pRightOut;
} else if(typeRight == TSDB_DATA_TYPE_BINARY) {
} else if (typeRight == TSDB_DATA_TYPE_BINARY) {
ASSERT(0);
// convertBinaryToDouble(*pRightData, pRightOut);
// *pRightData = pRightOut;
} else if(typeRight != type) {
} else if (typeRight != type) {
convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
*pRightData = pRightOut;
}
}else if(type == TSDB_DATA_TYPE_BINARY){
if(typeLeft == TSDB_DATA_TYPE_NCHAR){
} else if (type == TSDB_DATA_TYPE_BINARY) {
if (typeLeft == TSDB_DATA_TYPE_NCHAR) {
*pLeftData = ncharTobinary(*pLeftData);
*freeLeft = true;
}
if(typeRight == TSDB_DATA_TYPE_NCHAR){
if (typeRight == TSDB_DATA_TYPE_NCHAR) {
*pRightData = ncharTobinary(*pRightData);
*freeRight = true;
}
}else{
} else {
ASSERT(0);
}
@ -636,7 +636,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1
colDataAppendNULL(pOutputCol, i);
continue;
}
int64_t value = 0;
GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i));
int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value);
@ -653,7 +653,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1
colDataAppendNULL(pOutputCol, i);
continue;
}
uint64_t value = 0;
GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i));
int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value);
@ -670,7 +670,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1
colDataAppendNULL(pOutputCol, i);
continue;
}
double value = 0;
GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i));
int32_t len = sprintf(varDataVal(tmp), "%lf", value);
@ -698,7 +698,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
sclError("input column is NULL, hashFilter %p", pIn->pHashFilter);
return TSDB_CODE_APP_ERROR;
}
int16_t inType = pInputCol->info.type;
int16_t outType = pOutputCol->info.type;
@ -710,14 +710,14 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
ASSERT(1 == pIn->numOfRows);
pOut->numOfRows = 0;
if (IS_SIGNED_NUMERIC_TYPE(outType)) {
int64_t minValue = tDataTypes[outType].minValue;
int64_t maxValue = tDataTypes[outType].maxValue;
double value = 0;
GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0));
if (value > maxValue) {
*overflow = 1;
return TSDB_CODE_SUCCESS;
@ -730,10 +730,10 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
} else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) {
uint64_t minValue = (uint64_t)tDataTypes[outType].minValue;
uint64_t maxValue = (uint64_t)tDataTypes[outType].maxValue;
double value = 0;
GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0));
if (value > maxValue) {
*overflow = 1;
return TSDB_CODE_SUCCESS;
@ -820,7 +820,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
colDataAppendNULL(pOutputCol, i);
continue;
}
uint8_t value = 0;
GET_TYPED_DATA(value, uint8_t, inType, colDataGetData(pInputCol, i));
colDataAppendInt8(pOutputCol, i, (int8_t *)&value);
@ -833,7 +833,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
colDataAppendNULL(pOutputCol, i);
continue;
}
uint16_t value = 0;
GET_TYPED_DATA(value, uint16_t, inType, colDataGetData(pInputCol, i));
colDataAppendInt16(pOutputCol, i, (int16_t *)&value);
@ -846,7 +846,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
colDataAppendNULL(pOutputCol, i);
continue;
}
uint32_t value = 0;
GET_TYPED_DATA(value, uint32_t, inType, colDataGetData(pInputCol, i));
colDataAppendInt32(pOutputCol, i, (int32_t *)&value);
@ -859,7 +859,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
colDataAppendNULL(pOutputCol, i);
continue;
}
uint64_t value = 0;
GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i));
colDataAppendInt64(pOutputCol, i, (int64_t*)&value);
@ -872,12 +872,12 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
colDataAppendNULL(pOutputCol, i);
continue;
}
float value = 0;
GET_TYPED_DATA(value, float, inType, colDataGetData(pInputCol, i));
colDataAppendFloat(pOutputCol, i, (float*)&value);
}
break;
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
for (int32_t i = 0; i < pIn->numOfRows; ++i) {
@ -885,14 +885,14 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
colDataAppendNULL(pOutputCol, i);
continue;
}
double value = 0;
GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i));
colDataAppendDouble(pOutputCol, i, (double*)&value);
}
break;
break;
}
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
return vectorConvertToVarData(pIn, pOut, inType, outType);
}
@ -966,10 +966,10 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p
return TSDB_CODE_SUCCESS;
}
SScalarParam *param1 = NULL, *paramOut1 = NULL;
SScalarParam *param1 = NULL, *paramOut1 = NULL;
SScalarParam *param2 = NULL, *paramOut2 = NULL;
int32_t code = 0;
if (leftType < rightType) {
param1 = pLeft;
param2 = pRight;
@ -993,7 +993,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p
return code;
}
}
if (type != GET_PARAM_TYPE(param2)) {
code = vectorConvertScalarParam(param2, paramOut2, type);
if (code) {
@ -1136,7 +1136,7 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
}
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i);
}
}
}
} else {
double *output = (double *)pOutputCol->pData;
_getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type);
@ -1198,7 +1198,7 @@ static void vectorMathTsSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pR
}
*output = taosTimeAdd(getVectorBigintValueFnLeft(pLeftCol->pData, i), -getVectorBigintValueFnRight(pRightCol->pData, 0),
pRightCol->info.scale, pRightCol->info.precision);
}
}
}
@ -1706,7 +1706,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
int32_t lType = GET_PARAM_TYPE(pLeft);
int32_t rType = GET_PARAM_TYPE(pRight);
__compar_fn_t fp = NULL;
if (lType == rType) {
fp = filterGetCompFunc(lType, optr);
} else {
@ -1736,9 +1736,9 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
}
void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) {
SScalarParam pLeftOut = {0};
SScalarParam pLeftOut = {0};
SScalarParam pRightOut = {0};
SScalarParam *param1 = NULL;
SScalarParam *param1 = NULL;
SScalarParam *param2 = NULL;
if (SCL_NO_NEED_CONVERT_COMPARISION(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
@ -1762,7 +1762,7 @@ void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
vectorCompareImpl(param1, param2, pOut, _ord, optr);
sclFreeParam(&pLeftOut);
sclFreeParam(&pRightOut);
sclFreeParam(&pRightOut);
}
void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {

2
source/libs/tdb/src/db/tdbBtree.c
View File

@ -1485,7 +1485,7 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
if (dropOfp) {
int ret = 0;
SPgno pgno = *(SPgno *)(pCell + nLocal - sizeof(SPgno));
int nLeft = nPayload - nLocal + sizeof(SPgno);
int nLeft = nPayload - nLocal + sizeof(SPgno) + nHeader;
SPage *ofp;
int bytes;

11
source/libs/tdb/test/tdbExOVFLTest.cpp
View File

@ -238,8 +238,8 @@ TEST(TdbOVFLPagesTest, TbGetTest) {
}
}
TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) {
// TEST(TdbOVFLPagesTest, TbDeleteTest) {
// TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) {
TEST(TdbOVFLPagesTest, TbDeleteTest) {
int ret = 0;
taosRemoveDir("tdb");
@ -267,7 +267,8 @@ TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) {
tdbBegin(pEnv, &txn);
// generate value payload
char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
// char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
char val[((4083 - 4 - 3 - 2) + 1) * 2]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
int valLen = sizeof(val) / sizeof(val[0]);
generateBigVal(val, valLen);
@ -340,8 +341,8 @@ tdbBegin(pEnv, &txn);
tdbTxnClose(&txn);
}
TEST(tdb_test, DISABLED_simple_insert1) {
// TEST(tdb_test, simple_insert1) {
// TEST(tdb_test, DISABLED_simple_insert1) {
TEST(tdb_test, simple_insert1) {
int ret;
TDB *pEnv;
TTB *pDb;

14
source/libs/wal/src/walMeta.c
View File

@ -150,7 +150,6 @@ int walCheckAndRepairMeta(SWal* pWal) {
const char* idxPattern = "^[0-9]+.idx$";
regex_t logRegPattern;
regex_t idxRegPattern;
SArray* actualLog = taosArrayInit(8, sizeof(SWalFileInfo));
regcomp(&logRegPattern, logPattern, REG_EXTENDED);
regcomp(&idxRegPattern, idxPattern, REG_EXTENDED);
@ -163,6 +162,8 @@ int walCheckAndRepairMeta(SWal* pWal) {
return -1;
}
SArray* actualLog = taosArrayInit(8, sizeof(SWalFileInfo));
// scan log files and build new meta
TdDirEntryPtr pDirEntry;
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
@ -492,18 +493,21 @@ int walSaveMeta(SWal* pWal) {
int metaVer = walFindCurMetaVer(pWal);
char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer + 1, fnameStr);
TdFilePtr pMataFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE);
if (pMataFile == NULL) {
TdFilePtr pMetaFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE);
if (pMetaFile == NULL) {
return -1;
}
char* serialized = walMetaSerialize(pWal);
int len = strlen(serialized);
if (len != taosWriteFile(pMataFile, serialized, len)) {
if (len != taosWriteFile(pMetaFile, serialized, len)) {
// TODO:clean file
taosCloseFile(&pMetaFile);
taosRemoveFile(fnameStr);
return -1;
}
taosCloseFile(&pMataFile);
taosCloseFile(&pMetaFile);
// delete old file
if (metaVer > -1) {
walBuildMetaName(pWal, metaVer, fnameStr);

4
source/os/CMakeLists.txt
View File

@ -52,3 +52,7 @@ else()
os PUBLIC dl m rt
)
endif()
IF (JEMALLOC_ENABLED)
target_link_libraries(os PUBLIC -ljemalloc)
ENDIF ()

14
source/os/src/osSocket.c
View File

@ -267,21 +267,29 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void
return -1;
}
#ifdef WINDOWS
#ifdef TCP_KEEPCNT
if (level == SOL_SOCKET && optname == TCP_KEEPCNT) {
return 0;
}
#endif
#ifdef TCP_KEEPIDLE
if (level == SOL_TCP && optname == TCP_KEEPIDLE) {
return 0;
}
#endif
#ifdef TCP_KEEPINTVL
if (level == SOL_TCP && optname == TCP_KEEPINTVL) {
return 0;
}
#endif
#ifdef TCP_KEEPCNT
if (level == SOL_TCP && optname == TCP_KEEPCNT) {
return 0;
}
#endif
return setsockopt(pSocket->fd, level, optname, optval, optlen);
#else
@ -601,26 +609,32 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) {
#ifndef __APPLE__
// all fails on macosx
#ifdef TCP_KEEPCNT
int32_t probes = 3;
if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) {
// printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(&pSocket);
return -1;
}
#endif
#ifdef TCP_KEEPIDLE
int32_t alivetime = 10;
if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) {
// printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(&pSocket);
return -1;
}
#endif
#ifdef TCP_KEEPINTVL
int32_t interval = 3;
if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) {
// printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(&pSocket);
return -1;
}
#endif
#endif // __APPLE__
int32_t nodelay = 1;

4
source/util/src/terror.c
View File

@ -633,6 +633,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message")
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch")
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed")
// TDLite
TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS, "Invalid TDLite open flags")
TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_DIR, "Invalid TDLite open directory")
#ifdef TAOS_ERROR_C
};
#endif

60
tests/system-test/0-others/show.py Normal file
View File

@ -0,0 +1,60 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from util.log import *
from util.cases import *
from util.sql import *
import subprocess
from util.common import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.dbname = 'db'
self.ins_param_list = ['dnodes','mnodes','qnodes','cluster','functions','users','grants','topics','subscriptions','streams']
self.perf_param = ['apps','connections','consumers','queries','transactions']
self.perf_param_list = ['apps','connections','consumers','queries','trans']
def ins_check(self):
for param in self.ins_param_list:
tdSql.query(f'show {param}')
show_result = tdSql.queryResult
tdSql.query(f'select * from information_schema.ins_{param}')
select_result = tdSql.queryResult
tdSql.checkEqual(show_result,select_result)
def perf_check(self):
for param in range(len(self.perf_param_list)):
tdSql.query(f'show {self.perf_param[param]}')
if len(tdSql.queryResult) != 0:
show_result = tdSql.queryResult[0][0]
tdSql.query(f'select * from performance_schema.perf_{self.perf_param_list[param]}')
select_result = tdSql.queryResult[0][0]
tdSql.checkEqual(show_result,select_result)
else :
continue
def run(self):
tdSql.prepare()
self.ins_check()
self.perf_check()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

2
tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py
View File

@ -132,7 +132,7 @@ class TDTestCase:
tdSql.error("create mnode on dnode 2")
tdSql.query("select * from information_schema.ins_dnodes;")
print(tdSql.queryResult)
clusterComCheck.checkDnodes(dnodeNumbers)
clusterComCheck.checkDnodes(dnodeNumbers, 60)
# create database and stable
clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])

8
tests/system-test/6-cluster/clusterCommonCheck.py
View File

@ -37,10 +37,10 @@ class ClusterComCheck:
tdSql.init(conn.cursor())
# tdSql.init(conn.cursor(), logSql) # output sql.txt file
def checkDnodes(self,dnodeNumbers):
def checkDnodes(self,dnodeNumbers, timeout=30):
count=0
# print(tdSql)
while count < 30:
while count < timeout:
tdSql.query("select * from information_schema.ins_dnodes")
# tdLog.debug(tdSql.queryResult)
status=0
@ -50,14 +50,14 @@ class ClusterComCheck:
tdLog.info(status)
if status == dnodeNumbers:
tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within 30s! " %dnodeNumbers)
tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within %ds! " % (dnodeNumbers, count))
return True
count+=1
time.sleep(1)
else:
tdSql.query("select * from information_schema.ins_dnodes")
tdLog.debug(tdSql.queryResult)
tdLog.exit("it find cluster with %d dnodes but check that there dnodes are not ready within 30s ! "%dnodeNumbers)
tdLog.exit("it find cluster with %d dnodes but check that there dnodes are not ready within %ds ! "% (dnodeNumbers, timeout))
def checkDbRows(self,dbNumbers):
dbNumbers=int(dbNumbers)

2
tests/system-test/fulltest.sh
View File

@ -37,7 +37,7 @@ python3 ./test.py -f 1-insert/table_param_ttl.py -R
python3 ./test.py -f 1-insert/update_data_muti_rows.py
python3 ./test.py -f 1-insert/db_tb_name_check.py
python3 ./test.py -f 1-insert/database_pre_suf.py
python3 ./test.py -f 0-others/show.py
python3 ./test.py -f 2-query/abs.py
python3 ./test.py -f 2-query/abs.py -R
python3 ./test.py -f 2-query/and_or_for_byte.py

3
tools/shell/CMakeLists.txt
View File

@ -28,7 +28,4 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
add_dependencies(taosd jemalloc)
ENDIF ()
SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos)

10
tools/shell/src/shellAuto.c
View File

@ -1133,6 +1133,10 @@ bool firstMatchCommand(TAOS* con, SShellCmd* cmd) {
// print to screen
printScreen(con, cmd, match);
#ifdef WINDOWS
printf("\r");
shellShowOnScreen(cmd);
#endif
freeCommand(input);
taosMemoryFree(input);
return true;
@ -1195,6 +1199,10 @@ bool nextMatchCommand(TAOS* con, SShellCmd* cmd, SWords* firstMatch) {
// print to screen
printScreen(con, cmd, match);
#ifdef WINDOWS
printf("\r");
shellShowOnScreen(cmd);
#endif
// free
if (input->source) {
@ -1268,7 +1276,7 @@ char* lastWord(char* p) {
char* p2 = strrchr(p, ',');
if (p1 && p2) {
return p1 > p2 ? p1 : p2 + 1;
return p1 > p2 ? p1 + 1 : p2 + 1;
} else if (p1) {
return p1 + 1;
} else if (p2) {

3
tools/shell/src/shellCommand.c
View File

@ -417,8 +417,9 @@ char taosGetConsoleChar() {
static char mbStr[5];
static unsigned long bufLen = 0;
static uint16_t bufIndex = 0, mbStrIndex = 0, mbStrLen = 0;
CONSOLE_READCONSOLE_CONTROL inputControl={ sizeof(CONSOLE_READCONSOLE_CONTROL), 0, 1<<TAB_KEY, 0 };
while (bufLen == 0) {
ReadConsoleW(console, buf, SHELL_INPUT_MAX_COMMAND_SIZE, &bufLen, NULL);
ReadConsoleW(console, buf, SHELL_INPUT_MAX_COMMAND_SIZE, &bufLen, &inputControl);
if (bufLen > 0 && buf[0] == 0) bufLen = 0;
bufIndex = 0;
}

2
tools/shell/src/shellEngine.c
View File

@ -1112,7 +1112,9 @@ int32_t shellExecute() {
#ifdef WEBSOCKET
if (!shell.args.restful && !shell.args.cloud) {
#endif
#ifndef WINDOWS
printfIntroduction();
#endif
shellGetGrantInfo();
#ifdef WEBSOCKET
}