xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/vnode_refact1

This commit is contained in:
Hongze Cheng 2022-04-20 12:29:57 +00:00
parent fbddf859c8 3ecf6df9e3
commit c8acaa77bb
36 changed files with 1302 additions and 415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
include/common/tdatablock.h
View File

@ -54,12 +54,34 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
} while (0)
#define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1)
#define colDataSetNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] = -1)
#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)])
#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
// SColumnInfoData, rowNumber
#define colDataGetData(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) \
: colDataGetNumData(p1_, r_))
static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON){
if(colDataIsNull_var(pColumnInfoData, row)){
return true;
}
char *data = colDataGetVarData(pColumnInfoData, row);
return (*data == TSDB_DATA_TYPE_NULL);
}
if (!pColumnInfoData->hasNull) {
return false;
}
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
return pColumnInfoData->varmeta.offset[row] == -1;
if (pColumnInfoData->info.type== TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) {
return colDataIsNull_var(pColumnInfoData, row);
} else {
if (pColumnInfoData->nullbitmap == NULL) {
return false;
@ -86,7 +108,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
}
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
return pColumnInfoData->varmeta.offset[row] == -1;
return colDataIsNull_var(pColumnInfoData, row);
} else {
if (pColumnInfoData->nullbitmap == NULL) {
return false;
@ -96,17 +118,10 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
}
}
#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
// SColumnInfoData, rowNumber
#define colDataGetData(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \
: ((p1_)->pData + ((r_) * (p1_)->info.bytes)))
static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) {
// There is a placehold for each NULL value of binary or nchar type.
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type.
colDataSetNull_var(pColumnInfoData, currentRow); // it is a null value of VAR type.
} else {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow);
}
@ -117,7 +132,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin
static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) {
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
for (int32_t i = start; i < start + nRows; ++i) {
pColumnInfoData->varmeta.offset[i] = -1; // it is a null value of VAR type.
colDataSetNull_var(pColumnInfoData,i); // it is a null value of VAR type.
}
} else {
for (int32_t i = start; i < start + nRows; ++i) {
@ -265,3 +280,4 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da
#endif
#endif /*_TD_COMMON_EP_H_*/

5
include/common/tdataformat.h
View File

@ -502,7 +502,7 @@ typedef struct {
#define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
#define kvRowLen(r) (*(TDRowLenT *)(r))
#define kvRowLen(r) (*(uint16_t *)(r))
#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t)))
#define kvRowSetLen(r, len) kvRowLen(r) = (len)
#define kvRowSetNCols(r, n) kvRowNCols(r) = (n)
@ -608,7 +608,7 @@ void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
void tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) {
static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) {
if (pBuilder->nCols >= pBuilder->tCols) {
pBuilder->tCols *= 2;
SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
@ -621,7 +621,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co
pBuilder->nCols++;
int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
if (tlen > pBuilder->alloc - pBuilder->size) {
while (tlen > pBuilder->alloc - pBuilder->size) {
pBuilder->alloc *= 2;

4
include/common/ttypes.h
View File

@ -49,7 +49,7 @@ typedef struct {
#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v))
#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
@ -252,7 +252,7 @@ typedef struct tDataTypeDescriptor {
int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
} tDataTypeDescriptor;
extern tDataTypeDescriptor tDataTypes[15];
extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX];
bool isValidDataType(int32_t type);

1
include/libs/scalar/scalar.h
View File

@ -76,6 +76,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
/* Time related functions */
int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

1
include/util/taoserror.h
View File

@ -609,6 +609,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_INTER_SLIDING_UNIT TAOS_DEF_ERROR_CODE(0, 0x2630)
#define TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG TAOS_DEF_ERROR_CODE(0, 0x2631)
#define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x2632)
#define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG TAOS_DEF_ERROR_CODE(0, 0x2633)
//planner
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)

2
include/util/tcompare.h
View File

@ -105,6 +105,8 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t compareJsonContainsKey(const void *pLeft, const void *pRight);
__compar_fn_t getComparFunc(int32_t type, int32_t optr);
__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
int32_t doCompare(const char *a, const char *b, int32_t type, size_t size);

2
include/util/tdef.h
View File

@ -273,6 +273,8 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_TAGS 128
#define TSDB_MAX_TAG_CONDITIONS 1024
#define TSDB_MAX_JSON_TAG_LEN 16384
#define TSDB_AUTH_LEN 16
#define TSDB_PASSWORD_LEN 32
#define TSDB_USET_PASSWORD_LEN 129

1
include/util/tjson.h
View File

@ -76,6 +76,7 @@ char* tjsonToString(const SJson* pJson);
char* tjsonToUnformattedString(const SJson* pJson);
SJson* tjsonParse(const char* pStr);
bool tjsonValidateJson(const char* pJson);
#ifdef __cplusplus
}

2
include/util/tutil.h
View File

@ -26,8 +26,6 @@ extern "C" {
#endif
int32_t strdequote(char *src);
int32_t strndequote(char *dst, const char *z, int32_t len);
int32_t strRmquote(char *z, int32_t len);
size_t strtrim(char *src);
char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote);
char **strsplit(char *src, const char *delim, int32_t *num);

70
source/client/src/clientImpl.c
View File

@ -725,6 +725,76 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
}
if (type == TSDB_DATA_TYPE_JSON) {
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pResultInfo->convertBuf[i] = p;
int32_t len = 0;
SResultColumn* pCol = &pResultInfo->pCol[i];
for (int32_t j = 0; j < numOfRows; ++j) {
if (pCol->offset[j] != -1) {
char* pStart = pCol->offset[j] + pCol->pData;
int32_t jsonInnerType = *pStart;
char *jsonInnerData = pStart + CHAR_BYTES;
char dst[TSDB_MAX_JSON_TAG_LEN] = {0};
if(jsonInnerType == TSDB_DATA_TYPE_NULL){
sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
varDataSetLen(dst, strlen(varDataVal(dst)));
}else if(jsonInnerType == TSDB_DATA_TYPE_JSON){
int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst));
if (length <= 0) {
tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData));
length = 0;
}
varDataSetLen(dst, length);
}else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"
*(char*)varDataVal(dst) = '\"';
int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst) + CHAR_BYTES);
if (length <= 0) {
tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData));
length = 0;
}
varDataSetLen(dst, length + CHAR_BYTES*2);
*(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"';
}else if(jsonInnerType == TSDB_DATA_TYPE_DOUBLE){
double jsonVd = *(double*)(jsonInnerData);
sprintf(varDataVal(dst), "%.9lf", jsonVd);
varDataSetLen(dst, strlen(varDataVal(dst)));
}else if(jsonInnerType == TSDB_DATA_TYPE_BIGINT){
int64_t jsonVd = *(int64_t*)(jsonInnerData);
sprintf(varDataVal(dst), "%" PRId64, jsonVd);
varDataSetLen(dst, strlen(varDataVal(dst)));
}else if(jsonInnerType == TSDB_DATA_TYPE_BOOL){
sprintf(varDataVal(dst), "%s", (*((char *)jsonInnerData) == 1) ? "true" : "false");
varDataSetLen(dst, strlen(varDataVal(dst)));
}else {
ASSERT(0);
}
if(len + varDataTLen(dst) > colLength[i]){
p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst));
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pResultInfo->convertBuf[i] = p;
}
p = pResultInfo->convertBuf[i] + len;
memcpy(p, dst, varDataTLen(dst));
pCol->offset[j] = len;
len += varDataTLen(dst);
}
}
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
}
}
return TSDB_CODE_SUCCESS;

21
source/common/src/tdatablock.c
View File

@ -113,14 +113,27 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
int32_t type = pColumnInfoData->info.type;
if (IS_VAR_DATA_TYPE(type)) {
int32_t dataLen = varDataTLen(pData);
if(type == TSDB_DATA_TYPE_JSON) {
if(*pData == TSDB_DATA_TYPE_NULL) {
dataLen = 0;
}else if(*pData == TSDB_DATA_TYPE_NCHAR) {
dataLen = varDataTLen(pData+CHAR_BYTES);
}else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
dataLen = LONG_BYTES;
}else if(*pData == TSDB_DATA_TYPE_BOOL) {
dataLen = CHAR_BYTES;
}
dataLen += CHAR_BYTES;
}
SVarColAttr* pAttr = &pColumnInfoData->varmeta;
if (pAttr->allocLen < pAttr->length + varDataTLen(pData)) {
if (pAttr->allocLen < pAttr->length + dataLen) {
uint32_t newSize = pAttr->allocLen;
if (newSize == 0) {
newSize = 8;
}
while (newSize < pAttr->length + varDataTLen(pData)) {
while (newSize < pAttr->length + dataLen) {
newSize = newSize * 1.5;
}
@ -136,8 +149,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
uint32_t len = pColumnInfoData->varmeta.length;
pColumnInfoData->varmeta.offset[currentRow] = len;
memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData));
pColumnInfoData->varmeta.length += varDataTLen(pData);
memcpy(pColumnInfoData->pData + len, pData, dataLen);
pColumnInfoData->varmeta.length += dataLen;
} else {
memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes);
}

3
source/common/src/ttypes.c
View File

@ -379,7 +379,7 @@ static void getStatics_nchr(int8_t bitmapMode, const void *pBitmap, const void *
*maxIndex = 0;
}
tDataTypeDescriptor tDataTypes[15] = {
tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
{TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint,
@ -402,6 +402,7 @@ tDataTypeDescriptor tDataTypes[15] = {
{TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32},
{TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint,
getStatics_u64},
{TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
};
char tTokenTypeSwitcher[13] = {

2
source/common/src/tvariant.c
View File

@ -118,7 +118,7 @@ void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) {
}
case TSDB_DATA_TYPE_BINARY: {
pVar->pz = strndup(z, n);
pVar->nLen = strRmquote(pVar->pz, n);
//pVar->nLen = strRmquote(pVar->pz, n);
break;
}
case TSDB_DATA_TYPE_TIMESTAMP: {

2
source/libs/function/src/builtins.c
View File

@ -939,7 +939,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.translateFunc = translateToJson,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = NULL,
.sprocessFunc = toJsonFunction,
.finalizeFunc = NULL
}
};

2
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -1009,6 +1009,7 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
case OP_TYPE_NOT_LIKE:
case OP_TYPE_MATCH:
case OP_TYPE_NMATCH:
case OP_TYPE_JSON_CONTAINS:
case OP_TYPE_IS_NULL:
case OP_TYPE_IS_NOT_NULL:
case OP_TYPE_IS_TRUE:
@ -1027,7 +1028,6 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
bool nodesIsJsonOp(const SOperatorNode* pOp) {
switch (pOp->opType) {
case OP_TYPE_JSON_GET_VALUE:
case OP_TYPE_JSON_CONTAINS:
return true;
default:
break;

1
source/libs/parser/inc/parUtil.h
View File

@ -47,6 +47,7 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta);
int32_t getNumOfColumns(const STableMeta* pTableMeta);
int32_t getNumOfTags(const STableMeta* pTableMeta);
STableComInfo getTableInfo(const STableMeta* pTableMeta);
int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId);
int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);

1
source/libs/parser/src/parAstCreater.c
View File

@ -48,6 +48,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
}
static void trimEscape(SToken* pName) {
// todo need to deal with `ioo``ii` -> ioo`ii
if (NULL != pName && pName->n > 1 && '`' == pName->z[0]) {
pName->z += 1;
pName->n -= 2;

49
source/libs/parser/src/parInsert.c
View File

@ -397,35 +397,15 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha
return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z);
}
if (IS_NUMERIC_TYPE(type) && pToken->n == 0) {
return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z);
}
// Remove quotation marks
if (TK_NK_STRING == pToken->type) {
if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
}
// delete escape character: \\, \', \"
char delim = pToken->z[0];
int32_t cnt = 0;
int32_t j = 0;
for (uint32_t k = 1; k < pToken->n - 1; ++k) {
if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) {
tmpTokenBuf[j] = pToken->z[k + 1];
cnt++;
j++;
k++;
continue;
}
tmpTokenBuf[j] = pToken->z[k];
j++;
}
tmpTokenBuf[j] = 0;
int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW);
pToken->z = tmpTokenBuf;
pToken->n -= 2 + cnt;
pToken->n = len;
}
return TSDB_CODE_SUCCESS;
@ -603,6 +583,13 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
return func(pMsgBuf, pToken->z, pToken->n, param);
}
case TSDB_DATA_TYPE_JSON: {
if(pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z);
}
return func(pMsgBuf, pToken->z, pToken->n, param);
}
case TSDB_DATA_TYPE_TIMESTAMP: {
int64_t tmpVal;
if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) {
@ -723,8 +710,10 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo*
qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar);
}
if(pColList->numOfCols > pColList->numOfBound){
memset(&pColList->boundColumns[pColList->numOfBound], 0,
sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound));
}
return TSDB_CODE_SUCCESS;
}
@ -741,9 +730,18 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi
int8_t type = pa->schema->type;
int16_t colId = pa->schema->colId;
if(TSDB_DATA_TYPE_JSON == type){
return parseJsontoTagData(value, pa->builder, pMsgBuf, colId);
}
if (value == NULL) { // it is a null data
// tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx);
return TSDB_CODE_SUCCESS;
}
if (TSDB_DATA_TYPE_BINARY == type) {
STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len);
tdAddColToKVRow(pa->builder, colId, type, pa->buf);
tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf));
} else if (TSDB_DATA_TYPE_NCHAR == type) {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0;
@ -751,13 +749,12 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi
char buf[512] = {0};
snprintf(buf, tListLen(buf), "%s", strerror(errno));
return buildSyntaxErrMsg(pMsgBuf, buf, value);
;
}
varDataSetLen(pa->buf, output);
tdAddColToKVRow(pa->builder, colId, type, pa->buf);
tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf));
} else {
tdAddColToKVRow(pa->builder, colId, type, value);
tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]);
}
return TSDB_CODE_SUCCESS;

46
source/libs/parser/src/parTranslater.c
View File

@ -543,14 +543,18 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
}
} else if (nodesIsComparisonOp(pOp)) {
if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
TSDB_DATA_TYPE_BLOB == rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
} else {
// todo json operator
} else if (nodesIsJsonOp(pOp)){
if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes;
}
return DEAL_RES_CONTINUE;
}
@ -1813,6 +1817,17 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS
return TSDB_CODE_SUCCESS;
}
static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
SNode* pNode;
FOREACH(pNode, pStmt->pTags) {
SColumnDefNode* pCol = (SColumnDefNode*)pNode;
if(pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1){
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) {
if (NULL == pFuncs) {
return TSDB_CODE_SUCCESS;
@ -1848,6 +1863,9 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "delay", pStmt->pOptions->pDelay, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkTableTags(pCxt, pStmt);
}
return code;
}
@ -3249,17 +3267,25 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c
static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema,
SKVRowBuilder* pBuilder) {
if(pSchema->type == TSDB_DATA_TYPE_JSON){
if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
}
return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId);
}
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
SVariant var;
valueNodeToVariant(pVal, &var);
char tagVal[TSDB_MAX_TAGS_LEN] = {0};
int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true);
if (TSDB_CODE_SUCCESS == code) {
tdAddColToKVRow(pBuilder, pSchema->colId, pSchema->type, tagVal);
if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){
// todo
}else{
tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]);
}
return code;
return TSDB_CODE_SUCCESS;
}
static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta,

167
source/libs/parser/src/parUtil.c
View File

@ -14,6 +14,7 @@
*/
#include "parUtil.h"
#include "cJSON.h"
static char* getSyntaxErrFormat(int32_t errCode) {
switch (errCode) {
@ -115,6 +116,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "sliding value no larger than the interval value";
case TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL:
return "sliding value can not less than 1% of interval value";
case TSDB_CODE_PAR_ONLY_ONE_JSON_TAG:
return "Only one tag if there is a json tag";
case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory";
default:
@ -215,24 +218,178 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta) {
}
int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
// delete escape character: \\, \', \"
if (len <=0 || dlen <= 0) return 0;
char delim = src[0];
int32_t cnt = 0;
int32_t j = 0;
for (uint32_t k = 1; k < len - 1; ++k) {
if (j >= dlen) {
break;
dst[j - 1] = '\0';
return j;
}
if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) {
if (src[k] == delim && src[k + 1] == delim) { // deal with "", ''
dst[j] = src[k + 1];
cnt++;
j++;
k++;
continue;
}
if (src[k] == '\\') { // deal with escape character
if(src[k+1] == 'n'){
dst[j] = '\n';
}else if(src[k+1] == 'r'){
dst[j] = '\r';
}else if(src[k+1] == 't'){
dst[j] = '\t';
}else if(src[k+1] == '\\'){
dst[j] = '\\';
}else if(src[k+1] == '\''){
dst[j] = '\'';
}else if(src[k+1] == '"'){
dst[j] = '"';
}else if(src[k+1] == '%' || src[k+1] == '_'){
dst[j++] = src[k];
dst[j] = src[k+1];
}else{
dst[j] = src[k+1];
}
j++;
k++;
continue;
}
dst[j] = src[k];
j++;
}
dst[j] = '\0';
return j;
}
static bool isValidateTag(char *input) {
if (!input) return false;
for (size_t i = 0; i < strlen(input); ++i) {
if (isprint(input[i]) == 0) return false;
}
return true;
}
int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
// set json NULL data
uint8_t jsonNULL = TSDB_DATA_TYPE_NULL;
int jsonIndex = startColId + 1;
if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){
tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
return TSDB_CODE_SUCCESS;
}
// set json real data
cJSON *root = cJSON_Parse(json);
if (root == NULL){
return buildSyntaxErrMsg(pMsgBuf, "json parse error", json);
}
int size = cJSON_GetArraySize(root);
if(!cJSON_IsObject(root)){
return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json);
}
int retCode = 0;
char *tagKV = NULL;
SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
for(int i = 0; i < size; i++) {
cJSON* item = cJSON_GetArrayItem(root, i);
if (!item) {
qError("json inner error:%d", i);
retCode = buildSyntaxErrMsg(pMsgBuf, "json inner error", json);
goto end;
}
char *jsonKey = item->string;
if(!isValidateTag(jsonKey)){
retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey);
goto end;
}
// if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){
// tscError("json key too long error");
// retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL);
// goto end;
// }
size_t keyLen = strlen(jsonKey);
if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){
continue;
}
// key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: LONG_BYTES
tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + LONG_BYTES, 1);
if(!tagKV) {
retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto end;
}
strncpy(varDataVal(tagKV), jsonKey, keyLen);
varDataSetLen(tagKV, keyLen);
if(taosHashGetSize(keyHash) == 0){
uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type
}
taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless
if(item->type == cJSON_String){ // add json value format: type|data
char *jsonValue = item->valuestring;
int32_t valLen = (int32_t)strlen(jsonValue);
int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES;
char *tmp = taosMemoryRealloc(tagKV, totalLen);
if(!tmp) {
retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto end;
}
tagKV = tmp;
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
*valueType = TSDB_DATA_TYPE_NCHAR;
if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData),
(int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) {
qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno));
retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
goto end;
}
varDataSetLen(valueData, valLen);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen);
}else if(item->type == cJSON_Number){
if(!isfinite(item->valuedouble)){
qError("json value is invalidate");
retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json);
goto end;
}
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
*valueType = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_DOUBLE;
if(*valueType== TSDB_DATA_TYPE_DOUBLE) *((double *)valueData) = item->valuedouble;
else if(*valueType == TSDB_DATA_TYPE_BIGINT) *((int64_t *)valueData) = item->valueint;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES +LONG_BYTES);
}else if(item->type == cJSON_True || item->type == cJSON_False){
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
*valueType = TSDB_DATA_TYPE_BOOL;
*valueData = (char)(item->valueint);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES);
}else if(item->type == cJSON_NULL){
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
*valueType = TSDB_DATA_TYPE_NULL;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
}
else{
retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json);
goto end;
}
}
if(taosHashGetSize(keyHash) == 0){ // set json NULL true
tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
}
end:
taosMemoryFree(tagKV);
taosHashCleanup(keyHash);
cJSON_Delete(root);
return retCode;
}

2
source/libs/scalar/inc/filterInt.h
View File

@ -308,7 +308,6 @@ struct SFilterInfo {
#define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri)))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type)
#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc)
#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
@ -321,7 +320,6 @@ struct SFilterInfo {
#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
#define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)

4
source/libs/scalar/inc/sclvector.h
View File

@ -56,6 +56,8 @@ static FORCE_INLINE double getVectorDoubleValue_BOOL(void *src, int32_t index) {
return (double)*((bool *)src + index);
}
double getVectorDoubleValue_JSON(void *src, int32_t index);
static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) {
_getDoubleValue_fn_t p = NULL;
if (srcType == TSDB_DATA_TYPE_TINYINT) {
@ -80,6 +82,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType)
p = getVectorDoubleValue_DOUBLE;
} else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) {
p = getVectorDoubleValue_BIGINT;
} else if (srcType == TSDB_DATA_TYPE_JSON) {
p = getVectorDoubleValue_JSON;
} else if (srcType == TSDB_DATA_TYPE_BOOL) {
p = getVectorDoubleValue_BOOL;
} else {

56
source/libs/scalar/src/filter.c
View File

@ -57,28 +57,24 @@ OptrStr gOptrStr[] = {
{OP_TYPE_IS_NOT_UNKNOWN, "not unknown"},
// json operator
{OP_TYPE_JSON_GET_VALUE, "json get"},
{OP_TYPE_JSON_GET_VALUE, "->"},
{OP_TYPE_JSON_CONTAINS, "json contains"}
};
bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
int32_t result = cfunc(maxv, minr);
//if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result >= 0;
}
bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
int32_t result = cfunc(maxv, minr);
//if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result > 0;
}
bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
int32_t result = cfunc(minv, maxr);
//if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result <= 0;
}
bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
int32_t result = cfunc(minv, maxr);
//if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result < 0;
}
bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
@ -170,7 +166,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val
compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch,
compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8,
compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch
compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey
};
int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
@ -222,6 +218,11 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
}
}
if (optr == OP_TYPE_JSON_CONTAINS && type == TSDB_DATA_TYPE_JSON) {
return 28;
}
switch (type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break;
@ -1049,6 +1050,8 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) {
cell = cell->pNext;
}
colDataDestroy(out.columnData);
taosMemoryFree(out.columnData);
} else {
filterAddFieldFromNode(info, node->pRight, &right);
@ -1778,7 +1781,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
fi->data = taosMemoryCalloc(1, bytes);
} else if (type != TSDB_DATA_TYPE_JSON){
} else{
if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE
/*
fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes);
@ -1791,11 +1794,8 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
} else {
fi->data = taosMemoryCalloc(1, sizeof(int64_t));
}
} else{ // type == TSDB_DATA_TYPE_JSON
// fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc)
}
if(type != TSDB_DATA_TYPE_JSON) {
if (dType->type == type) {
assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type);
} else {
@ -1815,7 +1815,8 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
}
memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes);
}
colDataDestroy(out.columnData);
taosMemoryFree(out.columnData);
}
// match/nmatch for nchar type need convert from ucs4 to mbs
@ -2556,11 +2557,7 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
if (unit->right.type == FLD_TYPE_VALUE) {
if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant
info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit);
}else{
info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
}
} else {
info->cunits[i].valData = NULL;
}
@ -2886,18 +2883,8 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows,
for (int32_t i = 0; i < numOfRows; ++i) {
uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i);
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
if (!colData){ // for json->'key' is null
(*p)[i] = 1;
}else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null
colData = POINTER_SHIFT(colData, CHAR_BYTES);
(*p)[i] = colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL);
}else{
(*p)[i] = 0;
}
}else{
(*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
}
if ((*p)[i] == 0) {
all = false;
}
@ -2921,19 +2908,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i);
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
if (!colData) { // for json->'key' is not null
(*p)[i] = 0;
}else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null
colData = POINTER_SHIFT(colData, CHAR_BYTES);
(*p)[i] = !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL);
}else{ // for json->'key' is not null
(*p)[i] = 1;
}
}else {
(*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
}
if ((*p)[i] == 0) {
all = false;
}
@ -3566,6 +3541,11 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
if (OP_TYPE_JSON_CONTAINS == node->opType) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) {
SNode *t = node->pLeft;
node->pLeft = node->pRight;

30
source/libs/scalar/src/scalar.c
View File

@ -132,6 +132,7 @@ void sclFreeRes(SHashObj *res) {
void sclFreeParam(SScalarParam *param) {
if (param->columnData != NULL) {
colDataDestroy(param->columnData);
taosMemoryFree(param->columnData);
}
if (param->pHashFilter != NULL) {
@ -612,13 +613,30 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
return DEAL_RES_ERROR;
}
SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId);
int32_t index = -1;
for(int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) {
SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, i);
if (pb->info.blockId == target->dataBlockId) {
index = i;
break;
}
}
if (index == -1) {
sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR;
}
SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index);
if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR;
}
// if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
@ -628,14 +646,8 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
return DEAL_RES_ERROR;
}
for (int32_t i = 0; i < res->numOfRows; ++i) {
if (colDataIsNull(res->columnData, res->numOfRows, i, NULL)) {
colDataAppend(col, i, NULL, true);
} else {
char *p = colDataGetData(res->columnData, i);
colDataAppend(col, i, p, false);
}
}
colDataAssign(col, res->columnData, res->numOfRows);
block->info.rows = res->numOfRows;
sclFreeParam(res);
taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);

54
source/libs/scalar/src/sclfunc.c
View File

@ -4,6 +4,7 @@
#include "ttime.h"
#include "sclInt.h"
#include "sclvector.h"
#include "tjson.h"
typedef float (*_float_fn)(float);
typedef double (*_double_fn)(double);
@ -872,6 +873,59 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
return TSDB_CODE_SUCCESS;
}
int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int32_t type = GET_PARAM_TYPE(pInput);
if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) {
return TSDB_CODE_FAILED;
}
if (inputNum != 1) {
return TSDB_CODE_FAILED;
}
char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0];
char *tmp = taosMemoryCalloc(pInput[0].columnData->info.bytes + 1, 1);
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
if (colDataIsNull_s(pInput[0].columnData, i)) {
colDataAppendNULL(pOutput->columnData, i);
continue;
}
if(type == TSDB_DATA_TYPE_NCHAR){
if (varDataTLen(input) > TSDB_MAX_JSON_TAG_LEN){
colDataAppendNULL(pOutput->columnData, i);
continue;
}
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), tmp);
if (len < 0) {
colDataAppendNULL(pOutput->columnData, i);
continue;
}
tmp[len] = 0;
}else{
if (varDataLen(input) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
colDataAppendNULL(pOutput->columnData, i);
continue;
}
memcpy(tmp, varDataVal(input), varDataLen(input));
tmp[varDataTLen(input)] = 0;
}
if(!tjsonValidateJson(tmp)){
colDataAppendNULL(pOutput->columnData, i);
continue;
}
colDataAppend(pOutput->columnData, i, input, false);
input += varDataTLen(input);
}
taosMemoryFree(tmp);
pOutput->numOfRows = pInput->numOfRows;
return TSDB_CODE_SUCCESS;
}
int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int32_t type = GET_PARAM_TYPE(&pInput[0]);
int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]);

511
source/libs/scalar/src/sclvector.c
View File

@ -25,6 +25,78 @@
#include "tdatablock.h"
#include "ttypes.h"
#define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData))
#define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData))
void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
switch (outType) {
case TSDB_DATA_TYPE_BOOL: {
GET_TYPED_DATA(*((bool *)outData), bool, inType, inData);
break;
}
case TSDB_DATA_TYPE_TINYINT: {
GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_INT: {
GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: {
GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UTINYINT: {
GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_USMALLINT: {
GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UINT: {
GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UBIGINT: {
GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
GET_TYPED_DATA(*((float *)outData), float, inType, inData);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
GET_TYPED_DATA(*((double *)outData), double, inType, inData);
break;
}
default:{
ASSERT(0);
}
}
}
void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
char *tmp = taosMemoryMalloc(varDataTLen(inData));
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
if (len < 0) {
sclError("castConvert taosUcs4ToMbs error 1");
}
tmp[len] = 0;
ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
double value = strtod(tmp, NULL);
*((double *)outData) = value;
taosMemoryFreeClear(tmp);
}
typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index);
int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) {
@ -61,6 +133,20 @@ int64_t getVectorBigintValue_BOOL(void *src, int32_t index) {
return (int64_t)*((bool *)src + index);
}
int64_t getVectorBigintValue_JSON(void *src, int32_t index){
ASSERT(!colDataIsNull_var(((SColumnInfoData*)src), index));
char *data = colDataGetVarData((SColumnInfoData*)src, index);
double out = 0;
if (*data == TSDB_DATA_TYPE_NULL){
return 0;
} else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY
convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
} else {
convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
}
return (int64_t)out;
}
_getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
_getBigintValue_fn_t p = NULL;
if(srcType==TSDB_DATA_TYPE_TINYINT) {
@ -87,6 +173,8 @@ _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
p = getVectorBigintValue_BIGINT;
}else if(srcType==TSDB_DATA_TYPE_BOOL) {
p = getVectorBigintValue_BOOL;
}else if(srcType==TSDB_DATA_TYPE_JSON) {
p = getVectorBigintValue_JSON;
}else {
assert(0);
}
@ -213,7 +301,8 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
func = varToUnsigned;
} else if (IS_FLOAT_TYPE(outType)) {
func = varToFloat;
} else if (outType == TSDB_DATA_TYPE_NCHAR) {
} else if (outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar
ASSERT(inType == TSDB_DATA_TYPE_VARCHAR);
func = varToNchar;
vton = true;
} else {
@ -228,14 +317,28 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
continue;
}
char* data = colDataGetData(pIn->columnData, i);
char* data = colDataGetVarData(pIn->columnData, i);
int32_t convertType = inType;
if(inType == TSDB_DATA_TYPE_JSON){
if(*data == TSDB_DATA_TYPE_NULL) {
colDataAppendNULL(pOut->columnData, i);
continue;
}
else if(*data == TSDB_DATA_TYPE_NCHAR) {
data += CHAR_BYTES;
convertType = TSDB_DATA_TYPE_NCHAR;
} else {
convertNumberToNumber(data+CHAR_BYTES, colDataGetNumData(pOut->columnData, i), *data, outType);
continue;
}
}
if (vton) {
memcpy(tmp, data, varDataTLen(data));
} else {
if (TSDB_DATA_TYPE_VARCHAR == inType) {
if (TSDB_DATA_TYPE_VARCHAR == convertType) {
memcpy(tmp, varDataVal(data), varDataLen(data));
tmp[varDataLen(data)] = 0;
} else {
} else if (TSDB_DATA_TYPE_NCHAR == convertType){
ASSERT(varDataLen(data) <= bufSize);
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp);
@ -256,6 +359,66 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
return TSDB_CODE_SUCCESS;
}
double getVectorDoubleValue_JSON(void *src, int32_t index){
char *data = colDataGetVarData((SColumnInfoData*)src, index);
double out = 0;
if (*data == TSDB_DATA_TYPE_NULL){
return out;
} else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY
convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
} else {
convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
}
return out;
}
void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){
if(optr == OP_TYPE_JSON_CONTAINS) {
return;
}
if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){
return;
}
if(typeLeft == TSDB_DATA_TYPE_JSON){
typeLeft = **pLeftData;
(*pLeftData) ++;
}
if(typeRight == TSDB_DATA_TYPE_JSON){
typeRight = **pRightData;
(*pRightData) ++;
}
if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){
*isNull = true;
return;
}
int8_t type = vectorGetConvertType(typeLeft, typeRight);
if(type == 0) {
*fp = filterGetCompFunc(typeLeft, optr);
return;
}
*fp = filterGetCompFunc(type, optr);
if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
convertStringToDouble(*pLeftData, pLeftOut, typeLeft, type);
*pLeftData = pLeftOut;
} else if(typeLeft != type) {
convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
*pLeftData = pLeftOut;
}
if(typeRight == TSDB_DATA_TYPE_NCHAR) {
convertStringToDouble(*pRightData, pRightOut, typeRight, type);
*pRightData = pRightOut;
} else if(typeRight != type) {
convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
*pRightData = pRightOut;
}
}
// TODO opt performance
int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
SColumnInfoData* pInputCol = pIn->columnData;
@ -422,24 +585,24 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
}
int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = {
/* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG VARB JSON DECI BLOB */
/* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB */
/*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 7, 0, 0, 0,
/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 0, 0, 0,
/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 0, 0, 0,
/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 7, 0, 0, 0,
/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 7, 0, 0, 0,
/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 7, 0, 0, 0,
/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 7, 0, 0, 0,
/*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 0, 7, 0, 0,
/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0,
/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0,
/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0,
/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 0, 7, 0, 0,
/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0,
/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0,
/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 7, 7, 7, 0, 0, 0, 0,
/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 7, 0, 0, 0,
/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 0, 7, 0, 0,
/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0,
/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 7, 0, 0, 0,
/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 7, 0, 0, 0,
/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7, 0, 0, 0,
/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0,
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0,
/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0,
/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0,
/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0,
/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
@ -530,7 +693,7 @@ enum {
static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) {
SColumnInfoData* pCol = pParam->columnData;
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) {
pDest->numOfRows = pParam->numOfRows;
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
@ -564,11 +727,12 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, 0);
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
*output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+ getVectorDoubleValueFnRight(RIGHT_COL, 0);
}
}
}
@ -583,11 +747,11 @@ static void vectorMathBigintAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, 0);
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, 0);
}
}
}
@ -614,6 +778,50 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) {
}
}
char *getJsonValue(char *json, char *key){ //todo
int16_t cols = kvRowNCols(json);
for (int i = 0; i < cols; ++i) {
SColIdx *pColIdx = kvRowColIdxAt(json, i);
char *data = kvRowColVal(json, pColIdx);
if(i == 0){
if(*data == TSDB_DATA_TYPE_NULL) {
return NULL;
}
continue;
}
if(memcmp(key, data, varDataTLen(data)) == 0){
return data + varDataTLen(data);
}
}
return NULL;
}
void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
SColumnInfoData *pOutputCol = pOut->columnData;
int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1;
pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
char *pRightData = colDataGetVarData(pRight->columnData, 0);
for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (colDataIsNull_var(pLeft->columnData, i)) {
colDataSetNull_var(pOutputCol, i);
pOutputCol->hasNull = true;
continue;
}
char *pLeftData = colDataGetVarData(pLeft->columnData, i);
char *value = getJsonValue(pLeftData, pRightData);
if (!value) {
colDataSetNull_var(pOutputCol, i);
pOutputCol->hasNull = true;
continue;
}
colDataAppend(pOutputCol, i, value, false);
}
}
void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
SColumnInfoData *pOutputCol = pOut->columnData;
@ -636,17 +844,12 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i);
}
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) {
vectorMathBigintAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) {
@ -659,17 +862,12 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i);
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i);
}
}
} else if (pLeft->numOfRows == 1) {
vectorMathAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) {
@ -692,11 +890,12 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = (getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, 0)) * factor;
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
*output = (getVectorDoubleValueFnLeft(LEFT_COL, i)
- getVectorDoubleValueFnRight(RIGHT_COL, 0)) * factor;
}
}
}
@ -711,11 +910,11 @@ static void vectorMathBigintSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = (getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, 0)) * factor;
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
*output = (getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, 0)) * factor;
}
}
}
@ -740,17 +939,12 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, i);
}
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) {
vectorMathBigintSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i);
} else if (pRight->numOfRows == 1) {
@ -763,17 +957,12 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i);
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i);
}
}
} else if (pLeft->numOfRows == 1) {
vectorMathSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i);
} else if (pRight->numOfRows == 1) {
@ -796,11 +985,12 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, 0);
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
*output = getVectorDoubleValueFnLeft(LEFT_COL, i)
* getVectorDoubleValueFnRight(RIGHT_COL, 0);
}
}
}
@ -822,17 +1012,13 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
double *output = (double *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, i);
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
*output = getVectorDoubleValueFnLeft(LEFT_COL, i)
* getVectorDoubleValueFnRight(RIGHT_COL, i);
}
}
} else if (pLeft->numOfRows == 1) {
vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) {
@ -860,39 +1046,37 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
double *output = (double *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) { // check for the 0 value
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, i);
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
*output = getVectorDoubleValueFnLeft(LEFT_COL, i)
/getVectorDoubleValueFnRight(RIGHT_COL, i);
}
}
} else if (pLeft->numOfRows == 1) {
if (colDataIsNull_f(pLeftCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value
if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
} else {
for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, 0) / getVectorDoubleValueFnRight(pRightCol->pData, i);
if (colDataIsNull_s(pRightCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = pRightCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pRightCol->nullbitmap, BitmapLen(pRight->numOfRows));
*output = getVectorDoubleValueFnLeft(LEFT_COL, 0)
/ getVectorDoubleValueFnRight(RIGHT_COL, i);
}
}
} else if (pRight->numOfRows == 1) {
if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value
if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
} else {
for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, 0);
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows));
*output = getVectorDoubleValueFnLeft(LEFT_COL, i)
/ getVectorDoubleValueFnRight(RIGHT_COL, 0);
}
}
}
@ -920,13 +1104,13 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_f(pLeftCol->nullbitmap, i) || colDataIsNull_f(pRightCol->nullbitmap, i)) {
if (colDataIsNull_s(pLeftCol, i) || colDataIsNull_s(pRightCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue;
}
double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i);
double rx = getVectorDoubleValueFnRight(pRightCol->pData, i);
double lx = getVectorDoubleValueFnLeft(LEFT_COL, i);
double rx = getVectorDoubleValueFnRight(RIGHT_COL, i);
if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) {
colDataAppendNULL(pOutputCol, i);
continue;
@ -935,17 +1119,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
*output = lx - ((int64_t)(lx / rx)) * rx;
}
} else if (pLeft->numOfRows == 1) {
double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, 0);
if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value
double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0);
if (colDataIsNull_s(pLeftCol, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
} else {
for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
if (colDataIsNull_f(pRightCol->nullbitmap, i)) {
if (colDataIsNull_s(pRightCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue;
}
double rx = getVectorDoubleValueFnRight(pRightCol->pData, i);
double rx = getVectorDoubleValueFnRight(RIGHT_COL, i);
if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) {
colDataAppendNULL(pOutputCol, i);
continue;
@ -955,17 +1139,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
}
}
} else if (pRight->numOfRows == 1) {
double rx = getVectorDoubleValueFnRight(pRightCol->pData, 0);
if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value
double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0);
if (colDataIsNull_s(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
} else {
for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
if (colDataIsNull_f(pLeftCol->nullbitmap, i)) {
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue;
}
double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i);
double lx = getVectorDoubleValueFnLeft(LEFT_COL, i);
if (isnan(lx) || isinf(lx)) {
colDataAppendNULL(pOutputCol, i);
continue;
@ -995,12 +1179,11 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
double *output = (double *)pOutputCol->pData;
for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) {
*output = - getVectorDoubleValueFnLeft(pLeftCol->pData, i);
if (colDataIsNull_s(pLeft->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows));
*output = - getVectorDoubleValueFnLeft(LEFT_COL, i);
}
doReleaseVec(pLeftCol, leftConvert);
@ -1063,15 +1246,15 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh
double *output = (double *)pOutputCol->pData;
if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value
if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) & getVectorBigintValueFnRight(pRightCol->pData, 0);
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
*output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, 0);
}
}
}
@ -1093,17 +1276,12 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
int64_t *output = (int64_t *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) & getVectorBigintValueFnRight(pRightCol->pData, i);
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] & pRightCol->nullbitmap[j];
*output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, i);
}
}
} else if (pLeft->numOfRows == 1) {
vectorBitAndHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) {
@ -1120,16 +1298,16 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight
int64_t *output = (int64_t *)pOutputCol->pData;
if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value
if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
int64_t rx = getVectorBigintValueFnRight(pRightCol->pData, 0);
int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0);
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) | rx;
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
*output = getVectorBigintValueFnLeft(LEFT_COL, i) | rx;
}
}
}
@ -1151,15 +1329,11 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
int64_t *output = (int64_t *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) | getVectorBigintValueFnRight(pRightCol->pData, i);
}
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorBigintValueFnLeft(LEFT_COL, i) | getVectorBigintValueFnRight(RIGHT_COL, i);
}
} else if (pLeft->numOfRows == 1) {
vectorBitOrHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
@ -1181,6 +1355,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
if (pRight->pHashFilter != NULL) {
for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (colDataIsNull_s(pLeft->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue;
}
@ -1194,35 +1369,62 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore
}
char *pLeftData = colDataGetData(pLeft->columnData, i);
char *pRightData = colDataGetData(pRight->columnData, i);
int64_t leftOut = 0;
int64_t rightOut = 0;
bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore
}
bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
}
} else if (pRight->numOfRows == 1) {
char *pRightData = colDataGetData(pRight->columnData, 0);
ASSERT(pLeft->pHashFilter == NULL);
for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (colDataIsNull_s(pLeft->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue;
}
char *pLeftData = colDataGetData(pLeft->columnData, i);
char *pRightData = colDataGetData(pRight->columnData, 0);
int64_t leftOut = 0;
int64_t rightOut = 0;
bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore
}
bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
}
} else if (pLeft->numOfRows == 1) {
char *pLeftData = colDataGetData(pLeft->columnData, 0);
for (; i >= 0 && i < pRight->numOfRows; i += step) {
if (colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue;
}
char *pLeftData = colDataGetData(pLeft->columnData, 0);
char *pRightData = colDataGetData(pLeft->columnData, i);
int64_t leftOut = 0;
int64_t rightOut = 0;
bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore
}
bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
}
@ -1303,6 +1505,10 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu
vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH);
}
void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_JSON_CONTAINS);
}
void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0;
@ -1371,8 +1577,13 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
return vectorBitOr;
case OP_TYPE_IS_TRUE:
return vectorIsTrue;
case OP_TYPE_JSON_GET_VALUE:
return vectorJsonArrow;
case OP_TYPE_JSON_CONTAINS:
return vectorJsonContains;
default:
assert(0);
return NULL;
}
}

3
source/libs/scalar/test/scalar/CMakeLists.txt
View File

@ -8,11 +8,12 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(scalarTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
scalarTest
PUBLIC os util common gtest qcom function nodes scalar
PUBLIC os util common gtest qcom function nodes scalar parser
)
TARGET_INCLUDE_DIRECTORIES(
scalarTest
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/"
PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc"
PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
)

261
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -38,6 +38,7 @@
#include "scalar.h"
#include "nodes.h"
#include "tlog.h"
#include "parUtil.h"
#define _DEBUG_PRINT_ 0
@ -92,6 +93,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s
blockDataEnsureCapacity(res, rows);
*dataBlockId = taosArrayGetSize(pBlockList) - 1;
res->info.blockId = *dataBlockId;
*slotId = 0;
} else {
SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList);
@ -909,6 +911,264 @@ TEST(constantTest, greater_and_lower) {
nodesDestroyNode(res);
}
void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){
char keyVar[32] = {0};
memcpy(varDataVal(keyVar), key, strlen(key));
varDataLen(keyVar) = strlen(key);
SNode *pLeft = NULL, *pRight = NULL;
scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, keyVar);
scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, varDataLen(json), 1, json);
scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
}
void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData){
int32_t resType = TSDB_DATA_TYPE_NULL;
if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI ||
opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
resType = TSDB_DATA_TYPE_DOUBLE;
}else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
resType = TSDB_DATA_TYPE_BIGINT;
}else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL ||
opType == OP_TYPE_LOWER_THAN || opType == OP_TYPE_LOWER_EQUAL ||
opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL ||
opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE ||
opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH ||
opType == OP_TYPE_NMATCH){
resType = TSDB_DATA_TYPE_BOOL;
}
SNode *right = NULL;
scltMakeValueNode(&right, rightType, rightData);
scltMakeOpNode(opNode, opType, resType, *opNode, right);
SColumnInfo colInfo = createColumnInfo(1, resType, tDataTypes[resType].bytes);
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, 1, &colInfo);
scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode);
}
void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType){
SArray *blockList = taosArrayInit(2, POINTER_BYTES);
SSDataBlock *src = NULL;
SNode *opNode = NULL;
makeJsonArrow(&src, &opNode, json, (char*)key);
taosArrayPush(blockList, &src);
makeOperator(&opNode, blockList, opType, rightType, rightData);
int32_t code = scalarCalculate(opNode, blockList, NULL);
ASSERT_EQ(code, 0);
SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList);
ASSERT_EQ(res->info.rows, 1);
SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
if(colDataIsNull_f(column->nullbitmap, 0)){
ASSERT_EQ(DBL_MAX, exceptValue);
printf("result:NULL\n");
}else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV ||
opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
double tmp = *((double *)colDataGetData(column, 0));
ASSERT_TRUE(tmp == exceptValue);
printf("result:%lf\n", tmp);
}else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue);
printf("result:%ld\n", *((int64_t *)colDataGetData(column, 0)));
}else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN ||
opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL ||
opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE ||
opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){
ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue);
printf("result:%d\n", *((bool *)colDataGetData(column, 0)));
}
taosArrayDestroyEx(blockList, scltFreeDataBlock);
nodesDestroyNode(opNode);
}
TEST(columnTest, json_column_arith_op) {
scltInitLogFile();
char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
SKVRowBuilder kvRowBuilder;
tdInitKVRowBuilder(&kvRowBuilder);
parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
const int32_t len = 8;
EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV,
OP_TYPE_MOD, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR};
int32_t input[len] = {1, 8, 2, 2, 3, 0, -4, 9};
printf("--------------------json int---------------------\n");
char *key = "k1";
double eRes[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
}
printf("--------------------json string---------------------\n");
key = "k2";
double eRes1[len] = {1.0, -8, 0, 0, 0, 0, 0, 9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
}
printf("---------------------json null--------------------\n");
key = "k3";
double eRes2[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
}
printf("---------------------json bool--------------------\n");
key = "k4";
double eRes3[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
}
printf("----------------------json double-------------------\n");
key = "k5";
double eRes4[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
}
printf("---------------------json not exist--------------------\n");
key = "k10";
double eRes5[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
}
}
void *prepareNchar(char* rightData){
int32_t len = 0;
int32_t inputLen = strlen(rightData);
char* t = (char*)taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
taosMbsToUcs4(rightData, inputLen, (TdUcs4*) varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len);
varDataSetLen(t, len);
return t;
}
TEST(columnTest, json_column_logic_op) {
scltInitLogFile();
char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}";
SKVRowBuilder kvRowBuilder;
tdInitKVRowBuilder(&kvRowBuilder);
parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
const int32_t len = 9;
const int32_t len1 = 4;
EOperatorType op[len+len1] = {OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, OP_TYPE_NOT_EQUAL,
OP_TYPE_IS_NULL, OP_TYPE_IS_NOT_NULL, OP_TYPE_IS_TRUE, OP_TYPE_LIKE, OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH};
int32_t input[len] = {1, 8, 2, 2, 3, 0, 0, 0, 0};
char *inputNchar[len1] = {"hell_", "hel%", "hell", "llll"};
printf("--------------------json int---------------------\n");
char *key = "k1";
bool eRes[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json string---------------------\n");
key = "k2";
bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json null---------------------\n");
key = "k3";
double eRes2[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json bool---------------------\n");
key = "k4";
bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json double---------------------\n");
key = "k5";
bool eRes4[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json double---------------------\n");
key = "k6";
bool eRes5[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, true};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i]);
taosMemoryFree(rightData);
}
printf("---------------------json not exist--------------------\n");
key = "k10";
double eRes10[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i]);
taosMemoryFree(rightData);
}
}
TEST(columnTest, smallint_value_add_int_column) {
scltInitLogFile();
@ -3097,3 +3357,4 @@ int main(int argc, char** argv) {
}
#pragma GCC diagnostic pop

5
source/util/src/tcompare.c
View File

@ -222,6 +222,11 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) {
return compareLenPrefixedWStr(pRight, pLeft);
}
int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) {
if(pLeft) return 0;
return 1;
}
/*
* Compare two strings
* TSDB_MATCH: Match

34
source/util/src/tjson.c
View File

@ -276,3 +276,37 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void
}
SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); }
bool tjsonValidateJson(const char *jIn) {
if (!jIn){
return false;
}
// set json real data
cJSON *root = cJSON_Parse(jIn);
if (root == NULL){
return false;
}
if(!cJSON_IsObject(root)){
return false;
}
int size = cJSON_GetArraySize(root);
for(int i = 0; i < size; i++) {
cJSON* item = cJSON_GetArrayItem(root, i);
if (!item) {
return false;
}
char* jsonKey = item->string;
if (!jsonKey) return false;
for (size_t j = 0; j < strlen(jsonKey); ++i) {
if (isprint(jsonKey[i]) == 0) return false;
}
if (item->type == cJSON_Object || item->type == cJSON_Array) {
return false;
}
}
return true;
}

59
source/util/src/tutil.c
View File

@ -47,65 +47,6 @@ int32_t strdequote(char *z) {
return j + 1; // only one quote, do nothing
}
int32_t strRmquote(char *z, int32_t len) {
// delete escape character: \\, \', \"
char delim = z[0];
if (delim != '\'' && delim != '\"') {
return len;
}
int32_t cnt = 0;
int32_t j = 0;
for (uint32_t k = 1; k < len - 1; ++k) {
if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) {
if (z[k] == '\\' && z[k + 1] == '_') {
// match '_' self
} else {
z[j] = z[k + 1];
cnt++;
j++;
k++;
continue;
}
}
z[j] = z[k];
j++;
}
z[j] = 0;
return len - 2 - cnt;
}
int32_t strndequote(char *dst, const char *z, int32_t len) {
assert(dst != NULL);
if (z == NULL || len == 0) {
return 0;
}
int32_t quote = z[0];
int32_t i = 1, j = 0;
while (z[i] != 0) {
if (z[i] == quote) {
if (z[i + 1] == quote) {
dst[j++] = (char)quote;
i++;
} else {
dst[j++] = 0;
return (j - 1);
}
} else {
dst[j++] = z[i];
}
i++;
}
return j + 1; // only one quote, do nothing
}
size_t strtrim(char *z) {
int32_t i = 0;
int32_t j = 0;

3
tests/script/jenkins/basic.txt
View File

@ -93,4 +93,7 @@
# --- sma
./test.sh -f tsim/sma/tsmaCreateInsertData.sim
# --- valgrind
./test.sh -f tsim/valgrind/checkError.sim -v
#======================b1-end===============

38
tests/script/sh/checkValgrind.sh Executable file
View File

@ -0,0 +1,38 @@
#!/bin/bash
set +e
#set -x
NODE_NAME=
while getopts "n:" arg
do
case $arg in
n)
NODE_NAME=$OPTARG
;;
?)
echo "unkown argument"
;;
esac
done
SCRIPT_DIR=`dirname $0`
cd $SCRIPT_DIR/../
SCRIPT_DIR=`pwd`
IN_TDINTERNAL="community"
if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then
cd ../../..
else
cd ../../
fi
TAOS_DIR=`pwd`
LOG_DIR=$TAOS_DIR/sim/$NODE_NAME/log
#CFG_DIR=$TAOS_DIR/sim/$NODE_NAME/cfg
#echo ---- $LOG_DIR
errors=`grep "ERROR SUMMARY:" ${LOG_DIR}/valgrind-taosd-*.log | cut -d ' ' -f 2,3,4,5 | tr -d "\n"`
echo $errors

83
tests/script/tsim/valgrind/checkError.sim Normal file
View File

@ -0,0 +1,83 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
#system sh/deploy.sh -n dnode2 -i 2
#system sh/deploy.sh -n dnode3 -i 3
#system sh/deploy.sh -n dnode4 -i 4
#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
#system sh/exec.sh -n dnode2 -s start
#system sh/exec.sh -n dnode3 -s start
#system sh/exec.sh -n dnode4 -s start
sleep 2000
#$loop_cnt = 0
#check_dnode_ready:
# $loop_cnt = $loop_cnt + 1
# sleep 200
# if $loop_cnt == 10 then
# print ====> dnode not ready!
# return -1
# endi
#sql show dnodes
#print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
#print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
#print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
#print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6]
#if $data[0][0] != 1 then
# return -1
#endi
#if $data[0][4] != ready then
# goto check_dnode_ready
#endi
#
##sql connect
#sql create dnode $hostname port 7200
#sql create dnode $hostname port 7300
#sql create dnode $hostname port 7400
#
#$loop_cnt = 0
#check_dnode_ready_1:
#$loop_cnt = $loop_cnt + 1
#sleep 200
#if $loop_cnt == 10 then
# print ====> dnodes not ready!
# return -1
#endi
#sql show dnodes
#print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
#print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
#print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
#print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6]
#if $data[0][4] != ready then
# goto check_dnode_ready_1
#endi
#if $data[1][4] != ready then
# goto check_dnode_ready_1
#endi
#if $data[2][4] != ready then
# goto check_dnode_ready_1
#endi
#if $data[3][4] != ready then
# goto check_dnode_ready_1
#endi
#=========== please add any actions above =================
print ====> stop all dondes to output valgrind log file
system sh/exec.sh -n dnode1 -s stop -x SIGINT
print ====> start to check if there are ERRORS in vagrind log file for each dnode
# -n : dnode[x] be check
system_content sh/checkValgrind.sh -n dnode1
print cmd return result----> [ $system_content ]
if $system_content == @ERROR SUMMARY: 0 errors@ then
return 0
endi
$null=
if $system_content == $null then
return 0
endi
return -1

6
tests/tsim/src/simExe.c
View File

@ -441,6 +441,12 @@ void simStoreSystemContentResult(SScript *script, char *filename) {
// if ((fd = fopen(filename, "r")) != NULL) {
if ((pFile = taosOpenFile(filename, TD_FILE_READ)) != NULL) {
taosReadFile(pFile, script->system_ret_content, MAX_SYSTEM_RESULT_LEN - 1);
int32_t len = strlen(script->system_ret_content);
for (int32_t i = 0; i < len; ++i) {
if (script->system_ret_content[i] == '\n' || script->system_ret_content[i] == '\r') {
script->system_ret_content[i] = 0;
}
}
taosCloseFile(&pFile);
char rmCmd[MAX_FILE_NAME_LEN] = {0};
sprintf(rmCmd, "rm -f %s", filename);

74
tools/shell/src/shellEngine.c
View File

@ -224,63 +224,27 @@ int32_t shellRunCommand(TAOS *con, char *command) {
}
}
bool esc = false;
char quote = 0, *cmd = command, *p = command;
char quote = 0, *cmd = command;
for (char c = *command++; c != 0; c = *command++) {
if (esc) {
switch (c) {
case 'n':
c = '\n';
break;
case 'r':
c = '\r';
break;
case 't':
c = '\t';
break;
case 'G':
*p++ = '\\';
break;
case '\'':
case '"':
if (quote) {
*p++ = '\\';
}
break;
}
*p++ = c;
esc = false;
if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) {
command ++;
continue;
}
if (c == '\\') {
if (quote != 0 && (*command == '_' || *command == '\\')) {
// DO nothing
} else {
esc = true;
continue;
}
}
if (quote == c) {
quote = 0;
} else if (quote == 0 && (c == '\'' || c == '"')) {
} else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) {
quote = c;
}
*p++ = c;
if (c == ';' && quote == 0) {
c = *p;
*p = 0;
} else if (c == ';' && quote == 0) {
c = *command;
*command = 0;
if (shellRunSingleCommand(con, cmd) < 0) {
return -1;
}
*p = c;
p = cmd;
*command = c;
cmd = command;
}
}
*p = 0;
return shellRunSingleCommand(con, cmd);
}
@ -574,19 +538,23 @@ static void shellPrintNChar(const char *str, int length, int width) {
while (pos < length) {
TdWchar wc;
int bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
if (bytes == 0) {
if (bytes <= 0) {
break;
}
pos += bytes;
if (pos > length) {
if (pos + bytes > length) {
break;
}
int w = 0;
#ifdef WINDOWS
int w = bytes;
w = bytes;
#else
int w = taosWcharWidth(wc);
if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
w = bytes;
}else{
w = taosWcharWidth(wc);
}
#endif
pos += bytes;
if (w <= 0) {
continue;
}
@ -680,6 +648,7 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:
shellPrintNChar(val, length, width);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
@ -792,7 +761,8 @@ static int calcColWidth(TAOS_FIELD *field, int precision) {
return TMAX(field->bytes, width);
}
case TSDB_DATA_TYPE_NCHAR: {
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:{
int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
if (bytes > tsMaxBinaryDisplayWidth) {
return TMAX(tsMaxBinaryDisplayWidth, width);