xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/row_refact

This commit is contained in:
Hongze Cheng 2022-06-06 01:24:46 +00:00
parent f81d102747 10e4824194
commit 461cd037bf
31 changed files with 776 additions and 436 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/cmake.options
View File

@ -146,6 +146,6 @@ option(
option(
BUILD_WITH_INVERTEDINDEX
"If use invertedIndex"
OFF
ON
)

38
include/client/taos.h
View File

@ -103,10 +103,10 @@ typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
typedef struct TAOS_MULTI_BIND {
int buffer_type;
void *buffer;
void * buffer;
uintptr_t buffer_length;
int32_t *length;
char *is_null;
int32_t * length;
char * is_null;
int num;
} TAOS_MULTI_BIND;
@ -130,7 +130,7 @@ DLL_EXPORT void taos_cleanup(void);
DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
DLL_EXPORT setConfRet taos_set_config(const char *config);
DLL_EXPORT int taos_init(void);
DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
const char *db, int dbLen, uint16_t port);
DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
@ -147,17 +147,17 @@ DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name
DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt);
DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt);
DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt);
DLL_EXPORT char * taos_stmt_errstr(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt);
@ -179,11 +179,11 @@ DLL_EXPORT bool taos_is_update_query(TAOS_RES *res);
DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData);
DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
DLL_EXPORT int * taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql);
DLL_EXPORT void taos_reset_current_db(TAOS *taos);
DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res);
DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res);
DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res);
DLL_EXPORT const char *taos_get_server_info(TAOS *taos);
@ -204,7 +204,7 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
#endif
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
/* --------------------------TMQ INTERFACE------------------------------- */
@ -229,7 +229,7 @@ DLL_EXPORT tmq_list_t *tmq_list_new();
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *);
DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *);
DLL_EXPORT char ** tmq_list_to_c_array(const tmq_list_t *);
DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
@ -240,7 +240,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
DLL_EXPORT TAOS_RES * tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets);
DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param);
@ -260,7 +260,7 @@ enum tmq_conf_res_t {
typedef enum tmq_conf_res_t tmq_conf_res_t;
DLL_EXPORT tmq_conf_t *tmq_conf_new();
DLL_EXPORT tmq_conf_t * tmq_conf_new();
DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf);
DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);

12
include/common/tdatablock.h
View File

@ -71,20 +71,14 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
#define colDataGetData(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_))
static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) {
if (colDataIsNull_var(pColumnInfoData, row)) {
return true;
}
char* data = colDataGetVarData(pColumnInfoData, row);
return (*data == TSDB_DATA_TYPE_NULL);
}
#define IS_JSON_NULL(type,data) ((type) == TSDB_DATA_TYPE_JSON && *(data) == TSDB_DATA_TYPE_NULL)
static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
if (!pColumnInfoData->hasNull) {
return false;
}
if (pColumnInfoData->info.type == TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) {
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
return colDataIsNull_var(pColumnInfoData, row);
} else {
if (pColumnInfoData->nullbitmap == NULL) {

3
include/libs/index/index.h
View File

@ -193,8 +193,9 @@ void indexInit();
/* index filter */
typedef struct SIndexMetaArg {
void* metaHandle;
void* metaEx;
void* idx;
void* ivtIdx;
uint64_t suid;
} SIndexMetaArg;

2
include/util/tdef.h
View File

@ -129,7 +129,7 @@ typedef enum EOperatorType {
OP_TYPE_SUB,
OP_TYPE_MULTI,
OP_TYPE_DIV,
OP_TYPE_MOD,
OP_TYPE_REM,
// unary arithmetic operator
OP_TYPE_MINUS,
OP_TYPE_ASSIGN,

2
source/common/src/tdatablock.c
View File

@ -109,7 +109,7 @@ int32_t getJsonValueLen(const char *data) {
dataLen = DOUBLE_BYTES + CHAR_BYTES;
} else if (*data == TSDB_DATA_TYPE_BOOL) {
dataLen = CHAR_BYTES + CHAR_BYTES;
} else if (*data == TD_TAG_JSON) { // json string
} else if (*data & TD_TAG_JSON) { // json string
dataLen = ((STag*)(data))->len;
} else {
ASSERT(0);

19
source/dnode/vnode/inc/vnode.h
View File

@ -56,7 +56,7 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg);
int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp);
int32_t vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
int32_t vnodePreprocessQueryMsg(SVnode * pVnode, SRpcMsg * pMsg);
int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg);
int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg);
int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo);
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
@ -86,7 +86,7 @@ typedef struct SMetaFltParam {
tb_uid_t suid;
int16_t cid;
int16_t type;
char *val;
char * val;
bool reverse;
int (*filterFunc)(void *a, void *b, int16_t type);
@ -121,7 +121,8 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo
bool isTsdbCacheLastRow(tsdbReaderT *pReader);
int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list);
int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list);
void *tsdbGetIdx(SMeta *pMeta);
void * tsdbGetIdx(SMeta *pMeta);
void * tsdbGetIvtIdx(SMeta *pMeta);
int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle);
bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
@ -197,7 +198,7 @@ struct SMetaEntry {
int64_t version;
int8_t type;
tb_uid_t uid;
char *name;
char * name;
union {
struct {
SSchemaWrapper schemaRow;
@ -225,17 +226,17 @@ struct SMetaEntry {
struct SMetaReader {
int32_t flags;
SMeta *pMeta;
SMeta * pMeta;
SDecoder coder;
SMetaEntry me;
void *pBuf;
void * pBuf;
int32_t szBuf;
};
struct SMTbCursor {
TBC *pDbc;
void *pKey;
void *pVal;
TBC * pDbc;
void * pKey;
void * pVal;
int32_t kLen;
int32_t vLen;
SMetaReader mr;

11
source/dnode/vnode/src/inc/meta.h
View File

@ -69,12 +69,11 @@ struct SMeta {
TTB* pUidIdx;
TTB* pNameIdx;
TTB* pCtbIdx;
#ifdef USE_INVERTED_INDEX
// ivt idx and idx
void* pTagIvtIdx;
#else
TTB* pTagIdx;
#endif
TTB* pTtlIdx;
TTB* pTagIdx;
TTB* pTtlIdx;
TTB* pSmaIdx;
SMetaIdx* pIdx;
};
@ -117,7 +116,7 @@ typedef struct {
} SSmaIdxKey;
// metaTable ==================
int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid,
int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid,
STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey);
#ifndef META_REFACT

1
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -104,6 +104,7 @@ int32_t metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppRea
int32_t metaSnapshotReaderClose(SMetaSnapshotReader* pReader);
int32_t metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nData);
void* metaGetIdx(SMeta* pMeta);
void* metaGetIvtIdx(SMeta* pMeta);
int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);

6
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -93,7 +93,6 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
}
// open pTagIdx
#ifdef USE_INVERTED_INDEX
// TODO(yihaoDeng), refactor later
char indexFullPath[128] = {0};
sprintf(indexFullPath, "%s/%s", pMeta->path, "invert");
@ -104,13 +103,11 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
goto _err;
}
#else
ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx);
if (ret < 0) {
metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
#endif
// open pTtlIdx
ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx);
@ -141,11 +138,8 @@ _err:
if (pMeta->pIdx) metaCloseIdx(pMeta);
if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
#ifdef USE_INVERTED_INDEX
if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
#else
if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
#endif
if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);

8
source/dnode/vnode/src/meta/metaSma.c
View File

@ -28,9 +28,9 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
int vLen = 0;
const void *pKey = NULL;
const void *pVal = NULL;
void *pBuf = NULL;
void * pBuf = NULL;
int32_t szBuf = 0;
void *p = NULL;
void * p = NULL;
SMetaReader mr = {0};
// validate req
@ -83,8 +83,8 @@ int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) {
STbDbKey tbDbKey;
void *pKey = NULL;
void *pVal = NULL;
void * pKey = NULL;
void * pVal = NULL;
int kLen = 0;
int vLen = 0;
SEncoder coder = {0};

93
source/dnode/vnode/src/meta/metaTable.c
View File

@ -15,6 +15,7 @@
#include "meta.h"
static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
@ -25,7 +26,7 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry);
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type);
static int metaUpdateMetaRsp(tb_uid_t uid, char* tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) {
static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) {
pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema));
if (NULL == pMetaRsp->pSchemas) {
terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
@ -43,6 +44,68 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char* tbName, SSchemaWrapper *pSchema
return 0;
}
static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) {
#ifdef USE_INVERTED_INDEX
if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
return -1;
}
void * data = pCtbEntry->ctbEntry.pTags;
const char *tagName = pSchema->name;
tb_uid_t suid = pCtbEntry->ctbEntry.suid;
tb_uid_t tuid = pCtbEntry->uid;
const void *pTagData = pCtbEntry->ctbEntry.pTags;
int32_t nTagData = 0;
SArray *pTagVals = NULL;
if (tTagToValArray((const STag *)data, &pTagVals) != 0) {
return -1;
}
char key[512] = {0};
SIndexMultiTerm *terms = indexMultiTermCreate();
int16_t nCols = taosArrayGetSize(pTagVals);
for (int i = 0; i < nCols; i++) {
STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
char type = pTagVal->type;
sprintf(key, "%s_%s", tagName, pTagVal->pKey);
int32_t nKey = strlen(key);
SIndexTerm *term = NULL;
if (type == TSDB_DATA_TYPE_NULL) {
// handle null value
} else if (type == TSDB_DATA_TYPE_NCHAR) {
if (pTagVal->nData > 0) {
char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
type = TSDB_DATA_TYPE_VARCHAR;
term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, val, len);
} else if (pTagVal->nData == 0) {
char * val = NULL;
int32_t len = 0;
// handle NULL key
}
} else if (type == TSDB_DATA_TYPE_DOUBLE) {
double val = *(double *)(&pTagVal->i64);
int len = 0;
term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len);
} else if (type == TSDB_DATA_TYPE_BOOL) {
int val = *(int *)(&pTagVal->i64);
int len = 0;
term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len);
}
if (term != NULL) {
indexMultiTermAdd(terms, term);
}
memset(key, 0, sizeof(key));
}
tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
indexMultiTermDestroy(terms);
#endif
return -1;
}
int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
SMetaEntry me = {0};
int kLen = 0;
@ -341,7 +404,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
return 0;
}
static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) {
void * pVal = NULL;
int nVal = 0;
@ -824,28 +886,16 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
} else {
// pTagData = pCtbEntry->ctbEntry.pTags;
// nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len;
pTagData = pCtbEntry->ctbEntry.pTags;
nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len;
return metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn);
}
// update tag index
#ifdef USE_INVERTED_INDEX
tb_uid_t suid = pCtbEntry->ctbEntry.suid;
tb_uid_t tuid = pCtbEntry->uid;
SIndexMultiTerm *tmGroup = indexMultiTermCreate();
SIndexTerm *tm = indexTermCreate(suid, ADD_VALUE, pTagColumn->type, pTagColumn->name, sizeof(pTagColumn->name),
pTagData, pTagData == NULL ? 0 : strlen(pTagData));
indexMultiTermAdd(tmGroup, tm);
int ret = indexPut((SIndex *)pMeta->pTagIvtIdx, tmGroup, tuid);
indexMultiTermDestroy(tmGroup);
#else
if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type,
pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) {
return -1;
}
tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn);
metaDestroyTagIdxKey(pTagIdxKey);
#endif
tDecoderClear(&dc);
tdbFree(pData);
return 0;
@ -930,10 +980,5 @@ _err:
return -1;
}
// refactor later
void *metaGetIdx(SMeta *pMeta) {
#ifdef USE_INVERTED_INDEX
return pMeta->pTagIvtIdx;
#else
return pMeta->pTagIdx;
#endif
}
void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; }
void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; }

6
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -2834,6 +2834,12 @@ void* tsdbGetIdx(SMeta* pMeta) {
}
return metaGetIdx(pMeta);
}
void* tsdbGetIvtIdx(SMeta* pMeta) {
if (pMeta == NULL) {
return NULL;
}
return metaGetIvtIdx(pMeta);
}
int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) {
SMCtbCursor* pCur = metaOpenCtbCursor(pMeta, uid);

5
source/libs/executor/src/executorimpl.c
View File

@ -4389,7 +4389,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
STimeWindowAggSupp twSup = {
.waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN};
.waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN};
tsdbReaderT pDataReader = NULL;
if (pHandle->vnode) {
pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond);
@ -4797,7 +4797,8 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa
if (tableType == TSDB_SUPER_TABLE) {
if (pTagCond) {
SIndexMetaArg metaArg = {.metaEx = metaHandle, .metaHandle = tsdbGetIdx(metaHandle), .suid = tableUid};
SIndexMetaArg metaArg = {
.metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
SArray* res = taosArrayInit(8, sizeof(uint64_t));
code = doFilterTag(pTagCond, &metaArg, res);

4
source/libs/index/src/index.c
View File

@ -202,7 +202,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
char buf[128] = {0};
ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType};
int32_t sz = indexSerialCacheKey(&key, buf);
indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
indexDebug("w suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
IndexCache** cache = taosHashGet(index->colObj, buf, sz);
assert(*cache != NULL);
@ -330,7 +330,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
char buf[128] = {0};
ICacheKey key = {
.suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType};
indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
indexDebug("r suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
int32_t sz = indexSerialCacheKey(&key, buf);
taosThreadMutexLock(&sIdx->mtx);

8
source/libs/index/src/indexComm.c
View File

@ -402,16 +402,16 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
break;
}
case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY
tlen = taosEncodeBinary(NULL, src, strlen(src));
tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, strlen(src));
tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
*dst = (char*)*dst - tlen;
break;
}
case TSDB_DATA_TYPE_VARBINARY:
tlen = taosEncodeBinary(NULL, src, strlen(src));
tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, strlen(src));
tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
*dst = (char*)*dst - tlen;
break;
default:

131
source/libs/index/src/indexFilter.c
View File

@ -31,7 +31,7 @@ typedef struct SIFParam {
SHashObj *pFilter;
SArray *result;
char *condValue;
char * condValue;
SIdxFltStatus status;
uint8_t colValType;
@ -45,7 +45,7 @@ typedef struct SIFParam {
typedef struct SIFCtx {
int32_t code;
SHashObj *pRes; /* element is SIFParam */
SHashObj * pRes; /* element is SIFParam */
bool noExec; // true: just iterate condition tree, and add hint to executor plan
SIndexMetaArg arg;
// SIdxFltStatus st;
@ -128,7 +128,7 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) {
// covert data From snode;
SValueNode *vn = (SValueNode *)node;
char *pData = nodesGetValueFromNode(vn);
char * pData = nodesGetValueFromNode(vn);
SDataType *pType = &vn->node.resType;
int32_t type = pType->type;
int32_t valLen = 0;
@ -162,12 +162,27 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) {
return TSDB_CODE_SUCCESS;
}
static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
SOperatorNode *nd = (SOperatorNode *)node;
assert(nodeType(node) == QUERY_NODE_OPERATOR);
SColumnNode *l = (SColumnNode *)nd->pLeft;
SValueNode * r = (SValueNode *)nd->pRight;
param->colId = l->colId;
param->colValType = l->node.resType.type;
memcpy(param->dbName, l->dbName, sizeof(l->dbName));
sprintf(param->colName, "%s_%s", l->colName, r->literal);
param->colValType = r->typeData;
return 0;
// memcpy(param->colName, l->colName, sizeof(l->colName));
}
static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
switch (nodeType(node)) {
case QUERY_NODE_VALUE: {
SValueNode *vn = (SValueNode *)node;
SIF_ERR_RET(sifGetValueFromNode(node, &param->condValue));
param->colId = -1;
param->colValType = (uint8_t)(vn->node.resType.type);
break;
}
case QUERY_NODE_COLUMN: {
@ -219,17 +234,31 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx
indexError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight);
SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) {
return code;
}
SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam));
if (NULL == paramList) {
SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SIF_ERR_JRET(sifInitParam(node->pLeft, &paramList[0], ctx));
if (nParam > 1) {
SIF_ERR_JRET(sifInitParam(node->pRight, &paramList[1], ctx));
if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR) {
SNode *interNode = (node->pLeft);
SIF_ERR_JRET(sifInitJsonParam(interNode, &paramList[0], ctx));
if (nParam > 1) {
SIF_ERR_JRET(sifInitParam(node->pRight, &paramList[1], ctx));
}
paramList[0].colValType = TSDB_DATA_TYPE_JSON;
*params = paramList;
return TSDB_CODE_SUCCESS;
} else {
SIF_ERR_JRET(sifInitParam(node->pLeft, &paramList[0], ctx));
if (nParam > 1) {
SIF_ERR_JRET(sifInitParam(node->pRight, &paramList[1], ctx));
}
*params = paramList;
return TSDB_CODE_SUCCESS;
}
*params = paramList;
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFree(paramList);
SIF_RET(code);
@ -306,39 +335,34 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) {
}
static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
SIndexMetaArg *arg = &output->arg;
#ifdef USE_INVERTED_INDEX
SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName),
right->condValue, strlen(right->condValue));
if (tm == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
int ret = 0;
EIndexQueryType qtype = 0;
SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
if (left->colValType == TSDB_DATA_TYPE_JSON) {
SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, right->colValType, left->colName, strlen(left->colName),
right->condValue, strlen(right->condValue));
if (tm == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST);
indexMultiTermQueryAdd(mtm, tm, qtype);
ret = tIndexJsonSearch(arg->ivtIdx, mtm, output->result);
} else {
bool reverse;
Filter filterFunc = sifGetFilterFunc(qtype, &reverse);
SMetaFltParam param = {.suid = arg->suid,
.cid = left->colId,
.type = left->colValType,
.val = right->condValue,
.reverse = reverse,
.filterFunc = filterFunc};
ret = metaFilteTableIds(arg->metaEx, &param, output->result);
}
EIndexQueryType qtype = 0;
SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST);
indexMultiTermQueryAdd(mtm, tm, qtype);
int ret = indexSearch(arg->metaHandle, mtm, output->result);
indexDebug("index filter data size: %d", (int)taosArrayGetSize(output->result));
indexMultiTermQueryDestroy(mtm);
return ret;
#else
EIndexQueryType qtype = 0;
SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
bool reverse;
Filter filterFunc = sifGetFilterFunc(qtype, &reverse);
SMetaFltParam param = {.suid = arg->suid,
.cid = left->colId,
.type = left->colValType,
.val = right->condValue,
.reverse = reverse,
.filterFunc = filterFunc};
int ret = metaFilteTableIds(arg->metaEx, &param, output->result);
return ret;
#endif
return 0;
}
static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
@ -392,6 +416,14 @@ static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output
int id = OP_TYPE_NMATCH;
return sifDoIndex(left, right, id, output);
}
static int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) {
// return 0
return 0;
}
static int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) {
// return 0
return 0;
}
static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
// add more except
@ -445,6 +477,14 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta
*status = SFLT_NOT_INDEX;
*func = sifNotMatchFunc;
return 0;
case OP_TYPE_JSON_CONTAINS:
*status = SFLT_ACCURATE_INDEX;
*func = sifJsonContains;
return 0;
case OP_TYPE_JSON_GET_VALUE:
*status = SFLT_ACCURATE_INDEX;
*func = sifJsonGetValue;
return 0;
default:
*status = SFLT_NOT_INDEX;
*func = sifNullFunc;
@ -457,12 +497,17 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta
static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
int32_t code = 0;
int32_t nParam = sifGetOperParamNum(node->opType);
if (nParam <= 1) {
SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) {
return code;
}
SIFParam *params = NULL;
SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
// ugly code, refactor later
output->arg = ctx->arg;
sif_func_t operFn = sifNullFunc;
code = sifGetOperFn(node->opType, &operFn, &output->status);
if (!ctx->noExec) {
@ -567,7 +612,9 @@ EDealRes sifCalcWalker(SNode *node, void *context) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(node)) {
return sifWalkLogic(node, ctx);
}
if (QUERY_NODE_OPERATOR == nodeType(node)) {
indexInfo("node type for index filter, type: %d", nodeType(node));
return sifWalkOper(node, ctx);
}
@ -658,7 +705,7 @@ int32_t doFilterTag(const SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *re
// todo move to the initialization function
// SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0));
SArray *output = taosArrayInit(8, sizeof(uint64_t));
SArray * output = taosArrayInit(8, sizeof(uint64_t));
SIFParam param = {.arg = *metaArg, .result = output};
SIF_ERR_RET(sifCalculate((SNode *)pFilterNode, &param));

113
source/libs/index/test/indexTests.cc
View File

@ -498,10 +498,15 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
}
taosArrayDestroy(data);
std::string colName("voltage");
std::string colVal("ab");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
std::string colName("voltage");
std::string colVal("ab");
char buf[256] = {0};
int16_t sz = colVal.size();
memcpy(buf, (uint16_t*)&sz, 2);
memcpy(buf + 2, colVal.c_str(), colVal.size());
SIndexTerm* term =
indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), buf, sizeof(buf));
SIndexTermQuery query = {term, QUERY_TERM};
SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
@ -564,6 +569,18 @@ class IndexCacheEnv : public ::testing::Test {
CacheObj* coj;
};
SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
int32_t nColName, const char* colVal, int32_t nColVal) {
char buf[256] = {0};
int16_t sz = nColVal;
memcpy(buf, (uint16_t*)&sz, 2);
memcpy(buf + 2, colVal, nColVal);
if (colType == TSDB_DATA_TYPE_BINARY) {
return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf));
} else {
return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal);
}
}
#define MAX_TERM_KEY_LEN 128
TEST_F(IndexCacheEnv, cache_test) {
int version = 0;
@ -574,37 +591,37 @@ TEST_F(IndexCacheEnv, cache_test) {
std::string colName("voltage");
{
std::string colVal("v1");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
// indexTermDestry(term);
}
{
std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
}
{
std::string colVal("v2");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
}
{
std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
}
{
std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
}
@ -612,15 +629,15 @@ TEST_F(IndexCacheEnv, cache_test) {
std::cout << "--------first----------" << std::endl;
{
std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, othColId, version++, suid++);
indexTermDestroy(term);
}
{
std::string colVal("v4");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, othColId, version++, suid++);
indexTermDestroy(term);
}
@ -630,8 +647,8 @@ TEST_F(IndexCacheEnv, cache_test) {
std::string colVal("v4");
for (size_t i = 0; i < 10; i++) {
colVal[colVal.size() - 1] = 'a' + i;
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
}
@ -640,8 +657,8 @@ TEST_F(IndexCacheEnv, cache_test) {
// begin query
{
std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTermQuery query = {term, QUERY_TERM};
SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid));
STermValueType valType;
@ -655,8 +672,8 @@ TEST_F(IndexCacheEnv, cache_test) {
}
{
std::string colVal("v2");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTermQuery query = {term, QUERY_TERM};
SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid));
STermValueType valType;
@ -690,8 +707,8 @@ class IndexObj {
return ret;
}
void Del(const std::string& colName, const std::string& colVal, uint64_t uid) {
SIndexTerm* term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
Put(terms, uid);
@ -699,8 +716,8 @@ class IndexObj {
}
int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world",
size_t numOfTable = 100 * 10000) {
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < numOfTable; i++) {
@ -721,8 +738,8 @@ class IndexObj {
// opt
tColVal[taosRand() % colValSize] = 'a' + k % 26;
}
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
tColVal.c_str(), tColVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
tColVal.c_str(), tColVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t j = 0; j < skip; j++) {
@ -757,8 +774,8 @@ class IndexObj {
int SearchOne(const std::string& colName, const std::string& colVal) {
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermQueryAdd(mq, term, QUERY_TERM);
SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
@ -779,8 +796,8 @@ class IndexObj {
}
int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) {
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermQueryAdd(mq, term, QUERY_TERM);
SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
@ -804,16 +821,16 @@ class IndexObj {
void PutOne(const std::string& colName, const std::string& colVal) {
SIndexMultiTerm* terms = indexMultiTermCreate();
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermAdd(terms, term);
Put(terms, 10);
indexMultiTermDestroy(terms);
}
void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) {
SIndexMultiTerm* terms = indexMultiTermCreate();
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermAdd(terms, term);
Put(terms, val);
indexMultiTermDestroy(terms);
@ -858,8 +875,8 @@ TEST_F(IndexEnv2, testIndexOpen) {
{
std::string colName("tag1"), colVal("Hello");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < targetSize; i++) {
@ -873,8 +890,8 @@ TEST_F(IndexEnv2, testIndexOpen) {
size_t size = 200;
std::string colName("tag1"), colVal("hello");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < size; i++) {
@ -888,8 +905,8 @@ TEST_F(IndexEnv2, testIndexOpen) {
size_t size = 200;
std::string colName("tag1"), colVal("Hello");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = size * 3; i < size * 4; i++) {
@ -903,8 +920,8 @@ TEST_F(IndexEnv2, testIndexOpen) {
{
std::string colName("tag1"), colVal("Hello");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermQueryAdd(mq, term, QUERY_TERM);
SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
@ -926,8 +943,8 @@ TEST_F(IndexEnv2, testEmptyIndexOpen) {
{
std::string colName("tag1"), colVal("Hello");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < targetSize; i++) {

142
source/libs/index/test/jsonUT.cc
View File

@ -19,6 +19,18 @@
static std::string dir = TD_TMP_DIR_PATH "json";
static std::string logDir = TD_TMP_DIR_PATH "log";
SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
int32_t nColName, const char* colVal, int32_t nColVal) {
char buf[256] = {0};
int16_t sz = nColVal;
memcpy(buf, (uint16_t*)&sz, 2);
memcpy(buf + 2, colVal, nColVal);
if (colType == TSDB_DATA_TYPE_BINARY) {
return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf));
} else {
return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal);
}
}
static void initLog() {
const char* defaultLogFileNamePrefix = "taoslog";
const int32_t maxLogFileNum = 10;
@ -59,8 +71,8 @@ class JsonEnv : public ::testing::Test {
static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId,
int8_t operType = ADD_VALUE) {
SIndexTerm* term =
indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
SIndexTerm* term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(),
(const char*)data, dlen);
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, (int64_t)tableId);
@ -70,8 +82,8 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp
static void delData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId,
int8_t operType = DEL_VALUE) {
SIndexTerm* term =
indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
SIndexTerm* term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(),
(const char*)data, dlen);
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, (int64_t)tableId);
@ -83,7 +95,7 @@ static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, v
std::string colName(colNam);
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
SArray* res = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, (EIndexQueryType)filterType);
@ -95,8 +107,8 @@ TEST_F(JsonEnv, testWrite) {
{
std::string colName("test");
std::string colVal("ab");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -108,8 +120,8 @@ TEST_F(JsonEnv, testWrite) {
{
std::string colName("voltage");
std::string colVal("ab1");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -121,8 +133,8 @@ TEST_F(JsonEnv, testWrite) {
{
std::string colName("voltage");
std::string colVal("123");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -136,8 +148,8 @@ TEST_F(JsonEnv, testWrite) {
std::string colVal("ab");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@ -150,8 +162,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
{
std::string colName("test");
std::string colVal("ab");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -165,8 +177,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
std::string colVal("abxxxxxxxxxxxx");
for (int i = 0; i < 10; i++) {
colVal[i % colVal.size()] = '0' + i % 128;
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -179,8 +191,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
{
std::string colName("voltagefdadfa");
std::string colVal("abxxxxxxxxxxxx");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -194,8 +206,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
std::string colVal("ab");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@ -209,8 +221,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
std::string colVal("ab");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@ -224,8 +236,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
std::string colVal("ab");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
@ -241,8 +253,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
std::string colName("test");
// std::string colVal("10");
int val = 10;
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -254,8 +266,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
{
std::string colName("test2");
int val = 20;
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -267,8 +279,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
{
std::string colName("test");
int val = 15;
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -280,8 +292,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
{
std::string colName("test2");
const char* val = "test";
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
(const char*)val, strlen(val));
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
(const char*)val, strlen(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -294,8 +306,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
std::string colName("test");
int val = 15;
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@ -308,8 +320,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
int val = 15;
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@ -323,8 +335,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
;
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(int));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(int));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
@ -338,8 +350,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
// std::string colVal("10");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN);
@ -353,8 +365,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
// std::string colVal("10");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL);
@ -368,8 +380,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
{
std::string colName("test1");
int val = 10;
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -381,8 +393,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
{
std::string colName("test");
std::string colVal("xxxxxxxxxxxxxxxxxxx");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -396,8 +408,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
int val = 10;
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@ -410,8 +422,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
int val = 10;
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(int));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(int));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@ -425,8 +437,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
int val = 10;
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
@ -439,8 +451,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
int val = 10;
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@ -453,8 +465,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
int val = 10;
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL);
@ -466,8 +478,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
std::string colName("other_column");
int val = 100;
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -482,8 +494,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
// std::string colVal("10");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN);
@ -494,8 +506,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
{
std::string colName("test1");
int val = 15;
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@ -510,8 +522,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
// std::string colVal("10");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);

2
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -1096,7 +1096,7 @@ bool nodesIsArithmeticOp(const SOperatorNode* pOp) {
case OP_TYPE_SUB:
case OP_TYPE_MULTI:
case OP_TYPE_DIV:
case OP_TYPE_MOD:
case OP_TYPE_REM:
return true;
default:
break;

2
source/libs/parser/inc/sql.y
View File

@ -611,7 +611,7 @@ expression(A) ::= expression(B) NK_SLASH expression(C).
expression(A) ::= expression(B) NK_REM expression(C). {
SToken s = getTokenFromRawExprNode(pCxt, B);
SToken e = getTokenFromRawExprNode(pCxt, C);
A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)));
A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)));
}
expression(A) ::= column_reference(B) NK_ARROW NK_STRING(C). {
SToken s = getTokenFromRawExprNode(pCxt, B);

2
source/libs/parser/src/sql.c
View File

@ -4079,7 +4079,7 @@ static YYACTIONTYPE yy_reduce(
{
SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172);
SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172);
yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172)));
yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172)));
}
yymsp[-2].minor.yy172 = yylhsminor.yy172;
break;

2
source/libs/scalar/inc/filterInt.h
View File

@ -350,7 +350,7 @@ struct SFilterInfo {
extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right);
extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
extern OptrStr gOptrStr[];
#ifdef __cplusplus
}

5
source/libs/scalar/src/filter.c
View File

@ -29,8 +29,9 @@ OptrStr gOptrStr[] = {
{OP_TYPE_SUB, "-"},
{OP_TYPE_MULTI, "*"},
{OP_TYPE_DIV, "/"},
{OP_TYPE_MOD, "%"},
{OP_TYPE_REM, "%"},
{OP_TYPE_MINUS, "minus"},
{OP_TYPE_ASSIGN, "assign"},
// bit operator
{OP_TYPE_BIT_AND, "&"},
{OP_TYPE_BIT_OR, "|"},

153
source/libs/scalar/src/sclvector.c
View File

@ -29,6 +29,12 @@
#define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData))
#define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData))
#define IS_NULL colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i) \
|| IS_JSON_NULL(pLeft->columnData->info.type, colDataGetVarData(pLeft->columnData, i)) \
|| IS_JSON_NULL(pRight->columnData->info.type, colDataGetVarData(pRight->columnData, i))
#define IS_HELPER_NULL(col,i) colDataIsNull_s(col, i) || IS_JSON_NULL(col->info.type, colDataGetVarData(col, i))
void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
switch (outType) {
case TSDB_DATA_TYPE_BOOL: {
@ -387,7 +393,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
pOut->numOfRows = pIn->numOfRows;
for (int32_t i = 0; i < pIn->numOfRows; ++i) {
if (colDataIsNull_s(pIn->columnData, i)) {
if (IS_HELPER_NULL(pIn->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue;
}
@ -396,8 +402,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
int32_t convertType = inType;
if(inType == TSDB_DATA_TYPE_JSON){
if(*data == TSDB_DATA_TYPE_NULL) {
colDataAppendNULL(pOut->columnData, i);
continue;
ASSERT(0);
}
else if(*data == TSDB_DATA_TYPE_NCHAR) {
data += CHAR_BYTES;
@ -447,13 +452,13 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){
return out;
}
void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){
bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){
if(optr == OP_TYPE_JSON_CONTAINS) {
return;
return true;
}
if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){
return;
return true;
}
if(typeLeft == TSDB_DATA_TYPE_JSON){
@ -464,15 +469,22 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
typeRight = **pRightData;
(*pRightData) ++;
}
if(optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH){
if(typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY){
return false;
}
}
if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){
*isNull = true;
return;
return true;
}
int8_t type = vectorGetConvertType(typeLeft, typeRight);
if(type == 0) {
*fp = filterGetCompFunc(typeLeft, optr);
return;
return true;
}
*fp = filterGetCompFunc(type, optr);
@ -492,6 +504,7 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
*pRightData = pRightOut;
}
return true;
}
int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int16_t inType, int16_t outType) {
@ -867,11 +880,11 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
double *output = (double *)pOutputCol->pData;
if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value
if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
if (IS_HELPER_NULL(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -887,11 +900,11 @@ static void vectorMathBigintAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData
int64_t *output = (int64_t *)pOutputCol->pData;
if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value
if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
if (IS_HELPER_NULL(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -982,7 +995,7 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
if (IS_NULL) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1000,7 +1013,7 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
if (IS_NULL){
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1024,11 +1037,11 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
double *output = (double *)pOutputCol->pData;
if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value
if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
if (IS_HELPER_NULL(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1044,11 +1057,11 @@ static void vectorMathBigintSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData
int64_t *output = (int64_t *)pOutputCol->pData;
if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value
if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
if (IS_HELPER_NULL(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1077,7 +1090,7 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
if (IS_NULL) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1095,7 +1108,7 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
if (IS_NULL) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1119,11 +1132,11 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
double *output = (double *)pOutputCol->pData;
if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value
if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
if (IS_HELPER_NULL(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1149,7 +1162,7 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
double *output = (double *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
if (IS_NULL) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1182,7 +1195,7 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
double *output = (double *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { //divide by 0 check
if (IS_NULL || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { //divide by 0 check
colDataAppendNULL(pOutputCol, i);
continue;
}
@ -1190,11 +1203,11 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
/ getVectorDoubleValueFnRight(RIGHT_COL, i);
}
} else if (pLeft->numOfRows == 1) {
if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value
if (IS_HELPER_NULL(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
} else {
for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pRightCol, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { // divide by 0 check
if (IS_HELPER_NULL(pRightCol, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { // divide by 0 check
colDataAppendNULL(pOutputCol, i);
continue;
}
@ -1203,11 +1216,11 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
}
}
} else if (pRight->numOfRows == 1) {
if (colDataIsNull_s(pRightCol, 0) || (getVectorDoubleValueFnRight(RIGHT_COL, 0) == 0)) { // Set pLeft->numOfRows NULL value (divde by 0 check)
if (IS_HELPER_NULL(pRightCol, 0) || (getVectorDoubleValueFnRight(RIGHT_COL, 0) == 0)) { // Set pLeft->numOfRows NULL value (divde by 0 check)
colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
} else {
for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
if (IS_HELPER_NULL(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue;
}
@ -1236,18 +1249,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
_getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type);
double *output = (double *)pOutputCol->pData;
double zero = 0.0;
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i) || colDataIsNull_s(pRightCol, i)) {
if (IS_NULL) {
colDataAppendNULL(pOutputCol, i);
continue;
}
double lx = getVectorDoubleValueFnLeft(LEFT_COL, i);
double rx = getVectorDoubleValueFnRight(RIGHT_COL, i);
if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) {
if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) {
colDataAppendNULL(pOutputCol, i);
continue;
}
@ -1256,11 +1268,11 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
}
} else if (pLeft->numOfRows == 1) {
double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0);
if (colDataIsNull_s(pLeftCol, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value
if (IS_HELPER_NULL(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
} else {
for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pRightCol, i)) {
if (IS_HELPER_NULL(pRightCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue;
}
@ -1276,11 +1288,11 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
}
} else if (pRight->numOfRows == 1) {
double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0);
if (colDataIsNull_s(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value
if (IS_HELPER_NULL(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
} else {
for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
if (IS_HELPER_NULL(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue;
}
@ -1315,7 +1327,7 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
double *output = (double *)pOutputCol->pData;
for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i)) {
if (IS_HELPER_NULL(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue;
}
@ -1331,7 +1343,7 @@ void vectorAssign(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
pOut->numOfRows = pLeft->numOfRows;
if (colDataIsNull_s(pRight->columnData, 0)) {
if (IS_HELPER_NULL(pRight->columnData, 0)) {
for (int32_t i = 0; i < pOut->numOfRows; ++i) {
colDataAppend(pOutputCol, i, NULL, true);
}
@ -1397,13 +1409,13 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh
_getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type);
_getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type);
double *output = (double *)pOutputCol->pData;
int64_t *output = (int64_t *)pOutputCol->pData;
if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
if (IS_HELPER_NULL(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1429,7 +1441,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
int64_t *output = (int64_t *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
if (IS_NULL) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1451,12 +1463,12 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight
int64_t *output = (int64_t *)pOutputCol->pData;
if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else {
int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0);
for (; i >= 0 && i < numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
if (IS_HELPER_NULL(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1482,7 +1494,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
int64_t *output = (int64_t *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
if (IS_NULL) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
@ -1507,7 +1519,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
if (pRight->pHashFilter != NULL) {
for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (colDataIsNull_s(pLeft->columnData, i)) {
if (IS_HELPER_NULL(pLeft->columnData, i)) {
bool res = false;
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
continue;
@ -1522,7 +1534,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, i)) {
bool res = false;
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
continue; // TODO set null or ignore
@ -1534,18 +1546,21 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
int64_t leftOut = 0;
int64_t rightOut = 0;
bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore
ASSERT(0);
}
if(!result){
colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
}else{
bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
}
bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
}
} else if (pRight->numOfRows == 1) {
ASSERT(pLeft->pHashFilter == NULL);
for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, 0)) {
if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, 0)) {
bool res = false;
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
continue;
@ -1556,17 +1571,20 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
int64_t leftOut = 0;
int64_t rightOut = 0;
bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore
ASSERT(0);
}
if(!result){
colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
}else{
bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
}
bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
}
} else if (pLeft->numOfRows == 1) {
for (; i >= 0 && i < pRight->numOfRows; i += step) {
if (colDataIsNull_s(pRight->columnData, i) || colDataIsNull_s(pLeft->columnData, 0)) {
if (IS_HELPER_NULL(pRight->columnData, i) || IS_HELPER_NULL(pLeft->columnData, 0)) {
bool res = false;
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
continue;
@ -1577,13 +1595,16 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
int64_t leftOut = 0;
int64_t rightOut = 0;
bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore
ASSERT(0);
}
if(!result){
colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
}else{
bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
}
bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
}
}
}
@ -1668,7 +1689,7 @@ void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0;
int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0;
colDataAppendInt8(pOut->columnData, i, &v);
}
pOut->numOfRows = pLeft->numOfRows;
@ -1676,7 +1697,7 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
int8_t v = colDataIsNull_s(pLeft->columnData, i)? 0:1;
int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1;
colDataAppendInt8(pOut->columnData, i, &v);
}
pOut->numOfRows = pLeft->numOfRows;
@ -1696,7 +1717,7 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
return vectorMathMultiply;
case OP_TYPE_DIV:
return vectorMathDivide;
case OP_TYPE_MOD:
case OP_TYPE_REM:
return vectorMathRemainder;
case OP_TYPE_MINUS:
return vectorMathMinus;

290
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -42,6 +42,7 @@
#include "nodes.h"
#include "tlog.h"
#include "parUtil.h"
#include "filterInt.h"
#define _DEBUG_PRINT_ 0
@ -212,6 +213,24 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
*pNode = (SNode *)rnode;
}
void scltMakeOpNode2(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight, bool isReverse) {
SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_OPERATOR);
SOperatorNode *onode = (SOperatorNode *)node;
onode->node.resType.type = resType;
onode->node.resType.bytes = tDataTypes[resType].bytes;
onode->opType = opType;
if(isReverse){
onode->pLeft = pRight;
onode->pRight = pLeft;
}else{
onode->pLeft = pLeft;
onode->pRight = pRight;
}
*pNode = (SNode *)onode;
}
void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) {
SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_OPERATOR);
SOperatorNode *onode = (SOperatorNode *)node;
@ -1039,10 +1058,10 @@ void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){
scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
}
void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData){
void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData, bool isReverse){
int32_t resType = TSDB_DATA_TYPE_NULL;
if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI ||
opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
opType == OP_TYPE_DIV || opType == OP_TYPE_REM || opType == OP_TYPE_MINUS){
resType = TSDB_DATA_TYPE_DOUBLE;
}else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
resType = TSDB_DATA_TYPE_BIGINT;
@ -1057,7 +1076,7 @@ void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32
SNode *right = NULL;
scltMakeValueNode(&right, rightType, rightData);
scltMakeOpNode(opNode, opType, resType, *opNode, right);
scltMakeOpNode2(opNode, opType, resType, *opNode, right, isReverse);
SColumnInfo colInfo = createColumnInfo(1, resType, tDataTypes[resType].bytes);
int16_t dataBlockId = 0, slotId = 0;
@ -1065,7 +1084,7 @@ void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32
scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode);
}
void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType){
void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType, bool isReverse){
SArray *blockList = taosArrayInit(2, POINTER_BYTES);
SSDataBlock *src = NULL;
SNode *opNode = NULL;
@ -1073,7 +1092,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
makeJsonArrow(&src, &opNode, json, (char*)key);
taosArrayPush(blockList, &src);
makeOperator(&opNode, blockList, opType, rightType, rightData);
makeOperator(&opNode, blockList, opType, rightType, rightData, isReverse);
int32_t code = scalarCalculate(opNode, blockList, NULL);
ASSERT_EQ(code, 0);
@ -1087,17 +1106,17 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
printf("result:NULL\n");
}else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV ||
opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
printf("1result:%f,except:%f\n", *((double *)colDataGetData(column, 0)), exceptValue);
opType == OP_TYPE_REM || opType == OP_TYPE_MINUS){
printf("op:%s,1result:%f,except:%f\n", gOptrStr[opType].str, *((double *)colDataGetData(column, 0)), exceptValue);
ASSERT_TRUE(fabs(*((double *)colDataGetData(column, 0)) - exceptValue) < 0.0001);
}else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
printf("2result:%ld,except:%f\n", *((int64_t *)colDataGetData(column, 0)), exceptValue);
printf("op:%s,2result:%ld,except:%f\n", gOptrStr[opType].str, *((int64_t *)colDataGetData(column, 0)), exceptValue);
ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue);
}else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN ||
opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL ||
opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE ||
opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){
printf("3result:%d,except:%f\n", *((bool *)colDataGetData(column, 0)), exceptValue);
printf("op:%s,3result:%d,except:%f\n", gOptrStr[opType].str, *((bool *)colDataGetData(column, 0)), exceptValue);
ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue);
}
@ -1107,7 +1126,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
TEST(columnTest, json_column_arith_op) {
scltInitLogFile();
char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":-10,\"k7\":-9.8,\"k8\":false,\"k9\":\"8hel\"}";
char rightv[256] = {0};
memcpy(rightv, rightvTmp, strlen(rightvTmp));
@ -1117,54 +1136,126 @@ TEST(columnTest, json_column_arith_op) {
const int32_t len = 8;
EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV,
OP_TYPE_MOD, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR};
OP_TYPE_REM, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR};
int32_t input[len] = {1, 8, 2, 2, 3, 0, -4, 9};
printf("--------------------json int---------------------\n");
printf("--------------------json int-4 op {1, 8, 2, 2, 3, 0, -4, 9}--------------------\n");
char *key = "k1";
double eRes[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9};
double eRes00[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9};
double eRes01[len] = {5.0, 4, 8.0, 0.5, 3, 0, 4&-4, 4|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes00[i], op[i], false);
}
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes01[i], op[i], true);
}
printf("--------------------json string---------------------\n");
printf("--------------------json string- 0 op {1, 8, 2, 2, 3, 0, -4, 9}--------------------\n");
key = "k2";
double eRes1[len] = {1.0, -8, 0, 0, 0, 0, 0, 9};
double eRes10[len] = {1.0, -8, 0, 0, 0, 0, 0, 9};
double eRes11[len] = {1.0, 8, 0, DBL_MAX, DBL_MAX, 0, 0, 9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i], false);
}
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes11[i], op[i], true);
}
printf("---------------------json null--------------------\n");
printf("---------------------json null- null op {1, 8, 2, 2, 3, 0, -4, 9}-------------------\n");
key = "k3";
double eRes2[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
double eRes20[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
double eRes21[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, 0, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes20[i], op[i], false);
}
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes21[i], op[i], true);
}
printf("---------------------json bool--------------------\n");
printf("---------------------json bool- true op {1, 8, 2, 2, 3, 0, -4, 9}-------------------\n");
key = "k4";
double eRes3[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9};
double eRes30[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9};
double eRes31[len] = {2.0, 7, 2, 2, 0, 0, 1&-4, 1|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes30[i], op[i], false);
}
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes31[i], op[i], true);
}
printf("----------------------json double-------------------\n");
printf("----------------------json double-- 5.44 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
key = "k5";
double eRes4[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9};
double eRes40[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9};
double eRes41[len] = {6.44, 2.56, 10.88, 0.3676470588235294, 3, 0, 5&-4, 5|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes40[i], op[i], false);
}
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes41[i], op[i], true);
}
printf("---------------------json not exist--------------------\n");
printf("----------------------json int-- -10 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
key = "k6";
double eRes50[len] = {-9, -18, -20, -5, -10%3, 10, -10&-4, -10|9};
double eRes51[len] = {-9, 18, -20, -0.2, 3%-10, 0, -10&-4, -10|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes50[i], op[i], false);
}
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes51[i], op[i], true);
}
printf("----------------------json double-- -9.8 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
key = "k7";
double eRes60[len] = {-8.8, -17.8, -19.6, -4.9, -0.8, 9.8, -9&-4, -9|9};
double eRes61[len] = {-8.8, 17.8, -19.6, -0.2040816326530612, 3, 0, -9&-4, -9|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes60[i], op[i], false);
}
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes61[i], op[i], true);
}
printf("----------------------json bool-- 0 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
key = "k8";
double eRes70[len] = {1.0, -8, 0, 0, 0, 0, 0, 9};
double eRes71[len] = {1.0, 8, 0, DBL_MAX, DBL_MAX, 0, 0, 9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes70[i], op[i], false);
}
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes71[i], op[i], true);
}
printf("----------------------json string-- 8 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
key = "k9";
double eRes80[len] = {9, 0, 16, 4, 8%3, -8, 8&-4, 8|9};
double eRes81[len] = {9, 0, 16, 0.25, 3%8, 0, 8&-4, 8|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes80[i], op[i], false);
}
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes81[i], op[i], true);
}
printf("---------------------json not exist-- NULL op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
key = "k10";
double eRes5[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
double eRes90[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
double eRes91[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, 0, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes90[i], op[i], false);
}
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes91[i], op[i], true);
}
taosArrayDestroy(tags);
@ -1183,7 +1274,7 @@ void *prepareNchar(char* rightData){
TEST(columnTest, json_column_logic_op) {
scltInitLogFile();
char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}";
char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":-10,\"k7\":-9.8,\"k8\":false,\"k9\":\"6.6hello\"}";
char rightv[256] = {0};
memcpy(rightv, rightvTmp, strlen(rightvTmp));
@ -1191,6 +1282,7 @@ TEST(columnTest, json_column_logic_op) {
STag* row = NULL;
parseJsontoTagData(rightv, tags, &row, NULL);
const int32_t len0 = 6;
const int32_t len = 9;
const int32_t len1 = 4;
EOperatorType op[len+len1] = {OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, OP_TYPE_NOT_EQUAL,
@ -1199,93 +1291,183 @@ TEST(columnTest, json_column_logic_op) {
int32_t input[len] = {1, 8, 2, 2, 3, 0, 0, 0, 0};
char *inputNchar[len1] = {"hell_", "hel%", "hell", "llll"};
printf("--------------------json int---------------------\n");
printf("--------------------json int---4 {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n");
char *key = "k1";
bool eRes[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i], false);
}
bool eRes_0[len0] = {false, true, true, true, false, true};
for(int i = 0; i < len0; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_0[i], op[i], true);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i], false);
taosMemoryFree(rightData);
}
printf("--------------------json string---------------------\n");
printf("--------------------json string--0 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
key = "k2";
bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i], false);
}
bool eRes_1[len0] = {true, true, false, false, false, false};
for(int i = 0; i < len0; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_1[i], op[i], true);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i], false);
taosMemoryFree(rightData);
}
printf("--------------------json null---------------------\n");
printf("--------------------json null---null {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n");
key = "k3"; // (null is true) return NULL, so use DBL_MAX represent NULL
double eRes2[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i], false);
}
bool eRes_2[len0] = {false, false, false, false, false, false};
for(int i = 0; i < len0; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_2[i], op[i], true);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i], false);
taosMemoryFree(rightData);
}
printf("--------------------json bool---------------------\n");
printf("--------------------json bool--1 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
key = "k4";
bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i], false);
}
bool eRes_3[len0] = {false, true, false, false, false, true};
for(int i = 0; i < len0; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_3[i], op[i], true);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i], false);
taosMemoryFree(rightData);
}
printf("--------------------json double---------------------\n");
printf("--------------------json double--5.44 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
key = "k5";
bool eRes4[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i], false);
}
bool eRes_4[len0] = {false, true, true, true, false, true};
for(int i = 0; i < len0; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_4[i], op[i], true);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i], false);
taosMemoryFree(rightData);
}
printf("--------------------json double---------------------\n");
printf("--------------------json int-- -10 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
key = "k6";
bool eRes5[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, false, true};
bool eRes5[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i], false);
}
bool eRes_5[len0] = {true, true, false, false, false, true};
for(int i = 0; i < len0; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_5[i], op[i], true);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i], false);
taosMemoryFree(rightData);
}
printf("---------------------json not exist--------------------\n");
printf("--------------------json double-- -9.8 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
key = "k10"; // (NULL is true) return NULL, so use DBL_MAX represent NULL
double eRes10[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false};
key = "k7";
bool eRes6[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes6[i], op[i], false);
}
bool eRes_6[len0] = {true, true, false, false, false, true};
for(int i = 0; i < len0; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_6[i], op[i], true);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes6[i], op[i], false);
taosMemoryFree(rightData);
}
printf("--------------------json bool-- 0 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
key = "k8";
bool eRes7[len+len1] = {false, false, true, true, false, false, false, true, false, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes7[i], op[i], false);
}
bool eRes_7[len0] = {true, true, false, false, false, false};
for(int i = 0; i < len0; i++) {
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_7[i], op[i], true);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes7[i], op[i], false);
taosMemoryFree(rightData);
}
printf("--------------------json string-- 6.6hello {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
key = "k9";
bool eRes8[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, false, true};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes8[i], op[i], false);
}
bool eRes_8[len0] = {false, true, true, true, false, true};
for(int i = 0; i < len0; i++) {
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_8[i], op[i], true);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes8[i], op[i], false);
taosMemoryFree(rightData);
}
printf("---------------------json not exist-- NULL {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n");
key = "k10"; // (NULL is true) return NULL, so use DBL_MAX represent NULL
double eRes9[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes9[i], op[i], false);
}
bool eRes_9[len0] = {false, false, false, false, false, false};
for(int i = 0; i < len0; i++) {
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_9[i], op[i], true);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes9[i], op[i], false);
taosMemoryFree(rightData);
}

2
tests/pytest/util/dnodes.py
View File

@ -318,7 +318,7 @@ class TDDnode:
print(cmd)
if (not self.remoteIP == ""):
self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)\ntdDnodes.start(%d)"%(self.index, self.index))
self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].deployed=1\ntdDnodes.dnodes[%d].logDir=\"%%s/sim/dnode%%d/log\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.dnodes[%d].cfgDir=\"%%s/sim/dnode%%d/cfg\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.start(%d)"%(self.index-1,self.index-1,self.index-1,self.index,self.index-1,self.index-1,self.index,self.index))
self.running = 1
else:
if os.system(cmd) != 0:

62
tests/system-test/0-others/udf_create.py
View File

@ -25,7 +25,7 @@ class TDTestCase:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
if ("taosd" in files or "taosd.exe" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
@ -41,11 +41,19 @@ class TDTestCase:
projPath = selfPath[:selfPath.find("tests")]
print(projPath)
libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
os.system("mkdir /tmp/udf/")
os.system("cp %s /tmp/udf/ "%libudf1.replace("\n" ,""))
os.system("cp %s /tmp/udf/ "%libudf2.replace("\n" ,""))
if platform.system().lower() == 'windows':
self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
if (not tdDnodes.dnodes[0].remoteIP == ""):
tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\")
tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\")
self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so')
self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so')
else:
self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf1 = self.libudf1.replace('\r','').replace('\n','')
self.libudf2 = self.libudf2.replace('\r','').replace('\n','')
def prepare_data(self):
@ -136,11 +144,11 @@ class TDTestCase:
for i in range(5):
# create scalar functions
tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1)
# create aggregate functions
tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
functions = tdSql.getResult("show functions")
function_nums = len(functions)
@ -161,11 +169,11 @@ class TDTestCase:
tdLog.info("drop two udf functions success ")
# create scalar functions
tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1)
# create aggregate functions
tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
functions = tdSql.getResult("show functions")
function_nums = len(functions)
@ -533,8 +541,8 @@ class TDTestCase:
tdSql.query("drop function udf2 ")
# create function without buffer
tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int")
tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double")
tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1)
tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2)
udf1_sqls ,udf2_sqls = self.try_query_sql()
for scalar_sql in udf1_sqls:
@ -549,8 +557,8 @@ class TDTestCase:
tdSql.query("drop function udf2 ")
# create function without buffer
tdSql.execute("create aggregate function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
tdSql.execute("create function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1)
tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2)
udf1_sqls ,udf2_sqls = self.try_query_sql()
for scalar_sql in udf1_sqls:
@ -558,8 +566,8 @@ class TDTestCase:
for aggregate_sql in udf2_sqls:
tdSql.error(aggregate_sql)
tdSql.execute(" create function db as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
tdSql.execute(" create aggregate function test as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1)
tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1)
tdSql.error(" select db(c1) from stb1 ")
tdSql.error(" select db(c1,c6), db(c6) from stb1 ")
tdSql.error(" select db(num1,num2), db(num1) from tb ")
@ -607,17 +615,17 @@ class TDTestCase:
tdLog.info(" create function name is not build_in functions ")
tdSql.execute(" drop function udf1 ")
tdSql.execute(" drop function udf2 ")
tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8")
tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8")
tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function tbname as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function function as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function stable as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function union as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function 123 as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function 123db as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function mnode as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1)
tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1)
tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2)
def restart_taosd_query_udf(self):

62
tests/system-test/0-others/udf_restart_taosd.py
View File

@ -25,7 +25,7 @@ class TDTestCase:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
if ("taosd" in files or "taosd.exe" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
@ -41,11 +41,19 @@ class TDTestCase:
projPath = selfPath[:selfPath.find("tests")]
print(projPath)
libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
os.system("mkdir /tmp/udf/")
os.system("cp %s /tmp/udf/ "%libudf1.replace("\n" ,""))
os.system("cp %s /tmp/udf/ "%libudf2.replace("\n" ,""))
if platform.system().lower() == 'windows':
self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
if (not tdDnodes.dnodes[0].remoteIP == ""):
tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\")
tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\")
self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so')
self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so')
else:
self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf1 = self.libudf1.replace('\r','').replace('\n','')
self.libudf2 = self.libudf2.replace('\r','').replace('\n','')
def prepare_data(self):
@ -136,11 +144,11 @@ class TDTestCase:
for i in range(5):
# create scalar functions
tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1)
# create aggregate functions
tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
functions = tdSql.getResult("show functions")
function_nums = len(functions)
@ -161,11 +169,11 @@ class TDTestCase:
tdLog.info("drop two udf functions success ")
# create scalar functions
tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1)
# create aggregate functions
tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
functions = tdSql.getResult("show functions")
function_nums = len(functions)
@ -533,8 +541,8 @@ class TDTestCase:
tdSql.query("drop function udf2 ")
# create function without buffer
tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int")
tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double")
tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1)
tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2)
udf1_sqls ,udf2_sqls = self.try_query_sql()
for scalar_sql in udf1_sqls:
@ -549,8 +557,8 @@ class TDTestCase:
tdSql.query("drop function udf2 ")
# create function without buffer
tdSql.execute("create aggregate function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
tdSql.execute("create function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1)
tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2)
udf1_sqls ,udf2_sqls = self.try_query_sql()
for scalar_sql in udf1_sqls:
@ -558,8 +566,8 @@ class TDTestCase:
for aggregate_sql in udf2_sqls:
tdSql.error(aggregate_sql)
tdSql.execute(" create function db as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
tdSql.execute(" create aggregate function test as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1)
tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1)
tdSql.error(" select db(c1) from stb1 ")
tdSql.error(" select db(c1,c6), db(c6) from stb1 ")
tdSql.error(" select db(num1,num2), db(num1) from tb ")
@ -607,17 +615,17 @@ class TDTestCase:
tdLog.info(" create function name is not build_in functions ")
tdSql.execute(" drop function udf1 ")
tdSql.execute(" drop function udf2 ")
tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8")
tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8")
tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function tbname as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function function as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function stable as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function union as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function 123 as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function 123db as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create aggregate function mnode as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1)
tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1)
tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2)
tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2)
def restart_taosd_query_udf(self):

20
tests/system-test/2-query/json_tag.py
View File

@ -137,9 +137,9 @@ class TDTestCase:
tdSql.checkRows(9)
tdSql.query("select jtag from jsons1")
tdSql.checkRows(13)
# tdSql.query("select jtag from jsons1 where jtag is null")
tdSql.query("select jtag from jsons1 where jtag is null")
# tdSql.checkRows(5)
# tdSql.query("select jtag from jsons1 where jtag is not null")
tdSql.query("select jtag from jsons1 where jtag is not null")
# tdSql.checkRows(8)
# test jtag is NULL
@ -260,9 +260,9 @@ class TDTestCase:
# tdSql.checkRows(1)
#
# # where json is null
# tdSql.query("select * from jsons1 where jtag is null")
tdSql.query("select * from jsons1 where jtag is null")
# tdSql.checkRows(1)
# tdSql.query("select * from jsons1 where jtag is not null")
tdSql.query("select * from jsons1 where jtag is not null")
# tdSql.checkRows(8)
#
# # where json key is null
@ -389,8 +389,8 @@ class TDTestCase:
tdSql.checkData(2, 1, "11.000000000")
tdSql.checkData(5, 0, 1)
tdSql.checkData(5, 1, "false")
# tdSql.checkData(6, 0, 1)
# tdSql.checkData(6, 1, "null")
tdSql.checkData(6, 0, 1)
tdSql.checkData(6, 1, "null")
tdSql.checkData(7, 0, 2)
tdSql.checkData(7, 1, None)
@ -409,7 +409,7 @@ class TDTestCase:
tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'")
tdSql.checkRows(8)
tdSql.checkData(0, 0, 10)
# tdSql.checkData(0, 1, None)
tdSql.checkData(0, 1, None)
tdSql.checkData(4, 0, 0)
tdSql.checkData(4, 1, "5.000000000")
tdSql.checkData(7, 0, 11)
@ -424,10 +424,10 @@ class TDTestCase:
# test top/bottom with group by json tag
tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'")
tdSql.checkRows(11)
tdSql.checkData(0, 1, None)
tdSql.checkData(2, 0, 4)
tdSql.checkData(3, 0, 3)
tdSql.checkData(3, 1, "false")
# tdSql.checkData(3, 0, 24)
# tdSql.checkData(3, 1, None)
tdSql.checkData(10, 0, 23)
tdSql.checkData(10, 1, '"femail"')
@ -436,7 +436,7 @@ class TDTestCase:
# tdSql.checkRows(2)
# subquery with json tag
tdSql.query("select * from (select jtag, dataint from jsons1)")
tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint")
tdSql.checkRows(11)
tdSql.checkData(1, 1, 1)
tdSql.checkData(2, 0, '{"tag1":5,"tag2":"beijing"}')

4
tests/system-test/fulltest.bat
View File

@ -2,4 +2,6 @@
python3 .\test.py -f 0-others\taosShell.py
python3 .\test.py -f 0-others\taosShellError.py
python3 .\test.py -f 0-others\taosShellNetChk.py
python3 .\test.py -f 0-others\udfTest.py
python3 .\test.py -f 0-others\udfTest.py
python3 .\test.py -f 0-others\udf_create.py
python3 .\test.py -f 0-others\udf_restart_taosd.py