xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor:add schemaless function

This commit is contained in:
wangmm0220 2022-04-29 17:39:33 +08:00
parent a4629e56ce
commit ec97d05300
5 changed files with 657 additions and 534 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/client/taos.h
View File

@ -199,7 +199,7 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr);
#endif
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision, bool dataFormat);
/* --------------------------TMQ INTERFACE------------------------------- */

8
include/libs/parser/parser.h
View File

@ -93,12 +93,12 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf,
int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields);
int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields);
int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen);
void destroyBoundColumnInfo(void* pBoundInfo);
void destroyBoundColumnInfo(void* pBoundInfo);
int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen);
void* tscSmlInitHandle(SQuery *pQuery);
void tscSmlDestroyHandle(void *pHandle);
int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen);
void* smlInitHandle(SQuery *pQuery);
void smlDestroyHandle(void *pHandle);
int32_t smlBindData(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen);
int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);
#ifdef __cplusplus

116
source/client/inc/clientSml.h
View File

@ -1,116 +0,0 @@
/*
* Copyright (c) 2021 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_CLIENTSML_H
#define TDENGINE_CLIENTSML_H
#ifdef __cplusplus
extern "C" {
#endif
#include "thash.h"
#include "clientInt.h"
#include "catalog.h"
typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
typedef struct {
const char* measure;
const char* tags;
const char* cols;
const char* timestamp;
int32_t measureLen;
int32_t measureTagsLen;
int32_t tagsLen;
int32_t colsLen;
int32_t timestampLen;
} TAOS_PARSE_ELEMENTS;
typedef struct {
const char *sTableName; // super table name
uint8_t sTableNameLen;
char childTableName[TSDB_TABLE_NAME_LEN];
uint64_t uid;
SArray *tags;
SArray *cols; // elements are SHashObj<key, SSmlKv*> for find by key quickly
SArray colsColumn; // elements are cols key string
} TAOS_SML_DATA_POINT_TAGS;
typedef struct SSmlSTableMeta {
// char *sTableName; // super table name
// uint8_t sTableNameLen;
uint8_t precision; // the number of precision
SHashObj *tagHash;
SHashObj *fieldHash;
} SSmlSTableMeta;
typedef struct SMsgBuf {
int32_t len;
char *buf;
} SMsgBuf;
typedef struct {
uint64_t id;
SMLProtocolType protocol;
int32_t tsType;
SHashObj *childTables;
SHashObj *superTables;
SHashObj *metaHashObj;
SHashObj *pVgHash;
void *exec;
STscObj *taos;
SCatalog *pCatalog;
SRequestObj *pRequest;
SQuery *pQuery;
int32_t affectedRows;
SMsgBuf msgBuf;
} SSmlLinesInfo;
int smlInsert(TAOS* taos, SSmlLinesInfo* info);
bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info);
bool isValidInteger(char *str);
bool isValidFloat(char *str);
int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info);
bool convertSmlValueType(SSmlKv *pVal, char *value,
uint16_t len, SSmlLinesInfo* info, bool isTag);
int32_t convertSmlTimeStamp(SSmlKv *pVal, char *value,
uint16_t len, SSmlLinesInfo* info);
int sml_insert_lines(TAOS* taos, SRequestObj* request, char* lines[], int numLines, SMLProtocolType protocol,
SMLTimeStampType tsType);
int sml_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol,
SMLTimeStampType tsType, int* affectedRows);
int sml_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol,
SMLTimeStampType tsType, int* affectedRows);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_CLIENTSML_H

1007
source/client/src/clientSml.c
View File

File diff suppressed because it is too large Load Diff

58
source/libs/parser/src/parInsert.c
View File

@ -1514,6 +1514,7 @@ int32_t qBuildStmtColFields(void *pBlock, int32_t *fieldNum, TAOS_FIELD** fields
return TSDB_CODE_SUCCESS;
}
// schemaless logic start
typedef struct SmlExecHandle {
SHashObj* pBlockHash;
@ -1523,7 +1524,7 @@ typedef struct SmlExecHandle {
SVCreateTbReq createTblReq; // each table
SQuery* pQuery;
} SmlExecHandle;
} SSmlExecHandle;
static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSchema* pSchema) {
col_id_t nCols = pColList->numOfCols;
@ -1620,14 +1621,15 @@ static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDat
return TSDB_CODE_SUCCESS;
}
int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) {
int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format,
STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) {
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
int32_t rowNum = taosArrayGetSize(cols);
if(rowNum <= 0) {
return buildInvalidOperationMsg(&pBuf, "cols size <= 0");
}
SmlExecHandle *smlHandle = (SmlExecHandle *)handle;
SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle;
SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta));
int ret = smlBoundColumns(tags, &smlHandle->tags, pTagsSchema);
@ -1651,7 +1653,21 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta
SSchema* pSchema = getTableColumnSchema(pTableMeta);
ret = smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema);
if(format){
ret = smlBoundColumns(taosArrayGetP(colsFormat, 0), &pDataBlock->boundColumnInfo, pSchema);
}else{
SArray *columns = taosArrayInit(16, POINTER_BYTES);
void **p1 = taosHashIterate(colsHash, NULL);
while (p1) {
SSmlKv* kv = *p1;
taosArrayPush(columns, &kv);
p1 = taosHashIterate(colsHash, p1);
}
ret = smlBoundColumns(columns, &pDataBlock->boundColumnInfo, pSchema);
taosArrayDestroy(columns);
}
if(ret != TSDB_CODE_SUCCESS){
buildInvalidOperationMsg(&pBuf, "bound cols error");
return ret;
@ -1671,7 +1687,12 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta
for (int32_t r = 0; r < rowNum; ++r) {
STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header
tdSRowResetBuf(pBuilder, row);
SArray *rowData = taosArrayGetP(cols, r);
void *rowData = NULL;
if(format){
rowData = taosArrayGetP(colsFormat, r);
}else{
rowData = taosArrayGetP(cols, r);
}
// 1. set the parsed value from sql string
for (int c = 0; c < spd->numOfBound; ++c) {
@ -1680,7 +1701,18 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta
param.schema = pColSchema;
getSTSRowAppendInfo(pBuilder->rowType, spd, c, &param.toffset, &param.colIdx);
SSmlKv *kv = taosArrayGetP(rowData, c);
SSmlKv *kv = NULL;
if(format){
kv = taosArrayGetP(rowData, c);
if (!kv){
char msg[64] = {0};
sprintf(msg, "cols num not the same like before:%d", r);
return buildInvalidOperationMsg(&pBuf, msg);
}
}else{
void **p =taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));
kv = *p;
}
if (kv->valueLen == 0) {
MemRowAppend(&pBuf, NULL, 0, &param);
@ -1720,23 +1752,25 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta
return TSDB_CODE_SUCCESS;
}
void* tscSmlInitHandle(SQuery *pQuery){
SmlExecHandle *handle = taosMemoryCalloc(sizeof(SmlExecHandle));
void* smlInitHandle(SQuery *pQuery){
SSmlExecHandle *handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle));
if(!handle) return NULL;
handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
handle->pQuery = pQuery;
return handle;
}
void tscSmlDestroyHandle(void *pHandle){
void smlDestroyHandle(void *pHandle){
if(!pHandle) return;
SmlExecHandle *handle = (SmlExecHandle *)pHandle;
taosHashCleanup(handle->pBlockHash);
SSmlExecHandle *handle = (SSmlExecHandle *)pHandle;
destroyBlockHashmap(handle->pBlockHash);
taosMemoryFree(handle);
}
int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) {
SmlExecHandle *smlHandle = (SmlExecHandle *)handle;
SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle;
return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash);
}
// schemaless logic end