xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor:add schemaless function

This commit is contained in:
wangmm0220 2022-04-29 10:58:46 +08:00
parent 990205d684
commit a4629e56ce
5 changed files with 545 additions and 904 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/common/tcommon.h
View File

@ -246,7 +246,13 @@ typedef struct {
int32_t keyLen; int32_t keyLen;
uint8_t type; uint8_t type;
int16_t length; int16_t length;
union{
const char* value; const char* value;
int64_t i;
uint64_t u;
double d;
float f;
};
int32_t valueLen; int32_t valueLen;
} SSmlKv; } SSmlKv;

16
include/common/ttypes.h
View File

@ -186,14 +186,14 @@ typedef struct {
#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)))
#define IS_MATHABLE_TYPE(_t) (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) #define IS_MATHABLE_TYPE(_t) (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
#define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
#define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) #define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
#define IS_VALID_INT(_t) ((_t) > INT32_MIN && (_t) <= INT32_MAX) #define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX)
#define IS_VALID_BIGINT(_t) ((_t) > INT64_MIN && (_t) <= INT64_MAX) #define IS_VALID_BIGINT(_t) ((_t) >= INT64_MIN && (_t) <= INT64_MAX)
#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) < UINT8_MAX) #define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) <= UINT8_MAX)
#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX) #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX)
#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) #define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) <= UINT32_MAX)
#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) #define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) <= UINT64_MAX)
#define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
#define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)

37
source/client/inc/clientSml.h
View File

@ -36,6 +36,7 @@ typedef struct {
int32_t measureTagsLen; int32_t measureTagsLen;
int32_t tagsLen; int32_t tagsLen;
int32_t colsLen; int32_t colsLen;
int32_t timestampLen;
} TAOS_PARSE_ELEMENTS; } TAOS_PARSE_ELEMENTS;
typedef struct { typedef struct {
@ -44,40 +45,46 @@ typedef struct {
char childTableName[TSDB_TABLE_NAME_LEN]; char childTableName[TSDB_TABLE_NAME_LEN];
uint64_t uid; uint64_t uid;
SArray* tags; SArray *tags;
SArray *cols; SArray *cols; // elements are SHashObj<key, SSmlKv*> for find by key quickly
SArray colsColumn; // elements are cols key string
} TAOS_SML_DATA_POINT_TAGS; } TAOS_SML_DATA_POINT_TAGS;
typedef struct SSmlSTableMeta { typedef struct SSmlSTableMeta {
// char *sTableName; // super table name // char *sTableName; // super table name
// uint8_t sTableNameLen; // uint8_t sTableNameLen;
uint8_t precision; // the number of precision uint8_t precision; // the number of precision
SHashObj* tagHash; SHashObj *tagHash;
SHashObj* fieldHash; SHashObj *fieldHash;
} SSmlSTableMeta; } SSmlSTableMeta;
typedef struct SMsgBuf {
int32_t len;
char *buf;
} SMsgBuf;
typedef struct { typedef struct {
uint64_t id; uint64_t id;
SMLProtocolType protocol; SMLProtocolType protocol;
int32_t tsType; int32_t tsType;
SHashObj* childTables; SHashObj *childTables;
SHashObj* superTables; SHashObj *superTables;
SHashObj* metaHashObj; SHashObj *metaHashObj;
SHashObj* pVgHash; SHashObj *pVgHash;
void* exec; void *exec;
STscObj* taos; STscObj *taos;
SCatalog* pCatalog; SCatalog *pCatalog;
SRequestObj* pRequest; SRequestObj *pRequest;
SQuery* pQuery; SQuery *pQuery;
int32_t affectedRows; int32_t affectedRows;
char *msgBuf; SMsgBuf msgBuf;
int16_t msgLen;
} SSmlLinesInfo; } SSmlLinesInfo;
int smlInsert(TAOS* taos, SSmlLinesInfo* info); int smlInsert(TAOS* taos, SSmlLinesInfo* info);

1333
source/client/src/clientSml.c
View File

File diff suppressed because it is too large Load Diff

39
source/libs/parser/src/parInsert.c
View File

@ -1539,7 +1539,7 @@ static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSche
col_id_t lastColIdx = -1; // last column found col_id_t lastColIdx = -1; // last column found
for (int i = 0; i < taosArrayGetSize(cols); ++i) { for (int i = 0; i < taosArrayGetSize(cols); ++i) {
SSmlKv *kv = taosArrayGetP(cols, i); SSmlKv *kv = taosArrayGetP(cols, i);
SToken sToken = {.n=kv->keyLen, .z=kv->key}; SToken sToken = {.n=kv->keyLen, .z=(char*)kv->key};
col_id_t t = lastColIdx + 1; col_id_t t = lastColIdx + 1;
col_id_t index = findCol(&sToken, t, nCols, pSchema); col_id_t index = findCol(&sToken, t, nCols, pSchema);
if (index < 0 && t > 0) { if (index < 0 && t > 0) {
@ -1596,18 +1596,17 @@ static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSche
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SVCreateTbReq *createTblReq) { static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SVCreateTbReq *createTblReq, SMsgBuf *msg) {
if (tdInitKVRowBuilder(tagsBuilder) < 0) { if (tdInitKVRowBuilder(tagsBuilder) < 0) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
SKvParam param = {.builder = tagsBuilder}; SKvParam param = {.builder = tagsBuilder};
for (int i = 0; i < tags->numOfBound; ++i) { for (int i = 0; i < tags->numOfBound; ++i) {
SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1 SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1
param.schema = pTagSchema; param.schema = pTagSchema;
SSmlKv *kv = taosArrayGetP(cols, i); SSmlKv *kv = taosArrayGetP(cols, i);
KvRowAppend(NULL, kv->value, kv->valueLen, &param) ; KvRowAppend(msg, kv->value, kv->valueLen, &param) ;
} }
@ -1630,18 +1629,33 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta
SmlExecHandle *smlHandle = (SmlExecHandle *)handle; SmlExecHandle *smlHandle = (SmlExecHandle *)handle;
SSchema* pTagsSchema = getTableTagSchema(pTableMeta); SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
smlBoundColumns(tags, &smlHandle->tags, pTagsSchema); setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta));
smlParseTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &smlHandle->createTblReq); int ret = smlBoundColumns(tags, &smlHandle->tags, pTagsSchema);
if(ret != TSDB_CODE_SUCCESS){
buildInvalidOperationMsg(&pBuf, "bound tags error");
return ret;
}
ret = smlParseTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &smlHandle->createTblReq, &pBuf);
if(ret != TSDB_CODE_SUCCESS){
return ret;
}
STableDataBlocks* pDataBlock = NULL; STableDataBlocks* pDataBlock = NULL;
getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, ret = getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE,
sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta,
&pDataBlock, NULL, &smlHandle->createTblReq); &pDataBlock, NULL, &smlHandle->createTblReq);
if(ret != TSDB_CODE_SUCCESS){
buildInvalidOperationMsg(&pBuf, "create data block error");
return ret;
}
SSchema* pSchema = getTableColumnSchema(pTableMeta); SSchema* pSchema = getTableColumnSchema(pTableMeta);
smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema); ret = smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema);
if(ret != TSDB_CODE_SUCCESS){
buildInvalidOperationMsg(&pBuf, "bound cols error");
return ret;
}
int32_t extendedRowSize = getExtendedRowSize(pDataBlock); int32_t extendedRowSize = getExtendedRowSize(pDataBlock);
SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo;
SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SRowBuilder* pBuilder = &pDataBlock->rowBuilder;
@ -1649,8 +1663,11 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta
initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo);
allocateMemForSize(pDataBlock, extendedRowSize * rowNum); ret = allocateMemForSize(pDataBlock, extendedRowSize * rowNum);
if(ret != TSDB_CODE_SUCCESS){
buildInvalidOperationMsg(&pBuf, "allocate memory error");
return ret;
}
for (int32_t r = 0; r < rowNum; ++r) { for (int32_t r = 0; r < rowNum; ++r) {
STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header
tdSRowResetBuf(pBuilder, row); tdSRowResetBuf(pBuilder, row);