xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh: column/row max length support up to 64K

This commit is contained in:
kailixu 2023-04-03 17:00:52 +08:00
parent 41e29f418d
commit 37bc1bca36
16 changed files with 42 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/examples/c/async_query_example.c
View File

@ -8,7 +8,7 @@
#include <sys/time.h> #include <sys/time.h>
#include <taos.h> #include <taos.h>
typedef int16_t VarDataLenT; typedef uint16_t VarDataLenT;
#define TSDB_NCHAR_SIZE sizeof(int32_t) #define TSDB_NCHAR_SIZE sizeof(int32_t)
#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define VARSTR_HEADER_SIZE sizeof(VarDataLenT)

2
docs/examples/c/query_example.c
View File

@ -6,7 +6,7 @@
#include <string.h> #include <string.h>
#include <taos.h> #include <taos.h>
typedef int16_t VarDataLenT; typedef uint16_t VarDataLenT;
#define TSDB_NCHAR_SIZE sizeof(int32_t) #define TSDB_NCHAR_SIZE sizeof(int32_t)
#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define VARSTR_HEADER_SIZE sizeof(VarDataLenT)

10
include/libs/function/function.h
View File

@ -99,11 +99,11 @@ typedef struct SSubsidiaryResInfo {
} SSubsidiaryResInfo; } SSubsidiaryResInfo;
typedef struct SResultDataInfo { typedef struct SResultDataInfo {
int16_t precision; int16_t precision;
int16_t scale; int16_t scale;
int16_t type; int16_t type;
int16_t bytes; uint16_t bytes;
int32_t interBufSize; int32_t interBufSize;
} SResultDataInfo; } SResultDataInfo;
#define GET_RES_INFO(ctx) ((ctx)->resultInfo) #define GET_RES_INFO(ctx) ((ctx)->resultInfo)

2
include/libs/nodes/cmdnodes.h
View File

@ -30,7 +30,7 @@ extern "C" {
#define SHOW_CREATE_DB_RESULT_COLS 2 #define SHOW_CREATE_DB_RESULT_COLS 2
#define SHOW_CREATE_DB_RESULT_FIELD1_LEN (TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE) #define SHOW_CREATE_DB_RESULT_FIELD1_LEN (TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE)
#define SHOW_CREATE_DB_RESULT_FIELD2_LEN (TSDB_MAX_BINARY_LEN + VARSTR_HEADER_SIZE) #define SHOW_CREATE_DB_RESULT_FIELD2_LEN (TSDB_MAX_BINARY_LEN)
#define SHOW_CREATE_TB_RESULT_COLS 2 #define SHOW_CREATE_TB_RESULT_COLS 2
#define SHOW_CREATE_TB_RESULT_FIELD1_LEN (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) #define SHOW_CREATE_TB_RESULT_FIELD1_LEN (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE)

2
include/libs/parser/parser.h
View File

@ -114,7 +114,7 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta);
int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int16_t msgBufLen); char* msgBuf, int32_t msgBufLen);
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength); int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength);

11
include/util/tdef.h
View File

@ -236,8 +236,11 @@ typedef enum ELogicConditionType {
* - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.
* - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus
* the final value is 65531-(4096-1)*4 = 49151. * the final value is 65531-(4096-1)*4 = 49151.
*
* History value:49151/65531
* - 65531 compatible with 2.0
*/ */
#define TSDB_MAX_BYTES_PER_ROW 49151 #define TSDB_MAX_BYTES_PER_ROW 65531
#define TSDB_MAX_TAGS_LEN 16384 #define TSDB_MAX_TAGS_LEN 16384
#define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAGS 128
@ -406,9 +409,9 @@ typedef enum ELogicConditionType {
#define TSDB_EXPLAIN_RESULT_ROW_SIZE (16 * 1024) #define TSDB_EXPLAIN_RESULT_ROW_SIZE (16 * 1024)
#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN" #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN"
#define TSDB_MAX_FIELD_LEN 16384 #define TSDB_MAX_FIELD_LEN 65519 // compatible with 2.0
#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 #define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384:65519
#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 #define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384:65519
#define PRIMARYKEY_TIMESTAMP_COL_ID 1 #define PRIMARYKEY_TIMESTAMP_COL_ID 1
#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) #define COL_REACH_END(colId, maxColId) ((colId) > (maxColId))

2
source/client/src/clientSmlJson.c
View File

@ -575,7 +575,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) {
uError("OTD:invalid type(%s) for JSON String", typeStr); uError("OTD:invalid type(%s) for JSON String", typeStr);
return TSDB_CODE_TSC_INVALID_JSON_TYPE; return TSDB_CODE_TSC_INVALID_JSON_TYPE;
} }
pVal->length = (int16_t)strlen(value->valuestring); pVal->length = strlen(value->valuestring);
if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;

2
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -475,7 +475,7 @@ RETRIEVE_FUNC_OVER:
return code; return code;
} }
static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t len) { static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t len) {
char *msg = "unknown"; char *msg = "unknown";
if (type >= sizeof(tDataTypes) / sizeof(tDataTypes[0])) { if (type >= sizeof(tDataTypes) / sizeof(tDataTypes[0])) {
return msg; return msg;

2
source/libs/executor/src/timewindowoperator.c
View File

@ -1111,7 +1111,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
bool masterScan = true; bool masterScan = true;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs; int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
int16_t bytes = pStateColInfoData->info.bytes; int32_t bytes = pStateColInfoData->info.bytes;
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
TSKEY* tsList = (TSKEY*)pColInfoData->pData; TSKEY* tsList = (TSKEY*)pColInfoData->pData;

4
source/libs/function/inc/tpercentile.h
View File

@ -53,7 +53,7 @@ typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *va
typedef struct tMemBucket { typedef struct tMemBucket {
int16_t numOfSlots; int16_t numOfSlots;
int16_t type; int16_t type;
int16_t bytes; int32_t bytes;
int32_t total; int32_t total;
int32_t elemPerPage; // number of elements for each object int32_t elemPerPage; // number of elements for each object
int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result
@ -67,7 +67,7 @@ typedef struct tMemBucket {
SHashObj *groupPagesMap; // disk page map for different groups; SHashObj *groupPagesMap; // disk page map for different groups;
} tMemBucket; } tMemBucket;
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval); tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval);
void tMemBucketDestroy(tMemBucket *pBucket); void tMemBucketDestroy(tMemBucket *pBucket);

18
source/libs/function/src/builtinsimpl.c
View File

@ -196,11 +196,11 @@ typedef struct SMavgInfo {
} SMavgInfo; } SMavgInfo;
typedef struct SSampleInfo { typedef struct SSampleInfo {
int32_t samples; int32_t samples;
int32_t totalPoints; int32_t totalPoints;
int32_t numSampled; int32_t numSampled;
uint8_t colType; uint8_t colType;
int16_t colBytes; uint16_t colBytes;
STuplePos nullTuplePos; STuplePos nullTuplePos;
bool nullTupleSaved; bool nullTupleSaved;
@ -220,7 +220,7 @@ typedef struct STailInfo {
int32_t numAdded; int32_t numAdded;
int32_t offset; int32_t offset;
uint8_t colType; uint8_t colType;
int16_t colBytes; uint16_t colBytes;
STailItem** pItems; STailItem** pItems;
} STailInfo; } STailInfo;
@ -233,7 +233,7 @@ typedef struct SUniqueItem {
typedef struct SUniqueInfo { typedef struct SUniqueInfo {
int32_t numOfPoints; int32_t numOfPoints;
uint8_t colType; uint8_t colType;
int16_t colBytes; uint16_t colBytes;
bool hasNull; // null is not hashable, handle separately bool hasNull; // null is not hashable, handle separately
SHashObj* pHash; SHashObj* pHash;
char pItems[]; char pItems[];
@ -247,13 +247,13 @@ typedef struct SModeItem {
typedef struct SModeInfo { typedef struct SModeInfo {
uint8_t colType; uint8_t colType;
int16_t colBytes; uint16_t colBytes;
SHashObj* pHash; SHashObj* pHash;
STuplePos nullTuplePos; STuplePos nullTuplePos;
bool nullTupleSaved; bool nullTupleSaved;
char* buf; // serialize data buffer char* buf; // serialize data buffer
} SModeInfo; } SModeInfo;
typedef struct SDerivInfo { typedef struct SDerivInfo {

2
source/libs/function/src/tpercentile.c
View File

@ -236,7 +236,7 @@ static void resetSlotInfo(tMemBucket *pBucket) {
} }
} }
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) { tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval) {
tMemBucket *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket)); tMemBucket *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket));
if (pBucket == NULL) { if (pBucket == NULL) {
return NULL; return NULL;

2
source/libs/parser/src/parInsertSml.c
View File

@ -242,7 +242,7 @@ end:
int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int16_t msgBufLen) { char* msgBuf, int32_t msgBufLen) {
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
SSchema* pTagsSchema = getTableTagSchema(pTableMeta); SSchema* pTagsSchema = getTableTagSchema(pTableMeta);

16
source/libs/scalar/src/sclfunc.c
View File

@ -12,7 +12,7 @@ typedef double (*_double_fn)(double);
typedef double (*_double_fn_2)(double, double); typedef double (*_double_fn_2)(double, double);
typedef int (*_conv_fn)(int); typedef int (*_conv_fn)(int);
typedef void (*_trim_fn)(char *, char *, int32_t, int32_t); typedef void (*_trim_fn)(char *, char *, int32_t, int32_t);
typedef int16_t (*_len_fn)(char *, int32_t); typedef uint16_t (*_len_fn)(char *, int32_t);
/** Math functions **/ /** Math functions **/
static double tlog(double v) { return log(v); } static double tlog(double v) { return log(v); }
@ -286,9 +286,9 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
} }
/** String functions **/ /** String functions **/
static int16_t tlength(char *input, int32_t type) { return varDataLen(input); } static VarDataLenT tlength(char *input, int32_t type) { return varDataLen(input); }
static int16_t tcharlength(char *input, int32_t type) { static VarDataLenT tcharlength(char *input, int32_t type) {
if (type == TSDB_DATA_TYPE_VARCHAR) { if (type == TSDB_DATA_TYPE_VARCHAR) {
return varDataLen(input); return varDataLen(input);
} else { // NCHAR } else { // NCHAR
@ -377,7 +377,7 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, int16_t *dataLen) { static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen) {
if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) { if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) {
TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1); TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1);
int32_t len = varDataLen(input); int32_t len = varDataLen(input);
@ -457,7 +457,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
continue; continue;
} }
int16_t dataLen = 0; VarDataLenT dataLen = 0;
for (int32_t i = 0; i < inputNum; ++i) { for (int32_t i = 0; i < inputNum; ++i) {
int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k;
input[i] = colDataGetData(pInputData[i], rowIdx); input[i] = colDataGetData(pInputData[i], rowIdx);
@ -526,8 +526,8 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
continue; continue;
} }
int16_t dataLen = 0; VarDataLenT dataLen = 0;
bool hasNull = false; bool hasNull = false;
for (int32_t i = 1; i < inputNum; ++i) { for (int32_t i = 1; i < inputNum; ++i) {
if (colDataIsNull_s(pInputData[i], k) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i]))) { if (colDataIsNull_s(pInputData[i], k) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i]))) {
hasNull = true; hasNull = true;
@ -695,7 +695,7 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
/** Conversion functions **/ /** Conversion functions **/
int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int16_t inputType = GET_PARAM_TYPE(&pInput[0]); int16_t inputType = GET_PARAM_TYPE(&pInput[0]);
int16_t inputLen = GET_PARAM_BYTES(&pInput[0]); int32_t inputLen = GET_PARAM_BYTES(&pInput[0]);
int16_t outputType = GET_PARAM_TYPE(&pOutput[0]); int16_t outputType = GET_PARAM_TYPE(&pOutput[0]);
int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]); int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]);

2
source/util/src/tcompare.c
View File

@ -1241,7 +1241,7 @@ int32_t taosArrayCompareString(const void *a, const void *b) {
int32_t comparestrPatternMatch(const void *pLeft, const void *pRight) { int32_t comparestrPatternMatch(const void *pLeft, const void *pRight) {
SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER; SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER;
ASSERT(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); ASSERT(varDataTLen(pRight) <= TSDB_MAX_FIELD_LEN);
size_t pLen = varDataLen(pRight); size_t pLen = varDataLen(pRight);
size_t sz = varDataLen(pLeft); size_t sz = varDataLen(pLeft);

2
tools/shell/src/shellEngine.c
View File

@ -695,7 +695,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON: { case TSDB_DATA_TYPE_JSON: {
int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; uint16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
if (bytes > shell.args.displayWidth) { if (bytes > shell.args.displayWidth) {
return TMAX(shell.args.displayWidth, width); return TMAX(shell.args.displayWidth, width);
} else { } else {