xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh: column/row max length support up to 64K

This commit is contained in:
kailixu 2023-04-03 17:00:52 +08:00
parent 41e29f418d
commit 37bc1bca36
16 changed files with 42 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/examples/c/async_query_example.c
View File

@ -8,7 +8,7 @@
#include <sys/time.h>
#include <taos.h>
typedef int16_t VarDataLenT;
typedef uint16_t VarDataLenT;
#define TSDB_NCHAR_SIZE sizeof(int32_t)
#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)

2
docs/examples/c/query_example.c
View File

@ -6,7 +6,7 @@
#include <string.h>
#include <taos.h>
typedef int16_t VarDataLenT;
typedef uint16_t VarDataLenT;
#define TSDB_NCHAR_SIZE sizeof(int32_t)
#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)

10
include/libs/function/function.h
View File

@ -99,11 +99,11 @@ typedef struct SSubsidiaryResInfo {
} SSubsidiaryResInfo;
typedef struct SResultDataInfo {
int16_t precision;
int16_t scale;
int16_t type;
int16_t bytes;
int32_t interBufSize;
int16_t precision;
int16_t scale;
int16_t type;
uint16_t bytes;
int32_t interBufSize;
} SResultDataInfo;
#define GET_RES_INFO(ctx) ((ctx)->resultInfo)

2
include/libs/nodes/cmdnodes.h
View File

@ -30,7 +30,7 @@ extern "C" {
#define SHOW_CREATE_DB_RESULT_COLS 2
#define SHOW_CREATE_DB_RESULT_FIELD1_LEN (TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE)
#define SHOW_CREATE_DB_RESULT_FIELD2_LEN (TSDB_MAX_BINARY_LEN + VARSTR_HEADER_SIZE)
#define SHOW_CREATE_DB_RESULT_FIELD2_LEN (TSDB_MAX_BINARY_LEN)
#define SHOW_CREATE_TB_RESULT_COLS 2
#define SHOW_CREATE_TB_RESULT_FIELD1_LEN (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE)

2
include/libs/parser/parser.h
View File

@ -114,7 +114,7 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta);
int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int16_t msgBufLen);
char* msgBuf, int32_t msgBufLen);
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength);

11
include/util/tdef.h
View File

@ -236,8 +236,11 @@ typedef enum ELogicConditionType {
* - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.
* - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus
* the final value is 65531-(4096-1)*4 = 49151.
*
* History value:49151/65531
* - 65531 compatible with 2.0
*/
#define TSDB_MAX_BYTES_PER_ROW 49151
#define TSDB_MAX_BYTES_PER_ROW 65531
#define TSDB_MAX_TAGS_LEN 16384
#define TSDB_MAX_TAGS 128
@ -406,9 +409,9 @@ typedef enum ELogicConditionType {
#define TSDB_EXPLAIN_RESULT_ROW_SIZE (16 * 1024)
#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN"
#define TSDB_MAX_FIELD_LEN 16384
#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384
#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384
#define TSDB_MAX_FIELD_LEN 65519 // compatible with 2.0
#define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384:65519
#define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384:65519
#define PRIMARYKEY_TIMESTAMP_COL_ID 1
#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId))

2
source/client/src/clientSmlJson.c
View File

@ -575,7 +575,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) {
uError("OTD:invalid type(%s) for JSON String", typeStr);
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
pVal->length = (int16_t)strlen(value->valuestring);
pVal->length = strlen(value->valuestring);
if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;

2
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -475,7 +475,7 @@ RETRIEVE_FUNC_OVER:
return code;
}
static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t len) {
static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t len) {
char *msg = "unknown";
if (type >= sizeof(tDataTypes) / sizeof(tDataTypes[0])) {
return msg;

2
source/libs/executor/src/timewindowoperator.c
View File

@ -1111,7 +1111,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
bool masterScan = true;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
int16_t bytes = pStateColInfoData->info.bytes;
int32_t bytes = pStateColInfoData->info.bytes;
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
TSKEY* tsList = (TSKEY*)pColInfoData->pData;

4
source/libs/function/inc/tpercentile.h
View File

@ -53,7 +53,7 @@ typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *va
typedef struct tMemBucket {
int16_t numOfSlots;
int16_t type;
int16_t bytes;
int32_t bytes;
int32_t total;
int32_t elemPerPage; // number of elements for each object
int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result
@ -67,7 +67,7 @@ typedef struct tMemBucket {
SHashObj *groupPagesMap; // disk page map for different groups;
} tMemBucket;
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval);
tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval);
void tMemBucketDestroy(tMemBucket *pBucket);

18
source/libs/function/src/builtinsimpl.c
View File

@ -196,11 +196,11 @@ typedef struct SMavgInfo {
} SMavgInfo;
typedef struct SSampleInfo {
int32_t samples;
int32_t totalPoints;
int32_t numSampled;
uint8_t colType;
int16_t colBytes;
int32_t samples;
int32_t totalPoints;
int32_t numSampled;
uint8_t colType;
uint16_t colBytes;
STuplePos nullTuplePos;
bool nullTupleSaved;
@ -220,7 +220,7 @@ typedef struct STailInfo {
int32_t numAdded;
int32_t offset;
uint8_t colType;
int16_t colBytes;
uint16_t colBytes;
STailItem** pItems;
} STailInfo;
@ -233,7 +233,7 @@ typedef struct SUniqueItem {
typedef struct SUniqueInfo {
int32_t numOfPoints;
uint8_t colType;
int16_t colBytes;
uint16_t colBytes;
bool hasNull; // null is not hashable, handle separately
SHashObj* pHash;
char pItems[];
@ -247,13 +247,13 @@ typedef struct SModeItem {
typedef struct SModeInfo {
uint8_t colType;
int16_t colBytes;
uint16_t colBytes;
SHashObj* pHash;
STuplePos nullTuplePos;
bool nullTupleSaved;
char* buf; // serialize data buffer
char* buf; // serialize data buffer
} SModeInfo;
typedef struct SDerivInfo {

2
source/libs/function/src/tpercentile.c
View File

@ -236,7 +236,7 @@ static void resetSlotInfo(tMemBucket *pBucket) {
}
}
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) {
tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval) {
tMemBucket *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket));
if (pBucket == NULL) {
return NULL;

2
source/libs/parser/src/parInsertSml.c
View File

@ -242,7 +242,7 @@ end:
int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int16_t msgBufLen) {
char* msgBuf, int32_t msgBufLen) {
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
SSchema* pTagsSchema = getTableTagSchema(pTableMeta);

16
source/libs/scalar/src/sclfunc.c
View File

@ -12,7 +12,7 @@ typedef double (*_double_fn)(double);
typedef double (*_double_fn_2)(double, double);
typedef int (*_conv_fn)(int);
typedef void (*_trim_fn)(char *, char *, int32_t, int32_t);
typedef int16_t (*_len_fn)(char *, int32_t);
typedef uint16_t (*_len_fn)(char *, int32_t);
/** Math functions **/
static double tlog(double v) { return log(v); }
@ -286,9 +286,9 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
}
/** String functions **/
static int16_t tlength(char *input, int32_t type) { return varDataLen(input); }
static VarDataLenT tlength(char *input, int32_t type) { return varDataLen(input); }
static int16_t tcharlength(char *input, int32_t type) {
static VarDataLenT tcharlength(char *input, int32_t type) {
if (type == TSDB_DATA_TYPE_VARCHAR) {
return varDataLen(input);
} else { // NCHAR
@ -377,7 +377,7 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
return TSDB_CODE_SUCCESS;
}
static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, int16_t *dataLen) {
static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen) {
if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) {
TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1);
int32_t len = varDataLen(input);
@ -457,7 +457,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
continue;
}
int16_t dataLen = 0;
VarDataLenT dataLen = 0;
for (int32_t i = 0; i < inputNum; ++i) {
int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k;
input[i] = colDataGetData(pInputData[i], rowIdx);
@ -526,8 +526,8 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
continue;
}
int16_t dataLen = 0;
bool hasNull = false;
VarDataLenT dataLen = 0;
bool hasNull = false;
for (int32_t i = 1; i < inputNum; ++i) {
if (colDataIsNull_s(pInputData[i], k) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i]))) {
hasNull = true;
@ -695,7 +695,7 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
/** Conversion functions **/
int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int16_t inputType = GET_PARAM_TYPE(&pInput[0]);
int16_t inputLen = GET_PARAM_BYTES(&pInput[0]);
int32_t inputLen = GET_PARAM_BYTES(&pInput[0]);
int16_t outputType = GET_PARAM_TYPE(&pOutput[0]);
int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]);

2
source/util/src/tcompare.c
View File

@ -1241,7 +1241,7 @@ int32_t taosArrayCompareString(const void *a, const void *b) {
int32_t comparestrPatternMatch(const void *pLeft, const void *pRight) {
SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER;
ASSERT(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN);
ASSERT(varDataTLen(pRight) <= TSDB_MAX_FIELD_LEN);
size_t pLen = varDataLen(pRight);
size_t sz = varDataLen(pLeft);

2
tools/shell/src/shellEngine.c
View File

@ -695,7 +695,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON: {
int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
uint16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
if (bytes > shell.args.displayWidth) {
return TMAX(shell.args.displayWidth, width);
} else {