xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into feature/crash_gen

This commit is contained in:
Steven Li 2020-06-24 21:59:23 -07:00
parent 5f979568e1 071704922b
commit 89016a8a7e
129 changed files with 3801 additions and 2092 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
.travis.yml
View File

@ -135,92 +135,6 @@ matrix:
# https://scan.coverity.com/faq#frequency
branch_pattern: coverity_scan
- os: linux
dist: bionic
language: c
compiler: gcc
env: ENV_COVER=true
git:
- depth: 1
addons:
apt:
packages:
- build-essential
- cmake
- net-tools
- python-pip
- python-setuptools
- python3-pip
- python3-setuptools
- lcov
- psmisc
before_script:
- cd ${TRAVIS_BUILD_DIR}
- mkdir debug
- cd debug
script:
- cmake -DCOVER=true .. > /dev/null
- make > /dev/null
after_success:
- |-
case $TRAVIS_OS_NAME in
linux)
cd ${TRAVIS_BUILD_DIR}/debug
make install > /dev/null || travis_terminate $?
pip install numpy
pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
pip3 install numpy
pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
cd ${TRAVIS_BUILD_DIR}/tests
./test-all.sh smoke COVER
TEST_RESULT=$?
pkill taosd
sleep 1
cd ${TRAVIS_BUILD_DIR}
lcov -d . --capture --rc lcov_branch_coverage=1 -o coverage.info
lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' -o coverage.info
lcov -l --rc lcov_branch_coverage=1 coverage.info || travis_terminate $?
gem install coveralls-lcov
# Color setting
RED='\033[0;31m'
GREEN='\033[1;32m'
GREEN_DARK='\033[0;32m'
GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m'
coveralls-lcov coverage.info
if [ "$?" -eq "0" ]; then
echo -e "${GREEN} ## Uploaded to Coveralls.io! ## ${NC}"
else
echo -e "${RED} ## Coveralls.io not collect coverage report! ## ${NC} "
fi
bash <(curl -s https://codecov.io/bash) -y .codecov.yml -f coverage.info
if [ "$?" -eq "0" ]; then
echo -e "${GREEN} ## Uploaded to Codecov! ## ${NC} "
else
echo -e "${RED} ## Codecov did not collect coverage report! ## ${NC} "
fi
if [ "$TEST_RESULT" -ne "0" ]; then
travis_terminate $?
fi
;;
esac
- os: linux
dist: trusty
language: c

1
README.md
View File

@ -1,5 +1,6 @@
[![Build Status](https://travis-ci.org/taosdata/TDengine.svg?branch=master)](https://travis-ci.org/taosdata/TDengine)
[![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master)
[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
[![TDengine](TDenginelogo.png)](https://www.taosdata.com)

14
src/client/inc/tscUtil.h
View File

@ -89,27 +89,21 @@ typedef struct SVgroupTableInfo {
int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name,
STableMeta* pTableMeta, STableDataBlocks** dataBlocks);
void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks);
void tscDestroyDataBlock(STableDataBlocks* pDataBlock);
void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf);
SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes,
uint32_t offset);
SDataBlockList* tscCreateBlockArrayList();
void* tscDestroyBlockArrayList(SDataBlockList* pList);
void* tscDestroyBlockArrayList(SArray* pDataBlockList);
int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
void tscFreeUnusedDataBlocks(SDataBlockList* pList);
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pDataList);
int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int64_t id, int32_t size,
void tscFreeUnusedDataBlocks(SArray* pDataBlockList);
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pDataList);
int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t id, int32_t size,
int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta,
STableDataBlocks** dataBlocks);
//UNUSED_FUNC STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx);
/**
*
* for the projection query on metric or point interpolation query on metric,
* we iterate all the meters, instead of invoke query on all qualified meters simultaneously.
*

21
src/client/inc/tsclient.h
View File

@ -171,11 +171,7 @@ typedef struct STableDataBlocks {
* to avoid it to be removed from cache
*/
STableMeta *pTableMeta;
union {
char *filename;
char *pData;
};
char *pData;
// for parameter ('?') binding
uint32_t numOfAllocedParams;
@ -183,11 +179,11 @@ typedef struct STableDataBlocks {
SParamInfo *params;
} STableDataBlocks;
typedef struct SDataBlockList { // todo remove
uint32_t nSize;
uint32_t nAlloc;
STableDataBlocks **pData;
} SDataBlockList;
//typedef struct SDataBlockList { // todo remove
// uint32_t nSize;
// uint32_t nAlloc;
// STableDataBlocks **pData;
//} SDataBlockList;
typedef struct SQueryInfo {
int16_t command; // the command may be different for each subclause, so keep it seperately.
@ -238,8 +234,7 @@ typedef struct {
void * pTableList; // referred table involved in sql
int32_t batchSize; // for parameter ('?') binding and batch processing
int32_t numOfParams;
SDataBlockList *pDataBlocks; // submit data blocks after parsing sql
SArray *pDataBlocks; // SArray<STableDataBlocks*> submit data blocks after parsing sql
} SSqlCmd;
typedef struct SResRec {
@ -399,7 +394,7 @@ void waitForQueryRsp(void *param, TAOS_RES *tres, int code) ;
int doAsyncParseSql(SSqlObj* pSql);
void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen);
void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql);
void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql);
void tscKillSTableQuery(SSqlObj *pSql);
void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
bool tscIsUpdateQuery(SSqlObj* pSql);

9
src/client/src/TSDBJNIConnector.c
View File

@ -565,6 +565,11 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNI
sql = (char *)(*env)->GetStringUTFChars(env, jsql, NULL);
}
if (topic == NULL || sql == NULL) {
jniTrace("jobj:%p, invalid argument: topic or sql is NULL", jobj);
return sub;
}
TAOS_SUB *tsub = taos_subscribe(taos, (int)restart, topic, sql, NULL, NULL, jinterval);
sub = (jlong)tsub;
@ -574,8 +579,8 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNI
jniTrace("jobj:%p, successfully subscribe: topic: %s", jobj, topic);
}
if (topic != NULL) (*env)->ReleaseStringUTFChars(env, jtopic, topic);
if (sql != NULL) (*env)->ReleaseStringUTFChars(env, jsql, sql);
(*env)->ReleaseStringUTFChars(env, jtopic, topic);
(*env)->ReleaseStringUTFChars(env, jsql, sql);
return sub;
}

2
src/client/src/tscLocalMerge.c
View File

@ -123,7 +123,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
}
}
if (n == 0) {
if (n == 0 || pCtx == NULL) {
free(pTagCtx);
} else {
pCtx->tagInfo.pTagCtxList = pTagCtx;

319
src/client/src/tscParseInsert.c
View File

@ -46,18 +46,28 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) {
return TK_ILLEGAL;
}
int32_t radix = 10;
int32_t radixList[3] = {16, 8, 2}; // the integer number with different radix: hex, oct, bin
if (pToken->type == TK_HEX || pToken->type == TK_OCT || pToken->type == TK_BIN) {
radix = radixList[pToken->type - TK_HEX];
}
int32_t radix = 10;
if (pToken->type == TK_HEX) {
radix = 16;
} else if (pToken->type == TK_BIN) {
radix = 2;
}
errno = 0;
*value = strtoll(pToken->z, endPtr, radix);
if (**endPtr == 'e' || **endPtr == 'E' || **endPtr == '.') {
errno = 0;
double v = round(strtod(pToken->z, endPtr));
if (v > INT64_MAX || v <= INT64_MIN) {
errno = ERANGE;
} else {
*value = v;
}
}
// not a valid integer number, return error
if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
if (*endPtr - pToken->z != pToken->n) {
return TK_ILLEGAL;
}
@ -73,11 +83,11 @@ static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) {
*value = strtod(pToken->z, endPtr);
// not a valid integer number, return error
if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
if ((*endPtr - pToken->z) != pToken->n) {
return TK_ILLEGAL;
} else {
return pToken->type;
}
return pToken->type;
}
int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) {
@ -781,12 +791,12 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
sToken = tStrGetToken(sql, &index, false, 0, NULL);
sql += index;
STagData *pTag = (STagData *)pCmd->payload;
tscAllocPayload(pCmd, sizeof(STagData));
STagData *pTag = (STagData *) pCmd->payload;
memset(pTag, 0, sizeof(STagData));
/*
* the source super table is moved to the secondary position of the pTableMetaInfo list
*/
//the source super table is moved to the secondary position of the pTableMetaInfo list
if (pQueryInfo->numOfTables < 2) {
tscAddEmptyMetaInfo(pQueryInfo);
}
@ -887,9 +897,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
index = 0;
sToken = tStrGetToken(sql, &index, true, numOfIgnoreToken, &ignoreTokenTypes);
sql += index;
if (sToken.n == 0) {
break;
} else if (sToken.type == TK_RP) {
if (sToken.n == 0 || sToken.type == TK_RP) {
break;
}
@ -903,11 +912,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if ((pTagSchema[colIndex].type == TSDB_DATA_TYPE_BINARY || pTagSchema[colIndex].type == TSDB_DATA_TYPE_NCHAR) &&
sToken.n > pTagSchema[colIndex].bytes) {
return tscInvalidSQLErrMsg(pCmd->payload, "string too long", sToken.z);
}
}
index = 0;
@ -986,14 +990,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
return code;
}
int validateTableName(char *tblName, int len) {
char buf[TSDB_TABLE_ID_LEN] = {0};
tstrncpy(buf, tblName, sizeof(buf));
int validateTableName(char *tblName, int len, SSQLToken* psTblToken) {
tstrncpy(psTblToken->z, tblName, TSDB_TABLE_ID_LEN);
SSQLToken token = {.n = len, .type = TK_ID, .z = buf};
tSQLGetToken(buf, &token.type);
psTblToken->n = len;
psTblToken->type = TK_ID;
tSQLGetToken(psTblToken->z, &psTblToken->type);
return tscValidateName(&token);
return tscValidateName(psTblToken);
}
static int32_t validateDataSource(SSqlCmd *pCmd, int8_t type, const char *sql) {
@ -1031,15 +1035,13 @@ int tsParseInsertSql(SSqlObj *pSql) {
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
}
// TODO: 2048 is added because TSDB_MAX_TAGS_LEN now is 65536, but TSDB_PAYLOAD_SIZE is 65380
if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE + 2048)) != TSDB_CODE_SUCCESS) {
if ((code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) {
return code;
}
if (NULL == pCmd->pTableList) {
pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
pSql->cmd.pDataBlocks = tscCreateBlockArrayList();
pCmd->pDataBlocks = taosArrayInit(4, POINTER_BYTES);
if (NULL == pCmd->pTableList || NULL == pSql->cmd.pDataBlocks) {
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto _error;
@ -1077,14 +1079,16 @@ int tsParseInsertSql(SSqlObj *pSql) {
}
pCmd->curSql = sToken.z;
char buf[TSDB_TABLE_ID_LEN];
SSQLToken sTblToken;
sTblToken.z = buf;
// Check if the table name available or not
if (validateTableName(sToken.z, sToken.n) != TSDB_CODE_SUCCESS) {
if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) {
code = tscInvalidSQLErrMsg(pCmd->payload, "table name invalid", sToken.z);
goto _error;
}
if ((code = tscSetTableFullName(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) {
if ((code = tscSetTableFullName(pTableMetaInfo, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1149,29 +1153,19 @@ int tsParseInsertSql(SSqlObj *pSql) {
goto _error;
}
char fname[PATH_MAX] = {0};
strncpy(fname, sToken.z, sToken.n);
strdequote(fname);
strncpy(pCmd->payload, sToken.z, sToken.n);
strdequote(pCmd->payload);
// todo refactor extract method
wordexp_t full_path;
if (wordexp(fname, &full_path, 0) != 0) {
if (wordexp(pCmd->payload, &full_path, 0) != 0) {
code = tscInvalidSQLErrMsg(pCmd->payload, "invalid filename", sToken.z);
goto _error;
}
strcpy(fname, full_path.we_wordv[0]);
tstrncpy(pCmd->payload, full_path.we_wordv[0], pCmd->allocSize);
wordfree(&full_path);
STableDataBlocks *pDataBlock = NULL;
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
int32_t ret = tscCreateDataBlock(PATH_MAX, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name,
pTableMeta, &pDataBlock);
if (ret != TSDB_CODE_SUCCESS) {
goto _error;
}
tscAppendDataBlock(pCmd->pDataBlocks, pDataBlock);
strcpy(pDataBlock->filename, fname);
} else if (sToken.type == TK_LP) {
/* insert into tablename(col1, col2,..., coln) values(v1, v2,... vn); */
STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta;
@ -1258,7 +1252,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
goto _clean;
}
if (pCmd->pDataBlocks->nSize > 0) { // merge according to vgId
if (taosArrayGetSize(pCmd->pDataBlocks) > 0) { // merge according to vgId
if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1368,177 +1362,146 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock
return code;
}
// the pDataBlock is different from the pTableDataBlocks
STableDataBlocks *pDataBlock = pCmd->pDataBlocks->pData[0];
STableDataBlocks *pDataBlock = taosArrayGetP(pCmd->pDataBlocks, 0);
if ((code = tscCopyDataBlockToPayload(pSql, pDataBlock)) != TSDB_CODE_SUCCESS) {
return code;
}
if ((code = tscProcessSql(pSql)) != TSDB_CODE_SUCCESS) {
return code;
}
return TSDB_CODE_SUCCESS;
return tscProcessSql(pSql);
}
static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) {
size_t readLen = 0;
char * line = NULL;
size_t n = 0;
int len = 0;
int32_t maxRows = 0;
SSqlCmd * pCmd = &pSql->cmd;
int numOfRows = 0;
int32_t code = 0;
int nrows = 0;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
assert(pCmd->numOfClause == 1);
int32_t rowSize = tinfo.rowSize;
typedef struct SImportFileSupport {
SSqlObj *pSql;
FILE *fp;
} SImportFileSupport;
pCmd->pDataBlocks = tscCreateBlockArrayList();
STableDataBlocks *pTableDataBlock = NULL;
int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, rowSize, sizeof(SSubmitBlk),
pTableMetaInfo->name, pTableMeta, &pTableDataBlock);
if (ret != TSDB_CODE_SUCCESS) {
return -1;
static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
assert(param != NULL && tres != NULL);
SSqlObj *pSql = tres;
SSqlCmd *pCmd = &pSql->cmd;
SImportFileSupport *pSupporter = (SImportFileSupport *) param;
SSqlObj *pParentSql = pSupporter->pSql;
FILE *fp = pSupporter->fp;
if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { // handle error
assert(taos_errno(pSql) == code);
taos_free_result(pSql);
tfree(pSupporter);
fclose(fp);
pParentSql->res.code = code;
return;
}
tscAppendDataBlock(pCmd->pDataBlocks, pTableDataBlock);
// accumulate the total submit records
pParentSql->res.numOfRows += pSql->res.numOfRows;
code = tscAllocateMemIfNeed(pTableDataBlock, rowSize, &maxRows);
if (TSDB_CODE_SUCCESS != code) return -1;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
SSchema * pSchema = tscGetTableSchema(pTableMeta);
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
int count = 0;
SParsedDataColInfo spd = {.numOfCols = tinfo.numOfColumns};
SSchema * pSchema = tscGetTableSchema(pTableMeta);
tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns);
size_t n = 0;
ssize_t readLen = 0;
char * line = NULL;
int32_t count = 0;
int32_t maxRows = 0;
tscDestroyBlockArrayList(pSql->cmd.pDataBlocks);
pCmd->pDataBlocks = taosArrayInit(1, POINTER_BYTES);
STableDataBlocks *pTableDataBlock = NULL;
int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name, pTableMeta, &pTableDataBlock);
if (ret != TSDB_CODE_SUCCESS) {
// return ret;
}
taosArrayPush(pCmd->pDataBlocks, &pTableDataBlock);
tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows);
char *tokenBuf = calloc(1, 4096);
while ((readLen = getline(&line, &n, fp)) != -1) {
// line[--readLen] = '\0';
if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) line[--readLen] = 0;
if (readLen == 0) continue; // fang, <= to ==
if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
line[--readLen] = 0;
}
if (readLen == 0) {
continue;
}
char *lineptr = line;
strtolower(line, line);
len = tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd->payload, tinfo.precision, &code, tmpTokenBuf);
int32_t len =
tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd->payload, tinfo.precision, &code, tokenBuf);
if (len <= 0 || pTableDataBlock->numOfParams > 0) {
pSql->res.code = code;
return (-code);
break;
}
pTableDataBlock->size += len;
count++;
nrows++;
if (count >= maxRows) {
if ((code = doPackSendDataBlock(pSql, count, pTableDataBlock)) != TSDB_CODE_SUCCESS) {
return -code;
}
pTableDataBlock = pCmd->pDataBlocks->pData[0];
pTableDataBlock->size = sizeof(SSubmitBlk);
pTableDataBlock->rowSize = tinfo.rowSize;
numOfRows += pSql->res.numOfRows;
pSql->res.numOfRows = 0;
count = 0;
if (++count >= maxRows) {
break;
}
}
tfree(tokenBuf);
free(line);
if (count > 0) {
if ((code = doPackSendDataBlock(pSql, count, pTableDataBlock)) != TSDB_CODE_SUCCESS) {
return -code;
pParentSql->res.code = code;
}
numOfRows += pSql->res.numOfRows;
pSql->res.numOfRows = 0;
} else {
taos_free_result(pSql);
tfree(pSupporter);
fclose(fp);
pParentSql->fp = pParentSql->fetchFp;
// all data has been sent to vnode, call user function
int32_t v = (pParentSql->res.code != TSDB_CODE_SUCCESS) ? pParentSql->res.code : pParentSql->res.numOfRows;
(*pParentSql->fp)(pParentSql->param, pParentSql, v);
}
if (line) tfree(line);
return numOfRows;
}
void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) {
void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) {
SSqlCmd *pCmd = &pSql->cmd;
if (pCmd->command != TSDB_SQL_INSERT) {
return;
}
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
assert(pCmd->dataSourceType == DATA_FROM_DATA_FILE && strlen(pCmd->payload) != 0);
STableDataBlocks *pDataBlock = NULL;
int32_t affected_rows = 0;
SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport));
SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL);
assert(pCmd->dataSourceType == DATA_FROM_DATA_FILE && pCmd->pDataBlocks != NULL);
SDataBlockList *pDataBlockList = pCmd->pDataBlocks;
pCmd->pDataBlocks = NULL;
pNew->cmd.pDataBlocks = taosArrayInit(4, POINTER_BYTES);
pCmd->count = 1;
char path[PATH_MAX] = {0};
FILE *fp = fopen(pCmd->payload, "r");
if (fp == NULL) {
pSql->res.code = TAOS_SYSTEM_ERROR(errno);
tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code));
for (int32_t i = 0; i < pDataBlockList->nSize; ++i) {
pDataBlock = pDataBlockList->pData[i];
if (pDataBlock == NULL) {
continue;
}
tfree(pSupporter)
tscQueueAsyncRes(pSql);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE)) {
tscError("%p failed to malloc when insert file", pSql);
continue;
}
pCmd->count = 1;
tstrncpy(path, pDataBlock->filename, sizeof(path));
FILE *fp = fopen(path, "r");
if (fp == NULL) {
tscError("%p failed to open file %s to load data from file, reason:%s", pSql, path, strerror(errno));
continue;
}
tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
memset(pDataBlock->pData, 0, pDataBlock->nAllocSize);
int32_t ret = tscGetTableMeta(pSql, pTableMetaInfo);
if (ret != TSDB_CODE_SUCCESS) {
tscError("%p get meter meta failed, abort", pSql);
continue;
}
char *tmpTokenBuf = calloc(1, 4096); // used for deleting Escape character: \\, \', \"
if (NULL == tmpTokenBuf) {
tscError("%p calloc failed", pSql);
continue;
}
int nrows = tscInsertDataFromFile(pSql, fp, tmpTokenBuf);
free(tmpTokenBuf);
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
if (nrows < 0) {
fclose(fp);
tscTrace("%p no records(%d) in file %s", pSql, nrows, path);
continue;
}
fclose(fp);
affected_rows += nrows;
tscTrace("%p Insert data %d records from file %s", pSql, nrows, path);
return;
}
pSql->res.numOfRows = affected_rows;
pSupporter->pSql = pSql;
pSupporter->fp = fp;
// all data have been submit to vnode, release data blocks
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
tscDestroyBlockArrayList(pDataBlockList);
parseFileSendDataBlock(pSupporter, pNew, 0);
}

22
src/client/src/tscPrepare.c
View File

@ -331,8 +331,9 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
binded = pCmd->batchSize / 2;
}
for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) {
STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i];
size_t size = taosArrayGetSize(pCmd->pDataBlocks);
for (int32_t i = 0; i < size; ++i) {
STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, i);
uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk);
uint32_t dataSize = totalDataSize / alloced;
assert(dataSize * alloced == totalDataSize);
@ -370,8 +371,9 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) {
STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i];
size_t total = taosArrayGetSize(pCmd->pDataBlocks);
for (int32_t i = 0; i < total; ++i) {
STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, i);
uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk);
pBlock->size += totalDataSize / alloced;
@ -395,8 +397,10 @@ static int insertStmtReset(STscStmt* pStmt) {
SSqlCmd* pCmd = &pStmt->pSql->cmd;
if (pCmd->batchSize > 2) {
int32_t alloced = (pCmd->batchSize + 1) / 2;
for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) {
STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i];
size_t size = taosArrayGetSize(pCmd->pDataBlocks);
for (int32_t i = 0; i < size; ++i) {
STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, i);
uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk);
pBlock->size = sizeof(SSubmitBlk) + totalDataSize / alloced;
@ -423,15 +427,15 @@ static int insertStmtExecute(STscStmt* stmt) {
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
assert(pCmd->numOfClause == 1);
if (pCmd->pDataBlocks->nSize > 0) {
if (taosArrayGetSize(pCmd->pDataBlocks) > 0) {
// merge according to vgid
int code = tscMergeTableDataBlocks(stmt->pSql, pCmd->pDataBlocks);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
STableDataBlocks *pDataBlock = pCmd->pDataBlocks->pData[0];
STableDataBlocks *pDataBlock = taosArrayGetP(pCmd->pDataBlocks, 0);
code = tscCopyDataBlockToPayload(stmt->pSql, pDataBlock);
if (code != TSDB_CODE_SUCCESS) {
return code;

35
src/client/src/tscSQLParser.c
View File

@ -1483,6 +1483,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
const char* msg6 = "function applied to tags not allowed";
const char* msg7 = "normal table can not apply this function";
const char* msg8 = "multi-columns selection does not support alias column name";
const char* msg9 = "invalid function";
switch (optr) {
case TK_COUNT: {
@ -1683,7 +1684,9 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
bool requireAllFields = (pItem->pNode->pParam == NULL);
int16_t functionID = 0;
changeFunctionID(optr, &functionID);
if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(pQueryInfo->msg, msg9);
}
if (!requireAllFields) {
if (pItem->pNode->pParam->nExpr < 1) {
@ -3183,10 +3186,22 @@ static bool isValidExpr(tSQLExpr* pLeft, tSQLExpr* pRight, int32_t optr) {
*
* However, columnA < 4+12 is valid
*/
if ((pLeft->nSQLOptr >= TK_COUNT && pLeft->nSQLOptr <= TK_AVG_IRATE) ||
(pRight->nSQLOptr >= TK_COUNT && pRight->nSQLOptr <= TK_AVG_IRATE) ||
(pLeft->nSQLOptr >= TK_BOOL && pLeft->nSQLOptr <= TK_BINARY && pRight->nSQLOptr >= TK_BOOL &&
pRight->nSQLOptr <= TK_BINARY)) {
if (pLeft->nSQLOptr >= TK_COUNT && pLeft->nSQLOptr <= TK_AVG_IRATE) {
return false;
}
if (pRight == NULL) {
return true;
}
if (pRight->nSQLOptr >= TK_COUNT && pRight->nSQLOptr <= TK_AVG_IRATE) {
return false;
}
if (pLeft->nSQLOptr >= TK_BOOL
&& pLeft->nSQLOptr <= TK_BINARY
&& pRight->nSQLOptr >= TK_BOOL
&& pRight->nSQLOptr <= TK_BINARY) {
return false;
}
@ -3759,13 +3774,17 @@ static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* p
if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
SColumnIndex index = {0};
getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index);
if (getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
tscError("%p: invalid column name (left)", pQueryInfo);
}
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
tscColumnListInsert(pTableMetaInfo->tagColList, &index);
getColumnIndexByName(&pCondExpr->pJoinExpr->pRight->colInfo, pQueryInfo, &index);
if (getColumnIndexByName(&pCondExpr->pJoinExpr->pRight->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
tscError("%p: invalid column name (right)", pQueryInfo);
}
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
@ -4463,7 +4482,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) (pCmd->payload + tsRpcHeadSize);
SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload;
pUpdateMsg->head.vgId = htonl(pTableMeta->vgroupInfo.vgId);
pUpdateMsg->tid = htonl(pTableMeta->sid);
pUpdateMsg->uid = htobe64(pTableMeta->uid);

243
src/client/src/tscServer.c
View File

@ -176,18 +176,16 @@ int tscSendMsgToServer(SSqlObj *pSql) {
char *pMsg = rpcMallocCont(pCmd->payloadLen);
if (NULL == pMsg) {
tscError("%p msg:%s malloc fail", pSql, taosMsg[pSql->cmd.msgType]);
tscError("%p msg:%s malloc failed", pSql, taosMsg[pSql->cmd.msgType]);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
if (pSql->cmd.command < TSDB_SQL_MGMT) {
memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen);
} else {
// set the mgmt ip list
if (pSql->cmd.command >= TSDB_SQL_MGMT) {
pSql->ipList = tscMgmtIpSet;
memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
}
// tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]);
memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
SRpcMsg rpcMsg = {
.msgType = pSql->cmd.msgType,
@ -197,14 +195,18 @@ int tscSendMsgToServer(SSqlObj *pSql) {
.code = 0
};
pSql->pRpcCtx = rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
// NOTE: the rpc context should be acquired before sending data to server.
// Otherwise, the pSql object may have been released already during the response function, which is
// processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely
// cause crash.
/*pSql->pRpcCtx = */rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
return TSDB_CODE_SUCCESS;
}
void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
SSqlObj *pSql = (SSqlObj *)rpcMsg->handle;
if (pSql == NULL || pSql->signature != pSql) {
tscError("%p sql is already released", pSql->signature);
tscError("%p sql is already released", pSql);
return;
}
@ -222,8 +224,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) {
tscTrace("%p sqlObj needs to be released or DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command,
pObj, pObj->signature);
tscTrace("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p",
pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature);
tscFreeSqlObj(pSql);
rpcFreeCont(rpcMsg->pCont);
@ -449,18 +451,11 @@ void tscKillSTableQuery(SSqlObj *pSql) {
}
int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
char *pMsg, *pStart;
pStart = pSql->cmd.payload + tsRpcHeadSize;
pMsg = pStart;
SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *)pMsg;
SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload;
pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle);
pMsg += sizeof(pSql->res.qhandle);
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
pRetrieveMsg->free = htons(pQueryInfo->type);
pMsg += sizeof(pQueryInfo->type);
// todo valid the vgroupId at the client side
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@ -474,12 +469,12 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId);
}
pMsg += sizeof(SRetrieveTableMsg);
pRetrieveMsg->header.contLen = htonl(pSql->cmd.payloadLen);
pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg);
pSql->cmd.msgType = TSDB_MSG_TYPE_FETCH;
pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));
return TSDB_CODE_SUCCESS;
}
@ -487,30 +482,30 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta;
char* pMsg = pSql->cmd.payload + tsRpcHeadSize;
char* pMsg = pSql->cmd.payload;
// NOTE: shell message size should not include SMsgDesc
int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc);
SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg;
pMsgDesc->numOfVnodes = htonl(1); //todo set the right number of vnodes
pMsg += sizeof(SMsgDesc);
SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg;
int32_t vgId = pTableMeta->vgroupInfo.vgId;
SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg;
pMsgDesc->numOfVnodes = htonl(1); // always one vnode
pMsg += sizeof(SMsgDesc);
SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg;
pShellMsg->header.vgId = htonl(vgId);
pShellMsg->header.contLen = htonl(size);
pShellMsg->header.contLen = htonl(size); // the length not includes the size of SMsgDesc
pShellMsg->length = pShellMsg->header.contLen;
pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted
pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit); // number of tables to be inserted
// pSql->cmd.payloadLen is set during copying data into payload
pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
tscSetDnodeIpList(pSql, &pTableMeta->vgroupInfo);
tscTrace("%p build submit msg, vgId:%d numOfVgroup:%d numberOfIP:%d", pSql, vgId, htonl(pMsgDesc->numOfVnodes), pSql->ipList.numOfIps);
tscTrace("%p build submit msg, vgId:%d numOfTables:%d numberOfIP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit,
pSql->ipList.numOfIps);
return TSDB_CODE_SUCCESS;
}
@ -620,9 +615,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return -1;
}
char *pStart = pCmd->payload + tsRpcHeadSize;
SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart;
SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload;
int32_t numOfTags = taosArrayGetSize(pTableMetaInfo->tagColList);
@ -821,7 +814,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
// compressed ts block
pQueryMsg->tsOffset = htonl(pMsg - pStart);
pQueryMsg->tsOffset = htonl(pMsg - pCmd->payload);
int32_t tsLen = 0;
int32_t numOfBlocks = 0;
@ -830,8 +823,18 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL); // this query should not be sent
// todo refactor
fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET);
fread(pMsg, pBlockInfo->compLen, 1, pQueryInfo->tsBuf->f);
if (fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
tscError("%p: fseek failed: %s", pSql, tstrerror(code));
return code;
}
size_t s = fread(pMsg, 1, pBlockInfo->compLen, pQueryInfo->tsBuf->f);
if (s != pBlockInfo->compLen) {
int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
return code;
}
pMsg += pBlockInfo->compLen;
tsLen = pBlockInfo->compLen;
@ -844,7 +847,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder);
}
int32_t msgLen = pMsg - pStart;
int32_t msgLen = pMsg - pCmd->payload;
tscTrace("%p msg built success,len:%d bytes", pSql, msgLen);
pCmd->payloadLen = msgLen;
@ -1286,10 +1289,12 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
SSqlCmd* pCmd = &pSql->cmd;
pCmd->msgType = TSDB_MSG_TYPE_UPDATE_TAG_VAL;
SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) (pCmd->payload + tsRpcHeadSize);
SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload;
pCmd->payloadLen = htonl(pUpdateMsg->head.contLen);
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
tscSetDnodeIpList(pSql, &pTableMetaInfo->pTableMeta->vgroupInfo);
return TSDB_CODE_SUCCESS;
@ -1552,150 +1557,6 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
//}
int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
#if 0
SSuperTableMetaMsg *pMetaMsg;
char * pMsg, *pStart;
int msgLen = 0;
int tableIndex = 0;
SSqlCmd * pCmd = &pSql->cmd;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
STagCond *pTagCond = &pQueryInfo->tagCond;
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
int32_t size = tscEstimateMetricMetaMsgSize(pCmd);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) {
tscError("%p failed to malloc for metric meter msg", pSql);
return -1;
}
pStart = pCmd->payload + tsRpcHeadSize;
pMsg = pStart;
SMgmtHead *pMgmt = (SMgmtHead *)pMsg;
tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pMgmt->db);
pMsg += sizeof(SMgmtHead);
pMetaMsg = (SSuperTableMetaMsg *)pMsg;
pMetaMsg->numOfTables = htonl(pQueryInfo->numOfTables);
pMsg += sizeof(SSuperTableMetaMsg);
int32_t offset = pMsg - (char *)pMetaMsg;
pMetaMsg->join = htonl(offset);
// todo refactor
pMetaMsg->joinCondLen = htonl((TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2);
memcpy(pMsg, pTagCond->joinInfo.left.tableId, TSDB_TABLE_ID_LEN);
pMsg += TSDB_TABLE_ID_LEN;
*(int16_t *)pMsg = pTagCond->joinInfo.left.tagCol;
pMsg += sizeof(int16_t);
memcpy(pMsg, pTagCond->joinInfo.right.tableId, TSDB_TABLE_ID_LEN);
pMsg += TSDB_TABLE_ID_LEN;
*(int16_t *)pMsg = pTagCond->joinInfo.right.tagCol;
pMsg += sizeof(int16_t);
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i);
uint64_t uid = pTableMetaInfo->pTableMeta->uid;
offset = pMsg - (char *)pMetaMsg;
pMetaMsg->metaElem[i] = htonl(offset);
SSuperTableMetaElemMsg *pElem = (SSuperTableMetaElemMsg *)pMsg;
pMsg += sizeof(SSuperTableMetaElemMsg);
// convert to unicode before sending to mnode for metric query
int32_t condLen = 0;
if (pTagCond->numOfTagCond > 0) {
SCond *pCond = tsGetSTableQueryCond(pTagCond, uid);
if (pCond != NULL && pCond->cond != NULL) {
condLen = strlen(pCond->cond) + 1;
bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE);
if (!ret) {
tscError("%p mbs to ucs4 failed:%s", pSql, tsGetSTableQueryCond(pTagCond, uid));
return 0;
}
}
}
pElem->condLen = htonl(condLen);
offset = pMsg - (char *)pMetaMsg;
pElem->cond = htonl(offset);
pMsg += condLen * TSDB_NCHAR_SIZE;
pElem->rel = htons(pTagCond->relType);
if (pTagCond->tbnameCond.uid == uid) {
offset = pMsg - (char *)pMetaMsg;
pElem->tableCond = htonl(offset);
uint32_t len = 0;
if (pTagCond->tbnameCond.cond != NULL) {
len = strlen(pTagCond->tbnameCond.cond);
memcpy(pMsg, pTagCond->tbnameCond.cond, len);
}
pElem->tableCondLen = htonl(len);
pMsg += len;
}
SSqlGroupbyExpr *pGroupby = &pQueryInfo->groupbyExpr;
if (pGroupby->tableIndex != i && pGroupby->numOfGroupCols > 0) {
pElem->orderType = 0;
pElem->orderIndex = 0;
pElem->numOfGroupCols = 0;
} else {
pElem->numOfGroupCols = htons(pGroupby->numOfGroupCols);
for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) {
pElem->tagCols[j] = htons(pTableMetaInfo->tagColumnIndex[j]);
}
if (pGroupby->numOfGroupCols != 0) {
pElem->orderIndex = htons(pGroupby->orderIndex);
pElem->orderType = htons(pGroupby->orderType);
offset = pMsg - (char *)pMetaMsg;
pElem->groupbyTagColumnList = htonl(offset);
for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) {
SColIndex *pCol = &pQueryInfo->groupbyExpr.columnInfo[j];
SColIndex *pDestCol = (SColIndex *)pMsg;
pDestCol->colIdxInBuf = 0;
pDestCol->colIndex = htons(pCol->colIndex);
pDestCol->colId = htons(pDestCol->colId);
pDestCol->flag = htons(pDestCol->flag);
strncpy(pDestCol->name, pCol->name, tListLen(pCol->name));
pMsg += sizeof(SColIndex);
}
}
}
strcpy(pElem->tableId, pTableMetaInfo->name);
pElem->numOfTags = htons(pTableMetaInfo->numOfTags);
int16_t len = pMsg - (char *)pElem;
pElem->elemLen = htons(len); // redundant data for integrate check
}
msgLen = pMsg - pStart;
pCmd->payloadLen = msgLen;
pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP;
assert(msgLen + minMsgSize() <= size);
#endif
SSqlCmd *pCmd = &pSql->cmd;
char* pMsg = pCmd->payload;
@ -1775,7 +1636,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
return TSDB_CODE_TSC_INVALID_VALUE;
}
if (pMetaMsg->numOfTags > TSDB_MAX_TAGS || pMetaMsg->numOfTags < 0) {
if (pMetaMsg->numOfTags > TSDB_MAX_TAGS) {
tscError("invalid numOfTags:%d", pMetaMsg->numOfTags);
return TSDB_CODE_TSC_INVALID_VALUE;
}
@ -1795,7 +1656,11 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
for (int i = 0; i < numOfTotalCols; ++i) {
pSchema->bytes = htons(pSchema->bytes);
pSchema->colId = htons(pSchema->colId);
if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
assert(i == 0);
}
assert(pSchema->type >= TSDB_DATA_TYPE_BOOL && pSchema->type <= TSDB_DATA_TYPE_NCHAR);
pSchema++;
}

3
src/client/src/tscSql.c
View File

@ -133,8 +133,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
return NULL;
}
// tsRpcHeaderSize will be updated during RPC initialization, so only after it initialization, this value is valid
tsInsertHeadSize = tsRpcHeadSize + sizeof(SMsgDesc) + sizeof(SSubmitMsg);
tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
return pSql;
}

6
src/client/src/tscSub.c
View File

@ -308,10 +308,8 @@ void tscSaveSubscriptionProgress(void* sub) {
char path[256];
sprintf(path, "%s/subscribe", tsDataDir);
if (access(path, 0) != 0) {
if (mkdir(path, 0777) != 0 && errno != EEXIST) {
tscError("failed to create subscribe dir: %s", path);
}
if (tmkdir(path, 0777) != 0) {
tscError("failed to create subscribe dir: %s", path);
}
sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);

44
src/client/src/tscSubquery.c
View File

@ -180,6 +180,7 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in
getTmpfilePath("join-", pSupporter->path);
pSupporter->f = fopen(pSupporter->path, "w");
// todo handle error
if (pSupporter->f == NULL) {
tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno));
}
@ -234,7 +235,7 @@ static UNUSED_FUNC bool needSecondaryQuery(SQueryInfo* pQueryInfo) {
/*
* launch secondary stage query to fetch the result that contains timestamp in set
*/
static int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
int32_t numOfSub = 0;
SJoinSupporter* pSupporter = NULL;
@ -249,7 +250,7 @@ static int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
assert(numOfSub > 0);
// scan all subquery, if one sub query has only ts, ignore it
tscTrace("%p start to launch secondary subquery, total:%d, only:%d needs to query", pSql, pSql->numOfSubs, numOfSub);
tscTrace("%p start to launch secondary subqueries, total:%d, only:%d needs to query", pSql, pSql->numOfSubs, numOfSub);
//the subqueries that do not actually launch the secondary query to virtual node is set as completed.
SSubqueryState* pState = pSupporter->pState;
@ -451,7 +452,7 @@ static UNUSED_FUNC void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupport
freeJoinSubqueryObj(pParentSql);
} else {
updateQueryTimeRange(pParentQueryInfo, &win);
tscLaunchSecondPhaseSubqueries(pParentSql);
tscLaunchRealSubqueries(pParentSql);
}
}
@ -851,7 +852,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
// launch the query the retrieve actual results from vnode along with the filtered timestamp
SQueryInfo* pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex);
updateQueryTimeRange(pPQueryInfo, &win);
tscLaunchSecondPhaseSubqueries(pParentSql);
tscLaunchRealSubqueries(pParentSql);
}
static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfRows) {
@ -1159,7 +1160,6 @@ static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code);
static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj);
// todo merge with callback
int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) {
SSqlCmd * pCmd = &pSql->cmd;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
@ -1302,7 +1302,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) {
pState->numOfTotal = pQueryInfo->numOfTables;
pState->numOfRemain = pState->numOfTotal;
tscTrace("%p start launch subquery, total:%d", pSql, pQueryInfo->numOfTables);
tscTrace("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables);
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i);
@ -1378,6 +1378,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
if (ret != 0) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscQueueAsyncRes(pSql);
tfree(pMemoryBuf);
return ret;
}
@ -1729,7 +1730,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
pRes->numOfRows, pQueryInfo->groupbyExpr.orderType);
if (ret != 0) { // set no disk space error info, and abort retry
tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE);
pthread_mutex_unlock(&trsupport->queryMutex);
} else if (pRes->completed) {
tscAllDataRetrievedFromDnode(trsupport, pSql);
@ -1848,8 +1848,6 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) {
SInsertSupporter *pSupporter = (SInsertSupporter *)param;
SSqlObj* pParentObj = pSupporter->pSql;
SSqlCmd* pParentCmd = &pParentObj->cmd;
SSubqueryState* pState = pSupporter->pState;
// record the total inserted rows
@ -1864,7 +1862,6 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
pParentObj->res.code = pSql->res.code;
}
// it is not the initial sqlObj, free it
taos_free_result(tres);
tfree(pSupporter);
@ -1876,7 +1873,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
// release data block data
tfree(pState);
pParentCmd->pDataBlocks = tscDestroyBlockArrayList(pParentCmd->pDataBlocks);
// pParentCmd->pDataBlocks = tscDestroyBlockArrayList(pParentCmd->pDataBlocks);
// restore user defined fp
pParentObj->fp = pParentObj->fetchFp;
@ -1889,12 +1886,14 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
SDataBlockList *pDataBlocks = pCmd->pDataBlocks;
pSql->pSubs = calloc(pDataBlocks->nSize, POINTER_BYTES);
pSql->numOfSubs = pDataBlocks->nSize;
assert(pDataBlocks->nSize > 0);
tscTrace("%p submit data to %d vnode(s)", pSql, pDataBlocks->nSize);
size_t size = taosArrayGetSize(pCmd->pDataBlocks);
assert(size > 0);
pSql->pSubs = calloc(size, POINTER_BYTES);
pSql->numOfSubs = size;
tscTrace("%p submit data to %zu vnode(s)", pSql, size);
SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
pState->numOfTotal = pSql->numOfSubs;
pState->numOfRemain = pSql->numOfSubs;
@ -1920,12 +1919,14 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
pNew->fetchFp = pNew->fp;
pSql->pSubs[numOfSub] = pNew;
pRes->code = tscCopyDataBlockToPayload(pNew, pDataBlocks->pData[numOfSub++]);
STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, numOfSub);
pRes->code = tscCopyDataBlockToPayload(pNew, pTableDataBlock);
if (pRes->code == TSDB_CODE_SUCCESS) {
tscTrace("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, numOfSub);
numOfSub++;
} else {
tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql, numOfSub,
pDataBlocks->nSize, tstrerror(pRes->code));
tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%zu, code:%s", pSql, numOfSub,
size, tstrerror(pRes->code));
goto _error;
}
}
@ -1942,7 +1943,8 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
tscTrace("%p sub:%p launch sub insert, orderOfSub:%d", pSql, pSub, j);
tscProcessSql(pSub);
}
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
return TSDB_CODE_SUCCESS;
_error:

16
src/client/src/tscSystem.c
View File

@ -314,22 +314,6 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
}
break;
case TSDB_OPTION_SOCKET_TYPE:
cfg = taosGetConfigOption("sockettype");
assert(cfg != NULL);
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
// if (strcasecmp(pStr, TAOS_SOCKET_TYPE_NAME_UDP) != 0 && strcasecmp(pStr, TAOS_SOCKET_TYPE_NAME_TCP) != 0) {
// tscError("only 'tcp' or 'udp' allowed for configuring the socket type");
// return -1;
// }
tstrncpy(tsSocketType, pStr, sizeof(tsSocketType));
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscPrint("socket type is set:%s", tsSocketType);
}
break;
default:
// TODO return the correct error code to client in the format for taos_errstr()
tscError("Invalid option %d", option);

129
src/client/src/tscUtil.c
View File

@ -13,8 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tscUtil.h"
#include "hash.h"
#include "os.h"
#include "qast.h"
#include "taosmsg.h"
@ -29,6 +27,8 @@
#include "ttimer.h"
#include "ttokendef.h"
#include "tscLog.h"
#include "tscUtil.h"
#include "hash.h"
static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache);
@ -428,48 +428,18 @@ SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint
return param;
}
SDataBlockList* tscCreateBlockArrayList() {
const int32_t DEFAULT_INITIAL_NUM_OF_BLOCK = 16;
SDataBlockList* pDataBlockArrayList = calloc(1, sizeof(SDataBlockList));
if (pDataBlockArrayList == NULL) {
return NULL;
}
pDataBlockArrayList->nAlloc = DEFAULT_INITIAL_NUM_OF_BLOCK;
pDataBlockArrayList->pData = calloc(1, POINTER_BYTES * pDataBlockArrayList->nAlloc);
if (pDataBlockArrayList->pData == NULL) {
free(pDataBlockArrayList);
void* tscDestroyBlockArrayList(SArray* pDataBlockList) {
if (pDataBlockList == NULL) {
return NULL;
}
return pDataBlockArrayList;
}
void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks) {
if (pList->nSize >= pList->nAlloc) {
pList->nAlloc = (pList->nAlloc) << 1U;
pList->pData = realloc(pList->pData, POINTER_BYTES * (size_t)pList->nAlloc);
// reset allocated memory
memset(pList->pData + pList->nSize, 0, POINTER_BYTES * (pList->nAlloc - pList->nSize));
size_t size = taosArrayGetSize(pDataBlockList);
for (int32_t i = 0; i < size; i++) {
void* d = taosArrayGetP(pDataBlockList, i);
tscDestroyDataBlock(d);
}
pList->pData[pList->nSize++] = pBlocks;
}
void* tscDestroyBlockArrayList(SDataBlockList* pList) {
if (pList == NULL) {
return NULL;
}
for (int32_t i = 0; i < pList->nSize; i++) {
tscDestroyDataBlock(pList->pData[i]);
}
tfree(pList->pData);
tfree(pList);
taosArrayDestroy(pDataBlockList);
return NULL;
}
@ -484,7 +454,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
// set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache
if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) {
strcpy(pTableMetaInfo->name, pDataBlock->tableId);
tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false);
pTableMetaInfo->pTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pDataBlock->pTableMeta);
@ -497,31 +467,32 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
* the dataBlock only includes the RPC Header buffer and actual submit message body, space for digest needs
* additional space.
*/
int ret = tscAllocPayload(pCmd, pDataBlock->nAllocSize + 100);
int ret = tscAllocPayload(pCmd, pDataBlock->size + 100);
if (TSDB_CODE_SUCCESS != ret) {
return ret;
}
memcpy(pCmd->payload, pDataBlock->pData, pDataBlock->nAllocSize);
assert(pDataBlock->size <= pDataBlock->nAllocSize);
memcpy(pCmd->payload, pDataBlock->pData, pDataBlock->size);
/*
* the payloadLen should be actual message body size
* the old value of payloadLen is the allocated payload size
*/
pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize;
pCmd->payloadLen = pDataBlock->size;
assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + 100 && pCmd->payloadLen > 0);
assert(pCmd->allocSize >= pCmd->payloadLen + 100 && pCmd->payloadLen > 0);
return TSDB_CODE_SUCCESS;
}
void tscFreeUnusedDataBlocks(SDataBlockList* pList) {
/* release additional memory consumption */
for (int32_t i = 0; i < pList->nSize; ++i) {
STableDataBlocks* pDataBlock = pList->pData[i];
pDataBlock->pData = realloc(pDataBlock->pData, pDataBlock->size);
pDataBlock->nAllocSize = (uint32_t)pDataBlock->size;
}
}
//void tscFreeUnusedDataBlocks(SDataBlockList* pList) {
// /* release additional memory consumption */
// for (int32_t i = 0; i < pList->nSize; ++i) {
// STableDataBlocks* pDataBlock = pList->pData[i];
// pDataBlock->pData = realloc(pDataBlock->pData, pDataBlock->size);
// pDataBlock->nAllocSize = (uint32_t)pDataBlock->size;
// }
//}
/**
* create the in-memory buffer for each table to keep the submitted data block
@ -568,7 +539,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
return TSDB_CODE_SUCCESS;
}
int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int64_t id, int32_t size,
int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t id, int32_t size,
int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta,
STableDataBlocks** dataBlocks) {
*dataBlocks = NULL;
@ -585,7 +556,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList,
}
taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES);
tscAppendDataBlock(pDataBlockList, *dataBlocks);
taosArrayPush(pDataBlockList, dataBlocks);
}
return TSDB_CODE_SUCCESS;
@ -634,14 +605,18 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
return len;
}
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) {
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
SSqlCmd* pCmd = &pSql->cmd;
void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
SDataBlockList* pVnodeDataBlockList = tscCreateBlockArrayList();
// the expanded size when a row data is converted to SDataRow format
const int32_t MAX_EXPAND_SIZE = TD_DATA_ROW_HEAD_SIZE + TYPE_BYTES[TSDB_DATA_TYPE_BINARY];
for (int32_t i = 0; i < pTableDataBlockList->nSize; ++i) {
STableDataBlocks* pOneTableBlock = pTableDataBlockList->pData[i];
void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
size_t total = taosArrayGetSize(pTableDataBlockList);
for (int32_t i = 0; i < total; ++i) {
STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, i);
STableDataBlocks* dataBuf = NULL;
@ -655,7 +630,9 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
return ret;
}
int64_t destSize = dataBuf->size + pOneTableBlock->size + pOneTableBlock->size*sizeof(int32_t)*2;
SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * MAX_EXPAND_SIZE;
if (dataBuf->nAllocSize < destSize) {
while (dataBuf->nAllocSize < destSize) {
dataBuf->nAllocSize = dataBuf->nAllocSize * 1.5;
@ -676,26 +653,30 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
}
}
SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
tscSortRemoveDataBlockDupRows(pOneTableBlock);
char* e = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1);
char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1);
tscTrace("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId,
pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(e));
pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey));
int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + MAX_EXPAND_SIZE);
int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + sizeof(int32_t) * 2);
pBlocks->tid = htonl(pBlocks->tid);
pBlocks->uid = htobe64(pBlocks->uid);
pBlocks->sversion = htonl(pBlocks->sversion);
pBlocks->numOfRows = htons(pBlocks->numOfRows);
pBlocks->len = htonl(len);
// erase the empty space reserved for binary data
len = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
dataBuf->size += (len + sizeof(SSubmitBlk));
int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
assert(finalLen <= len);
dataBuf->size += (finalLen + sizeof(SSubmitBlk));
assert(dataBuf->size <= dataBuf->nAllocSize);
// the length does not include the SSubmitBlk structure
pBlocks->len = htonl(finalLen);
dataBuf->numOfTables += 1;
}
@ -704,7 +685,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
// free the table data blocks;
pCmd->pDataBlocks = pVnodeDataBlockList;
tscFreeUnusedDataBlocks(pCmd->pDataBlocks);
// tscFreeUnusedDataBlocks(pCmd->pDataBlocks);
taosHashCleanup(pVnodeDataBlockHashList);
return TSDB_CODE_SUCCESS;
@ -1664,6 +1645,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
SSqlCmd* pCmd = &pNew->cmd;
pCmd->command = cmd;
pCmd->parseFinished = 1;
if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
tscFreeSqlObj(pNew);
@ -1724,6 +1706,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
pnCmd->numOfClause = 0;
pnCmd->clauseIndex = 0;
pnCmd->pDataBlocks = NULL;
pnCmd->parseFinished = 1;
if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) {
tscFreeSqlObj(pNew);
@ -1897,7 +1880,7 @@ void tscDoQuery(SSqlObj* pSql) {
}
if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) {
tscProcessMultiVnodesInsertFromFile(pSql);
tscProcessMultiVnodesImportFromFile(pSql);
} else {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
uint16_t type = pQueryInfo->type;

189
src/common/inc/tglobal.h
View File

@ -20,60 +20,51 @@
extern "C" {
#endif
extern char configDir[];
extern char tsVnodeDir[];
extern char tsDnodeDir[];
extern char tsMnodeDir[];
extern char tsDataDir[];
extern char tsLogDir[];
extern char tsScriptDir[];
extern char tsOsName[];
// system info
extern int64_t tsPageSize;
extern int64_t tsOpenMax;
extern int64_t tsStreamMax;
extern int32_t tsNumOfCores;
extern int32_t tsAlternativeRole;
extern float tsTotalLogDirGB;
extern float tsTotalTmpDirGB;
extern float tsTotalDataDirGB;
extern float tsAvailLogDirGB;
extern float tsAvailTmpDirGB;
extern float tsAvailDataDirGB;
extern float tsMinimalLogDirGB;
extern float tsMinimalTmpDirGB;
extern float tsMinimalDataDirGB;
extern int32_t tsEnableCoreFile;
extern int32_t tsTotalMemoryMB;
extern int32_t tsVersion;
extern int32_t tscEmbedded;
extern int64_t tsMsPerDay[3];
extern char tsFirst[];
extern char tsSecond[];
extern char tsLocalFqdn[];
extern char tsLocalEp[];
// cluster
extern char tsFirst[];
extern char tsSecond[];
extern char tsLocalFqdn[];
extern char tsLocalEp[];
extern uint16_t tsServerPort;
extern uint16_t tsDnodeShellPort;
extern uint16_t tsDnodeDnodePort;
extern uint16_t tsSyncPort;
extern int32_t tsStatusInterval;
extern int32_t tsShellActivityTimer;
extern int32_t tsVnodePeerHBTimer;
extern int32_t tsMgmtPeerHBTimer;
extern int32_t tsTableMetaKeepTimer;
extern float tsNumOfThreadsPerCore;
extern float tsRatioOfQueryThreads;
extern char tsPublicIp[];
extern char tsPrivateIp[];
extern int32_t tsStatusInterval;
extern int16_t tsNumOfVnodesPerCore;
extern int16_t tsNumOfTotalVnodes;
extern uint32_t tsPublicIpInt;
extern int32_t tsNumOfMnodes;
// common
extern int tsRpcTimer;
extern int tsRpcMaxTime;
extern int32_t tsMaxConnections;
extern int32_t tsMaxShellConns;
extern int32_t tsShellActivityTimer;
extern uint32_t tsMaxTmrCtrl;
extern float tsNumOfThreadsPerCore;
extern float tsRatioOfQueryThreads;
extern int8_t tsDaylight;
extern char tsTimezone[64];
extern char tsLocale[64];
extern char tsCharset[64]; // default encode string
extern int32_t tsEnableCoreFile;
extern int32_t tsCompressMsgSize;
// client
extern int32_t tsTableMetaKeepTimer;
extern int32_t tsMaxSQLStringLen;
extern int32_t tsTscEnableRecordSql;
extern int32_t tsMaxNumOfOrderedResults;
extern int32_t tsMinSlidingTime;
extern int32_t tsMinIntervalTime;
extern int32_t tsMaxStreamComputDelay;
extern int32_t tsStreamCompStartDelay;
extern int32_t tsStreamCompRetryDelay;
extern float tsStreamComputDelayRatio; // the delayed computing ration of the whole time window
extern int32_t tsProjectExecInterval;
extern int64_t tsMaxRetentWindow;
// db parameters in client
extern int32_t tsCacheBlockSize;
extern int32_t tsBlocksPerVnode;
extern int32_t tsMaxTablePerVnode;
@ -87,42 +78,16 @@ extern int16_t tsCompression;
extern int16_t tsWAL;
extern int32_t tsReplications;
extern int16_t tsAffectedRowsMod;
extern int32_t tsNumOfMnodes;
extern int32_t tsMaxShellConns;
extern int32_t tsMaxTables;
extern char tsMqttBrokerAddress[];
extern char tsMqttBrokerClientId[];
extern int32_t tsMaxConnections;
// balance
extern int32_t tsEnableBalance;
extern int32_t tsAlternativeRole;
extern int32_t tsBalanceInterval;
extern int32_t tsOfflineThreshold;
extern int32_t tsMnodeEqualVnodeNum;
extern int32_t tsEnableHttpModule;
extern int32_t tsEnableMqttModule;
extern int32_t tsEnableMonitorModule;
extern int32_t tsRestRowLimit;
extern int32_t tsMaxSQLStringLen;
extern int32_t tsCompressMsgSize;
extern int32_t tsMaxNumOfOrderedResults;
extern char tsSocketType[4];
extern int32_t tsMinSlidingTime;
extern int32_t tsMinIntervalTime;
extern int32_t tsMaxStreamComputDelay;
extern int32_t tsStreamCompStartDelay;
extern int32_t tsStreamCompRetryDelay;
extern float tsStreamComputDelayRatio; // the delayed computing ration of the whole time window
extern int tsProjectExecInterval;
extern int64_t tsMaxRetentWindow;
extern char tsHttpIp[];
// restful
extern int32_t tsEnableHttpModule;
extern int32_t tsRestRowLimit;
extern uint16_t tsHttpPort;
extern int32_t tsHttpCacheSessions;
extern int32_t tsHttpSessionExpire;
@ -131,12 +96,54 @@ extern int32_t tsHttpEnableCompress;
extern int32_t tsHttpEnableRecordSql;
extern int32_t tsTelegrafUseFieldNum;
extern int32_t tsTscEnableRecordSql;
// mqtt
extern int32_t tsEnableMqttModule;
extern char tsMqttBrokerAddress[];
extern char tsMqttBrokerClientId[];
extern char tsMonitorDbName[];
extern char tsInternalPass[];
extern int32_t tsMonitorInterval;
// monitor
extern int32_t tsEnableMonitorModule;
extern char tsMonitorDbName[];
extern char tsInternalPass[];
extern int32_t tsMonitorInterval;
// internal
extern int32_t tscEmbedded;
extern char configDir[];
extern char tsVnodeDir[];
extern char tsDnodeDir[];
extern char tsMnodeDir[];
extern char tsDataDir[];
extern char tsLogDir[];
extern char tsScriptDir[];
extern int64_t tsMsPerDay[3];
// system info
extern char tsOsName[];
extern int64_t tsPageSize;
extern int64_t tsOpenMax;
extern int64_t tsStreamMax;
extern int32_t tsNumOfCores;
extern float tsTotalLogDirGB;
extern float tsTotalTmpDirGB;
extern float tsTotalDataDirGB;
extern float tsAvailLogDirGB;
extern float tsAvailTmpDirGB;
extern float tsAvailDataDirGB;
extern float tsMinimalLogDirGB;
extern float tsMinimalTmpDirGB;
extern float tsMinimalDataDirGB;
extern int32_t tsTotalMemoryMB;
extern int32_t tsVersion;
// build info
extern char version[];
extern char compatible_version[];
extern char gitinfo[];
extern char gitinfoOfInternal[];
extern char buildinfo[];
// log
extern int32_t tsAsyncLog;
extern int32_t tsNumOfLogLines;
extern int32_t dDebugFlag;
@ -155,22 +162,6 @@ extern int32_t debugFlag;
extern int32_t odbcDebugFlag;
extern int32_t qDebugFlag;
extern uint32_t taosMaxTmrCtrl;
extern int tsRpcTimer;
extern int tsRpcMaxTime;
extern int tsUdpDelay;
extern char version[];
extern char compatible_version[];
extern char gitinfo[];
extern char gitinfoOfInternal[];
extern char buildinfo[];
extern int8_t tsDaylight;
extern char tsTimezone[64];
extern char tsLocale[64];
extern char tsCharset[64]; // default encode string
#define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
void taosInitGlobalCfg();

321
src/common/src/tglobal.c
View File

@ -27,122 +27,35 @@
#include "ttimezone.h"
#include "tsync.h"
char configDir[TSDB_FILENAME_LEN] = "/etc/taos";
char tsVnodeDir[TSDB_FILENAME_LEN] = {0};
char tsDnodeDir[TSDB_FILENAME_LEN] = {0};
char tsMnodeDir[TSDB_FILENAME_LEN] = {0};
char tsDataDir[TSDB_FILENAME_LEN] = "/var/lib/taos";
char tsScriptDir[TSDB_FILENAME_LEN] = "/etc/taos";
char tsOsName[10] = "Linux";
// system info, not configurable
int64_t tsPageSize;
int64_t tsOpenMax;
int64_t tsStreamMax;
int32_t tsNumOfCores = 1;
int32_t tsAlternativeRole = 0;
float tsTotalTmpDirGB = 0;
float tsTotalDataDirGB = 0;
float tsAvailTmpDirGB = 0;
float tsAvailDataDirGB = 0;
float tsMinimalTmpDirGB = 0.1;
float tsMinimalDataDirGB = 0.5;
int32_t tsTotalMemoryMB = 0;
int32_t tsVersion = 0;
int32_t tsEnableCoreFile = 0;
// global, not configurable
int32_t tscEmbedded = 0;
/*
* minimum scale for whole system, millisecond by default
* for TSDB_TIME_PRECISION_MILLI: 86400000L
* TSDB_TIME_PRECISION_MICRO: 86400000000L
* TSDB_TIME_PRECISION_NANO: 86400000000000L
*/
int64_t tsMsPerDay[] = {86400000L, 86400000000L, 86400000000000L};
char tsFirst[TSDB_EP_LEN] = {0};
char tsSecond[TSDB_EP_LEN] = {0};
char tsArbitrator[TSDB_EP_LEN] = {0};
char tsLocalFqdn[TSDB_FQDN_LEN] = {0};
char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port
// cluster
char tsFirst[TSDB_EP_LEN] = {0};
char tsSecond[TSDB_EP_LEN] = {0};
char tsArbitrator[TSDB_EP_LEN] = {0};
char tsLocalFqdn[TSDB_FQDN_LEN] = {0};
char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port
uint16_t tsServerPort = 6030;
uint16_t tsDnodeShellPort = 6030; // udp[6035-6039] tcp[6035]
uint16_t tsDnodeDnodePort = 6035; // udp/tcp
uint16_t tsDnodeDnodePort = 6035; // udp/tcp
uint16_t tsSyncPort = 6040;
int32_t tsStatusInterval = 1; // second
int16_t tsNumOfVnodesPerCore = 8;
int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
int32_t tsNumOfMnodes = 3;
int32_t tsStatusInterval = 1; // second
int32_t tsShellActivityTimer = 3; // second
int32_t tsTableMetaKeepTimer = 7200; // second
// common
int32_t tsRpcTimer = 300;
int32_t tsRpcMaxTime = 600; // seconds;
float tsNumOfThreadsPerCore = 1.0;
float tsRatioOfQueryThreads = 0.5;
int16_t tsNumOfVnodesPerCore = 8;
int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
#ifdef _TD_ARM_32_
int32_t tsMaxTablePerVnode = 100;
#else
int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
#endif
int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS;
int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP;
int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds
int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION;
int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL;
int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM;
/**
* Change the meaning of affected rows:
* 0: affected rows not include those duplicate records
* 1: affected rows include those duplicate records
*/
int16_t tsAffectedRowsMod = 0;
int32_t tsNumOfMnodes = 3;
int32_t tsRpcMaxTime = 600; // seconds;
int32_t tsMaxShellConns = 5000;
int32_t tsMaxConnections = 5000;
int32_t tsBalanceInterval = 300; // seconds
int32_t tsOfflineThreshold = 86400*100; // seconds 10days
int32_t tsMnodeEqualVnodeNum = 4;
int32_t tsEnableHttpModule = 1;
int32_t tsEnableMqttModule = 0; // not finished yet, not started it by default
int32_t tsEnableMonitorModule = 0;
int32_t tsRestRowLimit = 10240;
int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN;
int32_t tsNumOfLogLines = 10000000;
int32_t mDebugFlag = 135;
int32_t sdbDebugFlag = 135;
int32_t dDebugFlag = 135;
int32_t vDebugFlag = 135;
int32_t cDebugFlag = 135;
int32_t jniDebugFlag = 131;
int32_t odbcDebugFlag = 131;
int32_t httpDebugFlag = 131;
int32_t mqttDebugFlag = 131;
int32_t monitorDebugFlag = 131;
int32_t qDebugFlag = 131;
int32_t rpcDebugFlag = 135;
int32_t uDebugFlag = 131;
int32_t debugFlag = 131;
int32_t sDebugFlag = 135;
int32_t tsdbDebugFlag = 135;
// the maximum number of results for projection query on super table that are returned from
// one virtual node, to order according to timestamp
int32_t tsMaxNumOfOrderedResults = 100000;
int32_t tsShellActivityTimer = 3; // second
float tsNumOfThreadsPerCore = 1.0;
float tsRatioOfQueryThreads = 0.5;
int8_t tsDaylight = 0;
char tsTimezone[64] = {0};
char tsLocale[TSDB_LOCALE_LEN] = {0};
char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string
int32_t tsEnableCoreFile = 0;
int32_t tsMaxBinaryDisplayWidth = 30;
/*
* denote if the server needs to compress response message at the application layer to client, including query rsp,
@ -154,8 +67,14 @@ int32_t tsMaxNumOfOrderedResults = 100000;
*/
int32_t tsCompressMsgSize = -1;
// use UDP by default[option: udp, tcp]
char tsSocketType[4] = "udp";
// client
int32_t tsTableMetaKeepTimer = 7200; // second
int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN;
int32_t tsTscEnableRecordSql = 0;
// the maximum number of results for projection query on super table that are returned from
// one virtual node, to order according to timestamp
int32_t tsMaxNumOfOrderedResults = 100000;
// 10 ms for sliding time, the value will changed in case of time precision changed
int32_t tsMinSlidingTime = 10;
@ -178,30 +97,104 @@ float tsStreamComputDelayRatio = 0.1;
int32_t tsProjectExecInterval = 10000; // every 10sec, the projection will be executed once
int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance
uint16_t tsHttpPort = 6020; // only tcp, range tcp[6020]
// uint16_t tsNginxPort = 6060; //only tcp, range tcp[6060]
int32_t tsHttpCacheSessions = 100;
int32_t tsHttpSessionExpire = 36000;
int32_t tsHttpMaxThreads = 2;
int32_t tsHttpEnableCompress = 0;
int32_t tsHttpEnableRecordSql = 0;
int32_t tsTelegrafUseFieldNum = 0;
// db parameters
int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS;
int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP;
int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds
int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION;
int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL;
int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM;
int32_t tsTscEnableRecordSql = 0;
uint32_t tsPublicIpInt = 0;
#ifdef _TD_ARM_32_
int32_t tsMaxTablePerVnode = 100;
#else
int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
#endif
char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
char tsInternalPass[] = "secretkey";
// balance
int32_t tsEnableBalance = 1;
int32_t tsAlternativeRole = 0;
int32_t tsBalanceInterval = 300; // seconds
int32_t tsOfflineThreshold = 86400*100; // seconds 10days
int32_t tsMnodeEqualVnodeNum = 4;
// restful
int32_t tsEnableHttpModule = 1;
int32_t tsRestRowLimit = 10240;
uint16_t tsHttpPort = 6020; // only tcp, range tcp[6020]
int32_t tsHttpCacheSessions = 100;
int32_t tsHttpSessionExpire = 36000;
int32_t tsHttpMaxThreads = 2;
int32_t tsHttpEnableCompress = 0;
int32_t tsHttpEnableRecordSql = 0;
int32_t tsTelegrafUseFieldNum = 0;
// mqtt
int32_t tsEnableMqttModule = 0; // not finished yet, not started it by default
char tsMqttBrokerAddress[128] = {0};
char tsMqttBrokerClientId[128] = {0};
// monitor
int32_t tsEnableMonitorModule = 0;
char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
char tsInternalPass[] = "secretkey";
int32_t tsMonitorInterval = 30; // seconds
int8_t tsDaylight = 0;
char tsTimezone[64] = {0};
char tsLocale[TSDB_LOCALE_LEN] = {0};
char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string
char tsMqttBrokerAddress[128] = {0};
char tsMqttBrokerClientId[128] = {0};
// internal
int32_t tscEmbedded = 0;
char configDir[TSDB_FILENAME_LEN] = "/etc/taos";
char tsVnodeDir[TSDB_FILENAME_LEN] = {0};
char tsDnodeDir[TSDB_FILENAME_LEN] = {0};
char tsMnodeDir[TSDB_FILENAME_LEN] = {0};
char tsDataDir[TSDB_FILENAME_LEN] = "/var/lib/taos";
char tsScriptDir[TSDB_FILENAME_LEN] = "/etc/taos";
int32_t tsMaxBinaryDisplayWidth = 30;
/*
* minimum scale for whole system, millisecond by default
* for TSDB_TIME_PRECISION_MILLI: 86400000L
* TSDB_TIME_PRECISION_MICRO: 86400000000L
* TSDB_TIME_PRECISION_NANO: 86400000000000L
*/
int64_t tsMsPerDay[] = {86400000L, 86400000000L, 86400000000000L};
// system info
char tsOsName[10] = "Linux";
int64_t tsPageSize;
int64_t tsOpenMax;
int64_t tsStreamMax;
int32_t tsNumOfCores = 1;
float tsTotalTmpDirGB = 0;
float tsTotalDataDirGB = 0;
float tsAvailTmpDirGB = 0;
float tsAvailDataDirGB = 0;
float tsMinimalTmpDirGB = 0.1;
float tsMinimalDataDirGB = 0.5;
int32_t tsTotalMemoryMB = 0;
int32_t tsVersion = 0;
// log
int32_t tsNumOfLogLines = 10000000;
int32_t mDebugFlag = 135;
int32_t sdbDebugFlag = 135;
int32_t dDebugFlag = 135;
int32_t vDebugFlag = 135;
int32_t cDebugFlag = 135;
int32_t jniDebugFlag = 131;
int32_t odbcDebugFlag = 131;
int32_t httpDebugFlag = 131;
int32_t mqttDebugFlag = 131;
int32_t monitorDebugFlag = 131;
int32_t qDebugFlag = 131;
int32_t rpcDebugFlag = 135;
int32_t uDebugFlag = 131;
int32_t debugFlag = 131;
int32_t sDebugFlag = 135;
int32_t tsdbDebugFlag = 135;
static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
@ -426,6 +419,16 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "balance";
cfg.ptr = &tsEnableBalance;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "balanceInterval";
cfg.ptr = &tsBalanceInterval;
cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -447,19 +450,9 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "affectedRowsMod";
cfg.ptr = &tsAffectedRowsMod;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// timer
cfg.option = "maxTmrCtrl";
cfg.ptr = &taosMaxTmrCtrl;
cfg.ptr = &tsMaxTmrCtrl;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 8;
@ -669,16 +662,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "ctime";
cfg.ptr = &tsCommitTime;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_COMMIT_TIME;
cfg.maxValue = TSDB_MAX_COMMIT_TIME;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_SECOND;
taosInitConfigOption(cfg);
cfg.option = "comp";
cfg.ptr = &tsCompression;
cfg.valType = TAOS_CFG_VTYPE_INT16;
@ -729,17 +712,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// socket type; udp by default
cfg.option = "sockettype";
cfg.ptr = tsSocketType;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = 3;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "compressMsgSize";
cfg.ptr = &tsCompressMsgSize;
cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -873,7 +845,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "mqtt";
cfg.ptr = &tsEnableMqttModule;
cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -894,16 +865,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "monitorDbName";
cfg.ptr = tsMonitorDbName;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// http configs
cfg.option = "httpCacheSessions";
cfg.ptr = &tsHttpCacheSessions;
@ -955,16 +916,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "httpEnableCompress";
cfg.ptr = &tsHttpEnableCompress;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// debug flag
cfg.option = "numOfLogLines";
cfg.ptr = &tsNumOfLogLines;
@ -1232,9 +1183,7 @@ bool taosCheckGlobalCfg() {
taosGetFqdn(tsLocalFqdn);
}
strcpy(tsLocalEp, tsLocalFqdn);
snprintf(tsLocalEp + strlen(tsLocalEp), sizeof(tsLocalEp), ":%d", tsServerPort);
snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%d", tsLocalFqdn, tsServerPort);
uPrint("localEp is: %s", tsLocalEp);
if (tsFirst[0] == 0) {

44
src/connector/jdbc/buildTDengine.sh Executable file
View File

@ -0,0 +1,44 @@
#!/bin/bash
ulimit -c unlimited
function buildTDengine {
cd /root/TDengine
git remote update
REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
LOCAL_COMMIT=`git rev-parse --short @`
echo " LOCAL: $LOCAL_COMMIT"
echo "REMOTE: $REMOTE_COMMIT"
if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
echo "repo up-to-date"
else
echo "repo need to pull"
git pull
LOCAL_COMMIT=`git rev-parse --short @`
cd /root/TDengine/debug
rm -rf /root/TDengine/debug/*
cmake ..
make > /dev/null
make install
fi
}
function restartTaosd {
systemctl stop taosd
pkill -KILL -x taosd
sleep 10
logDir=`grep 'logDir' /etc/taos/taos.cfg|awk 'END{print $2}'`
dataDir=`grep 'dataDir' /etc/taos/taos.cfg|awk '{print $2}'`
rm -rf $logDir/*
rm -rf $dataDir/*
taosd 2>&1 > /dev/null &
sleep 10
}
buildTDengine
restartTaosd

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/AsyncSubscribeTest.java
View File

@ -12,7 +12,7 @@ import java.util.Properties;
import static org.junit.Assert.assertTrue;
public class AsyncSubscribeTest {
public class AsyncSubscribeTest extends BaseTest {
Connection connection = null;
Statement statement = null;
String dbName = "test";

27
src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java Normal file
View File

@ -0,0 +1,27 @@
package com.taosdata.jdbc;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import org.junit.BeforeClass;
public class BaseTest {
@BeforeClass
public static void setupEnv() {
try{
String path = System.getProperty("user.dir");
String bashPath = path + "/buildTDengine.sh";
Process ps = Runtime.getRuntime().exec(bashPath);
ps.waitFor();
BufferedReader br = new BufferedReader(new InputStreamReader(ps.getInputStream()));
while(br.readLine() != null) {
System.out.println(br.readLine());
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java
View File

@ -10,7 +10,7 @@ import java.util.Properties;
import static org.junit.Assert.assertTrue;
public class ConnectionTest {
public class ConnectionTest extends BaseTest {
static Connection connection = null;
static Statement statement = null;
static String dbName = "test";

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java
View File

@ -7,7 +7,7 @@ import org.junit.Test;
import java.sql.*;
import java.util.Properties;
public class DatabaseMetaDataTest {
public class DatabaseMetaDataTest extends BaseTest {
static Connection connection = null;
static PreparedStatement statement = null;
static String dbName = "test";

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java
View File

@ -9,7 +9,7 @@ import java.util.Properties;
import static org.junit.Assert.assertEquals;
public class ImportTest {
public class ImportTest extends BaseTest {
Connection connection = null;
Statement statement = null;
String dbName = "test";

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java
View File

@ -13,7 +13,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@FixMethodOrder(MethodSorters.DEFAULT)
public class PreparedStatementTest {
public class PreparedStatementTest extends BaseTest {
static Connection connection = null;
static PreparedStatement statement = null;
static String dbName = "test";

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java
View File

@ -13,7 +13,7 @@ import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class ResultSetTest {
public class ResultSetTest extends BaseTest {
static Connection connection = null;
static Statement statement = null;
static String dbName = "test";

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java
View File

@ -9,7 +9,7 @@ import java.util.Properties;
import static org.junit.Assert.assertEquals;
public class SelectTest {
public class SelectTest extends BaseTest {
Connection connection = null;
Statement statement = null;
String dbName = "test";

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java
View File

@ -12,7 +12,7 @@ import java.util.Properties;
import static org.junit.Assert.assertEquals;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class StableTest {
public class StableTest extends BaseTest {
static Connection connection = null;
static Statement statement = null;
static String dbName = "test";

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java
View File

@ -10,7 +10,7 @@ import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class StatementTest {
public class StatementTest extends BaseTest {
static Connection connection = null;
static Statement statement = null;
static String dbName = "test";

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java
View File

@ -12,7 +12,7 @@ import java.util.Properties;
import static org.junit.Assert.assertTrue;
public class SubscribeTest {
public class SubscribeTest extends BaseTest {
Connection connection = null;
Statement statement = null;
String dbName = "test";

2
src/dnode/src/dnodeMPeer.c
View File

@ -74,6 +74,8 @@ void dnodeCleanupMnodePeer() {
}
}
taosCloseQset(tsMPeerQset);
tfree(tsMPeerPool.peerWorker);
dPrint("dnode mpeer is closed");
}

4
src/dnode/src/dnodeMWrite.c
View File

@ -76,6 +76,8 @@ void dnodeCleanupMnodeWrite() {
}
}
taosCloseQset(tsMWriteQset);
tfree(tsMWritePool.writeWorker);
dPrint("dnode mwrite is closed");
}
@ -194,4 +196,4 @@ void dnodeDelayReprocessMnodeWriteMsg(void *pMsg) {
SMnodeMsg *mnodeMsg = pMsg;
void *unUsed = NULL;
taosTmrReset(dnodeDoDelayReprocessMnodeWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
}
}

1
src/inc/mnode.h
View File

@ -48,6 +48,7 @@ typedef struct SMnodeMsg {
struct SDbObj * pDb;
struct SVgObj * pVgroup;
struct STableObj *pTable;
struct SSuperTableObj *pSTable;
} SMnodeMsg;
void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg);

1
src/inc/taos.h
View File

@ -48,7 +48,6 @@ typedef enum {
TSDB_OPTION_TIMEZONE,
TSDB_OPTION_CONFIGDIR,
TSDB_OPTION_SHELL_ACTIVITY_TIMER,
TSDB_OPTION_SOCKET_TYPE,
TSDB_MAX_OPTIONS
} TSDB_OPTION;

4
src/inc/taosdef.h
View File

@ -244,8 +244,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE
#define TSDB_PAYLOAD_SIZE (TSDB_DEFAULT_PKT_SIZE - 100)
#define TSDB_DEFAULT_PAYLOAD_SIZE 2048 // default payload size
#define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE
#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value
#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth
#define TSDB_CQ_SQL_SIZE 1024
#define TSDB_MAX_VNODES 256

4
src/inc/tbalance.h
View File

@ -25,8 +25,8 @@ struct SDnodeObj;
int32_t balanceInit();
void balanceCleanUp();
void balanceNotify();
void balanceUpdateMnode();
void balanceAsyncNotify();
void balanceSyncNotify();
void balanceReset();
int32_t balanceAllocVnodes(struct SVgObj *pVgroup);
int32_t balanceDropDnode(struct SDnodeObj *pDnode);

76
src/inc/tsdb.h
View File

@ -105,8 +105,9 @@ typedef struct {
void tsdbClearTableCfg(STableCfg *config);
void * tsdbGetTableTagVal(TSDB_REPO_T *repo, const STableId *id, int32_t colId, int16_t type, int16_t bytes);
char * tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id);
void* tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes);
char* tsdbGetTableName(void *pTable);
STableId tsdbGetTableId(void *pTable);
STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg);
int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg);
@ -176,18 +177,16 @@ typedef struct SQueryRowCond {
TSKEY ts;
} SQueryRowCond;
typedef void *TsdbPosT;
/**
* Get the data block iterator, starting from position according to the query condition
*
* @param tsdb tsdb handle
* @param pCond query condition, including time window, result set order, and basic required columns for each block
* @param groupInfo tableId list in the form of set, seperated into different groups according to group by condition
* @param tableqinfoGroupInfo tableId list in the form of set, seperated into different groups according to group by condition
* @param qinfo query info handle from query processor
* @return
*/
TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void *qinfo);
TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableqinfoGroupInfo, void *qinfo);
/**
* Get the last row of the given query time window for all the tables in STableGroupInfo object.
@ -197,13 +196,26 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
* @param tsdb tsdb handle
* @param pCond query condition, including time window, result set order, and basic required columns for each
* block
* @param groupInfo tableId list.
* @param tableqinfoGroupInfo tableId list.
* @return
*/
TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void *qinfo);
TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableqinfoGroupInfo, void *qinfo);
SArray *tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle);
/**
* get the queried table object list
* @param pHandle
* @return
*/
SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
/**
* get the group list according to table id from client
* @param tsdb
* @param pCond
* @param groupList
* @param qinfo
* @return
*/
TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
void *qinfo);
@ -247,37 +259,6 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataSta
*/
SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList);
/**
* todo remove this function later
* @param pQueryHandle
* @param pIdList
* @return
*/
SArray *tsdbRetrieveDataRow(TsdbQueryHandleT *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond);
/**
* Get iterator for super tables, of which tags values satisfy the tag filter info
*
* NOTE: the tagFilterStr is an bin-expression for tag filter, such as ((tag_col = 5) and (tag_col2 > 7))
* The filter string is sent from client directly.
* The build of the tags filter expression from string is done in the iterator generating function.
*
* @param pCond query condition
* @param pTagFilterStr tag filter info
* @return
*/
TsdbQueryHandleT *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr);
/**
* Get the qualified tables for (super) table query.
* Used to handle the super table projection queries, the last_row query, the group by on normal columns query,
* the interpolation query, and timestamp-comp query for join processing.
*
* @param pQueryHandle
* @return table sid list. the invoker is responsible for the release of this the sid list.
*/
SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle);
/**
* Get the qualified table id for a super table according to the tag query expression.
* @param stableid. super table sid
@ -287,6 +268,12 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, const char *pT
int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList,
SColIndex *pColIndex, int32_t numOfCols);
/**
* destory the created table group list, which is generated by tag query
* @param pGroupList
*/
void tsdbDestoryTableGroup(STableGroupInfo *pGroupList);
/**
* create the table group result including only one table, used to handle the normal table query
*
@ -297,6 +284,15 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, const char *pT
*/
int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, STableGroupInfo *pGroupInfo);
/**
*
* @param tsdb
* @param pTableIdList
* @param pGroupInfo
* @return
*/
int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo);
/**
* clean up the query handle
* @param queryHandle

10
src/inc/ttokendef.h
View File

@ -221,6 +221,16 @@
#define TK_INTO 203
#define TK_VALUES 204
#define TK_SPACE 300
#define TK_COMMENT 301
#define TK_ILLEGAL 302
#define TK_HEX 303 // hex number 0x123
#define TK_OCT 304 // oct number
#define TK_BIN 305 // bin format data 0b111
#define TK_FILE 306
#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query
#endif

7
src/kit/shell/src/shellEngine.c
View File

@ -474,7 +474,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
} while( row != NULL);
result = NULL;
taos_free_result(tres);
//taos_free_result(tres);
fclose(fp);
return numOfRows;
@ -803,6 +803,7 @@ void source_file(TAOS *con, char *fptr) {
char *fname = full_path.we_wordv[0];
/*
if (access(fname, F_OK) != 0) {
fprintf(stderr, "ERROR: file %s is not exist\n", fptr);
@ -810,6 +811,7 @@ void source_file(TAOS *con, char *fptr) {
free(cmd);
return;
}
*/
FILE *f = fopen(fname, "r");
if (f == NULL) {
@ -849,7 +851,7 @@ void source_file(TAOS *con, char *fptr) {
void shellGetGrantInfo(void *con) {
return;
#if 0
char sql[] = "show grants";
TAOS_RES* tres = taos_query(con, sql);
@ -900,4 +902,5 @@ void shellGetGrantInfo(void *con) {
}
fprintf(stdout, "\n");
#endif
}

5
src/kit/shell/src/shellImport.c
View File

@ -73,7 +73,7 @@ static void shellParseDirectory(const char *directoryName, const char *prefix, c
}
int fileNum = 0;
while (fscanf(fp, "%s", fileArray[fileNum++])) {
while (fscanf(fp, "%128s", fileArray[fileNum++])) {
if (strcmp(fileArray[fileNum-1], shellTablesSQLFile) == 0) {
fileNum--;
}
@ -150,9 +150,11 @@ static void shellSourceFile(TAOS *con, char *fptr) {
char *fname = full_path.we_wordv[0];
if (fname == NULL) {
fprintf(stderr, "ERROR: invalid filename\n");
free(cmd);
return;
}
/*
if (access(fname, F_OK) != 0) {
fprintf(stderr, "ERROR: file %s is not exist\n", fptr);
@ -168,6 +170,7 @@ static void shellSourceFile(TAOS *con, char *fptr) {
free(cmd);
return;
}
*/
FILE *f = fopen(fname, "r");
if (f == NULL) {

26
src/kit/shell/src/shellLinux.c
View File

@ -86,7 +86,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
wordfree(&full_path);
return -1;
}
strcpy(configDir, full_path.we_wordv[0]);
tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path);
break;
case 's':
@ -100,7 +100,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
fprintf(stderr, "Invalid path %s\n", arg);
return -1;
}
strcpy(arguments->file, full_path.we_wordv[0]);
tstrncpy(arguments->file, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path);
break;
case 'D':
@ -108,7 +108,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
fprintf(stderr, "Invalid path %s\n", arg);
return -1;
}
strcpy(arguments->dir, full_path.we_wordv[0]);
tstrncpy(arguments->dir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path);
break;
case 'T':
@ -162,13 +162,13 @@ void shellReadCommand(TAOS *con, char *command) {
// Read input.
char c;
while (1) {
c = getchar();
c = (char)getchar(); // getchar() return an 'int' value
if (c < 0) { // For UTF-8
int count = countPrefixOnes(c);
utf8_array[0] = c;
for (int k = 1; k < count; k++) {
c = getchar();
c = (char)getchar();
utf8_array[k] = c;
}
insertChar(&cmd, utf8_array, count);
@ -214,10 +214,10 @@ void shellReadCommand(TAOS *con, char *command) {
break;
}
} else if (c == '\033') {
c = getchar();
c = (char)getchar();
switch (c) {
case '[':
c = getchar();
c = (char)getchar();
switch (c) {
case 'A': // Up arrow
if (hist_counter != history.hstart) {
@ -244,35 +244,35 @@ void shellReadCommand(TAOS *con, char *command) {
moveCursorLeft(&cmd);
break;
case '1':
if ((c = getchar()) == '~') {
if ((c = (char)getchar()) == '~') {
// Home key
positionCursorHome(&cmd);
}
break;
case '2':
if ((c = getchar()) == '~') {
if ((c = (char)getchar()) == '~') {
// Insert key
}
break;
case '3':
if ((c = getchar()) == '~') {
if ((c = (char)getchar()) == '~') {
// Delete key
deleteChar(&cmd);
}
break;
case '4':
if ((c = getchar()) == '~') {
if ((c = (char)getchar()) == '~') {
// End key
positionCursorEnd(&cmd);
}
break;
case '5':
if ((c = getchar()) == '~') {
if ((c = (char)getchar()) == '~') {
// Page up key
}
break;
case '6':
if ((c = getchar()) == '~') {
if ((c = (char)getchar()) == '~') {
// Page down key
}
break;

45
src/kit/taosdemo/taosdemo.c
View File

@ -34,6 +34,7 @@
#include <wordexp.h>
#include "taos.h"
#include "tutil.h"
extern char configDir[];
@ -82,7 +83,7 @@ typedef struct DemoArguments {
bool insert_only;
char *output_file;
int mode;
char *datatype[MAX_NUM_DATATYPE];
char *datatype[MAX_NUM_DATATYPE+1];
int len_of_binary;
int num_of_CPR;
int num_of_threads;
@ -432,7 +433,7 @@ int main(int argc, char *argv[]) {
tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
printf("###################################################################\n\n");
printf("Press enter key to continue");
getchar();
(void)getchar();
fprintf(fp, "###################################################################\n");
fprintf(fp, "# Server IP: %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port);
@ -469,11 +470,12 @@ int main(int argc, char *argv[]) {
char command[BUFFER_SIZE] = "\0";
sprintf(command, "drop database %s;", db_name);
taos_query(taos, command);
TAOS_RES* res = taos_query(taos, command);
taos_free_result(res);
sprintf(command, "create database %s;", db_name);
taos_query(taos, command);
res = taos_query(taos, command);
taos_free_result(res);
char cols[STRING_LEN] = "\0";
int colIndex = 0;
@ -550,8 +552,8 @@ int main(int argc, char *argv[]) {
for (int i = 0; i < threads; i++) {
info *t_info = infos + i;
t_info->threadID = i;
strcpy(t_info->db_name, db_name);
strcpy(t_info->tb_prefix, tb_prefix);
tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
tstrncpy(t_info->tb_prefix, tb_prefix, MAX_TB_NAME_SIZE);
t_info->datatype = data_type;
t_info->ncols_per_record = ncols_per_record;
t_info->nrecords_per_table = nrecords_per_table;
@ -845,10 +847,10 @@ void *syncWrite(void *sarg) {
pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, tID);
int k;
for (k = 0; k < winfo->nrecords_per_request;) {
int rand_num = rand() % 100;
int rand_num = trand() % 100;
int len = -1;
if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) {
long d = tmp_time - rand() % 1000000 + rand_num;
long d = tmp_time - trand() % 1000000 + rand_num;
len = generateData(data, data_type, ncols_per_record, d, len_of_binary);
} else {
len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary);
@ -940,10 +942,10 @@ void callBack(void *param, TAOS_RES *res, int code) {
pstr += sprintf(pstr, "insert into %s values", tb_info->tb_name);
for (int i = 0; i < tb_info->nrecords_per_request; i++) {
int rand_num = rand() % 100;
int rand_num = trand() % 100;
if (tb_info->data_of_order ==1 && rand_num < tb_info->data_of_rate)
{
long d = tmp_time - rand() % 1000000 + rand_num;
long d = tmp_time - trand() % 1000000 + rand_num;
generateData(data, datatype, ncols_per_record, d, len_of_binary);
} else
{
@ -985,22 +987,27 @@ int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t times
}
}
if (0 == c) {
perror("data type error!");
exit(-1);
}
for (int i = 0; i < num_of_cols; i++) {
if (strcasecmp(data_type[i % c], "tinyint") == 0) {
pstr += sprintf(pstr, ", %d", (int)(rand() % 128));
pstr += sprintf(pstr, ", %d", (int)(trand() % 128));
} else if (strcasecmp(data_type[i % c], "smallint") == 0) {
pstr += sprintf(pstr, ", %d", (int)(rand() % 32767));
pstr += sprintf(pstr, ", %d", (int)(trand() % 32767));
} else if (strcasecmp(data_type[i % c], "int") == 0) {
pstr += sprintf(pstr, ", %d", (int)(rand() % 10));
pstr += sprintf(pstr, ", %d", (int)(trand() % 10));
} else if (strcasecmp(data_type[i % c], "bigint") == 0) {
pstr += sprintf(pstr, ", %" PRId64, rand() % 2147483648);
pstr += sprintf(pstr, ", %" PRId64, trand() % 2147483648);
} else if (strcasecmp(data_type[i % c], "float") == 0) {
pstr += sprintf(pstr, ", %10.4f", (float)(rand() / 1000));
pstr += sprintf(pstr, ", %10.4f", (float)(trand() / 1000.0));
} else if (strcasecmp(data_type[i % c], "double") == 0) {
double t = (double)(rand() / 1000000);
double t = (double)(trand() / 1000000.0);
pstr += sprintf(pstr, ", %20.8f", t);
} else if (strcasecmp(data_type[i % c], "bool") == 0) {
bool b = rand() & 1;
bool b = trand() & 1;
pstr += sprintf(pstr, ", %s", b ? "true" : "false");
} else if (strcasecmp(data_type[i % c], "binary") == 0) {
char s[len_of_binary];
@ -1026,7 +1033,7 @@ void rand_string(char *str, int size) {
--size;
int n;
for (n = 0; n < size; n++) {
int key = rand() % (int)(sizeof charset - 1);
int key = trand() % (int)(sizeof charset - 1);
str[n] = charset[key];
}
str[n] = 0;

78
src/kit/taosdump/taosdump.c
View File

@ -229,7 +229,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
fprintf(stderr, "Invalid path %s\n", arg);
return -1;
}
strcpy(arguments->output, full_path.we_wordv[0]);
tstrncpy(arguments->output, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path);
break;
case 'i':
@ -238,7 +238,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
fprintf(stderr, "Invalid path %s\n", arg);
return -1;
}
strcpy(arguments->input, full_path.we_wordv[0]);
tstrncpy(arguments->input, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path);
break;
case 'c':
@ -246,7 +246,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
fprintf(stderr, "Invalid path %s\n", arg);
return -1;
}
strcpy(configDir, full_path.we_wordv[0]);
tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path);
break;
case 'e':
@ -411,7 +411,7 @@ int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo) {
if ((row = taos_fetch_row(result)) != NULL) {
isSet = true;
pTableRecordInfo->isMetric = true;
strcpy(pTableRecordInfo->tableRecord.metric, table);
tstrncpy(pTableRecordInfo->tableRecord.metric, table, TSDB_TABLE_NAME_LEN);
}
taos_free_result(result);
@ -537,11 +537,11 @@ int taosDumpOut(SDumpArguments *arguments) {
if (arguments->databases || arguments->all_databases) {
for (int i = 0; i < count; i++) {
taosDumpDb(dbInfos[i], arguments, fp);
(void)taosDumpDb(dbInfos[i], arguments, fp);
}
} else {
if (arguments->arg_list_len == 1) {
taosDumpDb(dbInfos[0], arguments, fp);
(void)taosDumpDb(dbInfos[0], arguments, fp);
} else {
taosDumpCreateDbClause(dbInfos[0], arguments->with_property, fp);
@ -560,9 +560,9 @@ int taosDumpOut(SDumpArguments *arguments) {
}
if (tableRecordInfo.isMetric) { // dump whole metric
taosDumpMetric(tableRecordInfo.tableRecord.metric, arguments, fp);
(void)taosDumpMetric(tableRecordInfo.tableRecord.metric, arguments, fp);
} else { // dump MTable and NTable
taosDumpTable(tableRecordInfo.tableRecord.name, tableRecordInfo.tableRecord.metric, arguments, fp);
(void)taosDumpTable(tableRecordInfo.tableRecord.name, tableRecordInfo.tableRecord.metric, arguments, fp);
}
}
}
@ -642,17 +642,23 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) {
taos_free_result(result);
lseek(fd, 0, SEEK_SET);
(void)lseek(fd, 0, SEEK_SET);
while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) {
tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
STableRecord tableInfo;
while (1) {
memset(&tableInfo, 0, sizeof(STableRecord));
ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord));
if (ret <= 0) break;
tableInfo.name[sizeof(tableInfo.name) - 1] = 0;
tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0;
taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp);
}
close(fd);
(void)remove(".table.tmp");
return remove(".table.tmp");
return 0;
}
void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArguments *arguments, FILE *fp) {
@ -807,7 +813,7 @@ int taosGetTableDes(char *table, STableDef *tableDes) {
TAOS_FIELD *fields = taos_fetch_fields(result);
strcpy(tableDes->name, table);
tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN);
while ((row = taos_fetch_row(result)) != NULL) {
strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
@ -874,7 +880,7 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
int fd = -1;
STableRecord tableRecord;
tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
//tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
sprintf(command, "select tbname from %s", metric);
TAOS_RES* result = taos_query(taos, command);
@ -895,24 +901,42 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
while ((row = taos_fetch_row(result)) != NULL) {
memset(&tableRecord, 0, sizeof(STableRecord));
strncpy(tableRecord.name, (char *)row[0], fields[0].bytes);
strcpy(tableRecord.metric, metric);
tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes);
tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
twrite(fd, &tableRecord, sizeof(STableRecord));
}
taos_free_result(result);
result = NULL;
lseek(fd, 0, SEEK_SET);
(void)lseek(fd, 0, SEEK_SET);
while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) {
tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
//STableRecord tableInfo;
char tableName[TSDB_TABLE_NAME_LEN] ;
char metricName[TSDB_TABLE_NAME_LEN];
ssize_t ret;
while (1) {
//memset(&tableInfo, 0, sizeof(STableRecord));
memset(tableName, 0, TSDB_TABLE_NAME_LEN);
memset(metricName, 0, TSDB_TABLE_NAME_LEN);
//ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord));
//if (ret <= 0) break;
ret = read(fd, tableName, TSDB_TABLE_NAME_LEN);
if (ret <= 0) break;
ret = read(fd, metricName, TSDB_TABLE_NAME_LEN);
if (ret <= 0) break;
//tableInfo.name[sizeof(tableInfo.name) - 1] = 0;
//tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0;
//taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp);
//tstrncpy(tableName, tableInfo.name, TSDB_TABLE_NAME_LEN-1);
//tstrncpy(metricName, tableInfo.metric, TSDB_TABLE_NAME_LEN-1);
taosDumpTable(tableName, metricName, arguments, fp);
}
tclose(fd);
remove(".table.tmp");
close(fd);
(void)remove(".table.tmp");
return 0;
}
@ -1004,7 +1028,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) {
break;
}
}
pstr += sprintf(pstr, ")");
sprintf(pstr, ")");
count++;
fprintf(fp, "%s", buffer);
@ -1327,7 +1351,7 @@ int convertNCharToReadable(char *str, int size, char *buf, int bufsize) {
if ((int)wc < 256) {
pbuf = stpcpy(pbuf, ascii_literal_list[(int)wc]);
} else {
} else if (byte_width > 0) {
memcpy(pbuf, pstr, byte_width);
pbuf += byte_width;
}

2
src/mnode/src/mnodeAcct.c
View File

@ -126,8 +126,8 @@ int32_t mnodeInitAccts() {
}
void mnodeCleanupAccts() {
sdbCloseTable(tsAcctSdb);
acctCleanUp();
sdbCloseTable(tsAcctSdb);
}
void *mnodeGetAcct(char *name) {

4
src/mnode/src/mnodeBalance.c
View File

@ -25,8 +25,8 @@
int32_t balanceInit() { return TSDB_CODE_SUCCESS; }
void balanceCleanUp() {}
void balanceNotify() {}
void balanceUpdateMnode() {}
void balanceAsyncNotify() {}
void balanceSyncNotify() {}
void balanceReset() {}
int32_t balanceAllocVnodes(SVgObj *pVgroup) {

21
src/mnode/src/mnodeDb.c
View File

@ -311,6 +311,15 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->replications < 0) pCfg->replications = tsReplications;
}
static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
SDbObj *pDb = pMsg->pDb;
if (pDb != NULL) {
mLPrint("db:%s, is created by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
}
return code;
}
static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg) {
int32_t code = acctCheck(pAcct, ACCT_GRANT_DB);
if (code != 0) return code;
@ -364,18 +373,18 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs
.table = tsDbSdb,
.pObj = pDb,
.rowSize = sizeof(SDbObj),
.pMsg = pMsg
.pMsg = pMsg,
.cb = mnodeCreateDbCb
};
code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
tfree(pDb);
mLPrint("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code));
return code;
} else {
mLPrint("db:%s, is created by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
return code;
}
bool mnodeCheckIsMonitorDB(char *db, char *monitordb) {
@ -910,7 +919,7 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
mTrace("db:%s, all vgroups is altered", pDb->name);
mLPrint("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
balanceNotify();
balanceAsyncNotify();
return TSDB_CODE_SUCCESS;
}

11
src/mnode/src/mnodeDnode.c
View File

@ -81,7 +81,7 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
mnodeDropAllDnodeVgroups(pDnode);
#endif
mnodeDropMnodeLocal(pDnode->dnodeId);
balanceNotify();
balanceAsyncNotify();
mTrace("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId);
return TSDB_CODE_SUCCESS;
@ -348,7 +348,6 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
pRsp->dnodeCfg.dnodeId = htonl(pDnode->dnodeId);
pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes);
mnodeGetMnodeInfos(&pRsp->mnodes);
SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp));
for (int32_t j = 0; j < openVnodes; ++j) {
@ -383,8 +382,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
mTrace("dnode:%d, from offline to online", pDnode->dnodeId);
pDnode->status = TAOS_DN_STATUS_READY;
balanceUpdateMnode();
balanceNotify();
balanceSyncNotify();
balanceAsyncNotify();
}
if (openVnodes != pDnode->openVnodes) {
@ -392,6 +391,10 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
}
pDnode->lastAccess = tsAccessSquence;
//this func should be called after sdb replica changed
mnodeGetMnodeInfos(&pRsp->mnodes);
mnodeDecDnodeRef(pDnode);
pMsg->rpcRsp.len = contLen;

1
src/mnode/src/mnodeInt.c
View File

@ -54,6 +54,7 @@ void mnodeCleanupMsg(SMnodeMsg *pMsg) {
if (pMsg->pDb) mnodeDecDbRef(pMsg->pDb);
if (pMsg->pVgroup) mnodeDecVgroupRef(pMsg->pVgroup);
if (pMsg->pTable) mnodeDecTableRef(pMsg->pTable);
if (pMsg->pSTable) mnodeDecTableRef(pMsg->pSTable);
if (pMsg->pAcct) mnodeDecAcctRef(pMsg->pAcct);
if (pMsg->pDnode) mnodeDecDnodeRef(pMsg->pDnode);
}

51
src/mnode/src/mnodeSdb.c
View File

@ -63,7 +63,6 @@ typedef struct _SSdbTable {
int32_t (*encodeFp)(SSdbOper *pOper);
int32_t (*destroyFp)(SSdbOper *pOper);
int32_t (*restoredFp)();
pthread_mutex_t mutex;
} SSdbTable;
typedef struct {
@ -429,24 +428,18 @@ static SSdbRow *sdbGetRowMetaFromObj(SSdbTable *pTable, void *key) {
void *sdbGetRow(void *handle, void *key) {
SSdbTable *pTable = (SSdbTable *)handle;
SSdbRow * pMeta;
if (handle == NULL) return NULL;
pthread_mutex_lock(&pTable->mutex);
int32_t keySize = sizeof(int32_t);
if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
keySize = strlen((char *)key);
}
pMeta = taosHashGet(pTable->iHandle, key, keySize);
if (pMeta) sdbIncRef(pTable, pMeta->row);
pthread_mutex_unlock(&pTable->mutex);
if (pMeta == NULL) return NULL;
return pMeta->row;
SSdbRow *pMeta = taosHashGet(pTable->iHandle, key, keySize);
if (pMeta) {
sdbIncRef(pTable, pMeta->row);
return pMeta->row;
} else {
return NULL;
}
}
static void *sdbGetRowFromObj(SSdbTable *pTable, void *key) {
@ -458,8 +451,6 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
rowMeta.rowSize = pOper->rowSize;
rowMeta.row = pOper->pObj;
pthread_mutex_lock(&pTable->mutex);
void * key = sdbGetObjKey(pTable, pOper->pObj);
int32_t keySize = sizeof(int32_t);
@ -470,16 +461,14 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
taosHashPut(pTable->iHandle, key, keySize, &rowMeta, sizeof(SSdbRow));
sdbIncRef(pTable, pOper->pObj);
pTable->numOfRows++;
atomic_add_fetch_32(&pTable->numOfRows, 1);
if (pTable->keyType == SDB_KEY_AUTO) {
pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pOper->pObj));
} else {
pTable->autoIndex++;
atomic_add_fetch_32(&pTable->autoIndex, 1);
}
pthread_mutex_unlock(&pTable->mutex);
sdbTrace("table:%s, insert record:%s to hash, rowSize:%d numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion());
@ -490,20 +479,15 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
(*pTable->deleteFp)(pOper);
pthread_mutex_lock(&pTable->mutex);
void * key = sdbGetObjKey(pTable, pOper->pObj);
int32_t keySize = sizeof(int32_t);
if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
keySize = strlen((char *)key);
}
taosHashRemove(pTable->iHandle, key, keySize);
pTable->numOfRows--;
pthread_mutex_unlock(&pTable->mutex);
atomic_sub_fetch_32(&pTable->numOfRows, 1);
sdbTrace("table:%s, delete record:%s from hash, numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
@ -608,14 +592,12 @@ int32_t sdbInsertRow(SSdbOper *pOper) {
}
if (pTable->keyType == SDB_KEY_AUTO) {
pthread_mutex_lock(&pTable->mutex);
*((uint32_t *)pOper->pObj) = ++pTable->autoIndex;
*((uint32_t *)pOper->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1);
// let vgId increase from 2
if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) {
*((uint32_t *)pOper->pObj) = ++pTable->autoIndex;
*((uint32_t *)pOper->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1);
}
pthread_mutex_unlock(&pTable->mutex);
}
int32_t code = sdbInsertHash(pTable, pOper);
@ -805,8 +787,6 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
}
pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true);
pthread_mutex_init(&pTable->mutex, NULL);
tsSdbObj.numOfTables++;
tsSdbObj.tableList[pTable->tableId] = pTable;
return pTable;
@ -835,8 +815,6 @@ void sdbCloseTable(void *handle) {
taosHashDestroyIter(pIter);
taosHashCleanup(pTable->iHandle);
pthread_mutex_destroy(&pTable->mutex);
sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbObj.numOfTables);
free(pTable);
}
@ -873,6 +851,7 @@ void sdbCleanupWriteWorker() {
}
sdbFreeWritequeue();
tfree(tsSdbPool.writeWorker);
mPrint("sdb write is closed");
}

90
src/mnode/src/mnodeTable.c
View File

@ -780,6 +780,16 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
}
}
static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
if (pTable != NULL) {
mLPrint("app:%p:%p, stable:%s, is created in sdb, result:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
tstrerror(code));
}
return code;
}
static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
@ -819,27 +829,38 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
assert(tschema[col].type >= TSDB_DATA_TYPE_BOOL && tschema[col].type <= TSDB_DATA_TYPE_NCHAR);
}
pMsg->pTable = (STableObj *)pStable;
mnodeIncTableRef(pMsg->pTable);
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = sizeof(SSuperTableObj) + schemaSize,
.pMsg = pMsg
.pMsg = pMsg,
.cb = mnodeCreateSuperTableCb
};
int32_t code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
mnodeDestroySuperTable(pStable);
pMsg->pTable = NULL;
mError("app:%p:%p, table:%s, failed to create, sdb error", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return code;
} else {
mLPrint("app:%p:%p, table:%s, is created, tags:%d fields:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
pStable->numOfTags, pStable->numOfColumns);
code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
}
static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
if (pTable != NULL) {
mLPrint("app:%p:%p, stable:%s, is dropped from sdb, result:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
tstrerror(code));
}
return code;
}
static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
@ -870,19 +891,20 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
}
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.pMsg = pMsg
.pObj = pStable,
.pMsg = pMsg,
.cb = mnodeDropSuperTableCb
};
int32_t code = sdbDeleteRow(&oper);
if (code == TSDB_CODE_SUCCESS) {
mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code));
code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
return code;
int32_t code = sdbDeleteRow(&oper);
if (code != TSDB_CODE_SUCCESS) {
mError("app:%p:%p, table:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId);
return code;
} else {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
}
static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) {
@ -1535,10 +1557,16 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO
}
static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
if (pTable != NULL) {
mTrace("app:%p:%p, table:%s, create table in id:%d, uid:%" PRIu64 ", result:%s", pMsg->rpcMsg.ahandle, pMsg,
pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code));
}
if (code != TSDB_CODE_SUCCESS) return code;
SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, (SChildTableObj *)pMsg->pTable);
SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, pTable);
if (pMDCreate == NULL) {
return terrno;
}
@ -1578,19 +1606,18 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
if (pTable->info.type == TSDB_CHILD_TABLE) {
STagData *pTagData = (STagData *) pCreate->schema; // it is a tag key
SSuperTableObj *pSuperTable = mnodeGetSuperTable(pTagData->name);
if (pSuperTable == NULL) {
if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name);
if (pMsg->pSTable == NULL) {
mError("app:%p:%p, table:%s, corresponding super table:%s does not exist", pMsg->rpcMsg.ahandle, pMsg,
pCreate->tableId, pTagData->name);
mnodeDestroyChildTable(pTable);
return TSDB_CODE_MND_INVALID_TABLE_NAME;
}
mnodeDecTableRef(pSuperTable);
pTable->suid = pSuperTable->uid;
pTable->suid = pMsg->pSTable->uid;
pTable->uid = (((uint64_t)pTable->vgId) << 40) + ((((uint64_t)pTable->sid) & ((1ul << 24) - 1ul)) << 16) +
(sdbGetVersion() & ((1ul << 16) - 1ul));
pTable->superTable = pSuperTable;
pTable->superTable = pMsg->pSTable;
} else {
pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul));
pTable->sversion = 0;
@ -1639,15 +1666,13 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
int32_t code = sdbInsertRow(&desc);
if (code != TSDB_CODE_SUCCESS) {
free(pTable);
mnodeDestroyChildTable(pTable);
pMsg->pTable = NULL;
mError("app:%p:%p, table:%s, update sdb error, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
tstrerror(code));
pMsg->pTable = NULL;
return code;
return code;
} else {
mTrace("app:%p:%p, table:%s, create table in vgroup:%d, id:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
pTable->info.tableId, pVgroup->vgId, pTable->sid, pTable->uid);
return TSDB_CODE_SUCCESS;
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
}
@ -1683,12 +1708,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
mTrace("app:%p:%p, table:%s, create table in vgroup, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
pVgroup->vgId, sid);
code = mnodeDoCreateChildTable(pMsg, sid);
if (code != TSDB_CODE_SUCCESS) {
return code;
} else {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
return mnodeDoCreateChildTable(pMsg, sid);
}
} else {
if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId);

5
src/mnode/src/mnodeVgroup.c
View File

@ -678,8 +678,9 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
SMnodeMsg *mnodeMsg = rpcMsg->handle;
mnodeMsg->received++;
if (rpcMsg->code == TSDB_CODE_SUCCESS) {
mnodeMsg->code = rpcMsg->code;
mnodeMsg->successed++;
} else {
mnodeMsg->code = rpcMsg->code;
}
SVgObj *pVgroup = mnodeMsg->pVgroup;
@ -702,7 +703,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
code = TSDB_CODE_MND_SDB_ERROR;
}
dnodeSendRpcMnodeWriteRsp(mnodeMsg, code);
dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code);
}
}

77
src/os/linux/src/linuxPlatform.c
View File

@ -56,70 +56,8 @@ void taosMsleep(int mseconds) {
bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; }
void taosResetPthread(pthread_t *thread) { *thread = 0; }
int64_t taosGetPthreadId() { return (int64_t)pthread_self(); }
/*
* Function to get the private ip address of current machine. If get IP
* successfully, return 0, else, return -1. The return values is ip.
*
* Use:
* if (taosGetPrivateIp(ip) != 0) {
* perror("Fail to get private IP address\n");
* exit(EXIT_FAILURE);
* }
*/
int taosGetPrivateIp(char *const ip) {
bool hasLoCard = false;
struct ifaddrs *ifaddr, *ifa;
int family, s;
char host[NI_MAXHOST];
if (getifaddrs(&ifaddr) == -1) {
return -1;
}
/* Walk through linked list, maintaining head pointer so we can free list later */
int flag = 0;
for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
if (ifa->ifa_addr == NULL) continue;
family = ifa->ifa_addr->sa_family;
if (strcmp("lo", ifa->ifa_name) == 0) {
hasLoCard = true;
continue;
}
if (family == AF_INET) {
/* printf("%-8s", ifa->ifa_name); */
s = getnameinfo(ifa->ifa_addr, (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6),
host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
if (s != 0) {
freeifaddrs(ifaddr);
return -1;
}
strcpy(ip, host);
flag = 1;
break;
}
}
freeifaddrs(ifaddr);
if (flag) {
return 0;
} else {
if (hasLoCard) {
uPrint("no net card was found, use lo:127.0.0.1 as default");
strcpy(ip, "127.0.0.1");
return 0;
}
return -1;
}
}
int taosSetNonblocking(int sock, int on) {
int flags = 0;
if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
@ -294,21 +232,6 @@ ssize_t twrite(int fd, void *buf, size_t n) {
return n;
}
bool taosSkipSocketCheck() {
struct utsname buf;
if (uname(&buf)) {
uPrint("can't fetch os info");
return false;
}
if (strstr(buf.release, "Microsoft") != 0) {
uPrint("using WSLv1");
return true;
}
return false;
}
void taosBlockSIGPIPE() {
sigset_t signal_mask;
sigemptyset(&signal_mask);

16
src/os/linux/src/linuxSysPara.c
View File

@ -162,7 +162,13 @@ static void taosGetSystemTimezone() {
FILE *f = fopen("/etc/timezone", "r");
char buf[65] = {0};
if (f != NULL) {
fread(buf, 64, 1, f);
int len = fread(buf, 64, 1, f);
if(len < 64 && ferror(f)) {
fclose(f);
uError("read /etc/timezone error, reason:%s", strerror(errno));
return;
}
fclose(f);
}
@ -547,7 +553,7 @@ void taosSetCoreDump() {
struct rlimit rlim;
struct rlimit rlim_new;
if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
uPrint("the old unlimited para: rlim_cur=%d, rlim_max=%d", rlim.rlim_cur, rlim.rlim_max);
uPrint("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
rlim_new.rlim_cur = RLIM_INFINITY;
rlim_new.rlim_max = RLIM_INFINITY;
if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) {
@ -559,7 +565,7 @@ void taosSetCoreDump() {
}
if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
uPrint("the new unlimited para: rlim_cur=%d, rlim_max=%d", rlim.rlim_cur, rlim.rlim_max);
uPrint("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
}
#ifndef _TD_ARM_
@ -586,7 +592,7 @@ void taosSetCoreDump() {
uPrint("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno));
}
uPrint("The old core_uses_pid[%d]: %d", old_len, old_usespid);
uPrint("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
old_usespid = 0;
@ -603,7 +609,7 @@ void taosSetCoreDump() {
uPrint("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno));
}
uPrint("The new core_uses_pid[%d]: %d", old_len, old_usespid);
uPrint("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
#endif
#if 0

14
src/query/inc/qExecutor.h
View File

@ -102,7 +102,7 @@ typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct
int64_t tag;
STimeWindow win;
STSCursor cur;
STableId id; // for retrieve the page id list
void* pTable; // for retrieve the page id list
SWindowResInfo windowResInfo;
} STableQueryInfo;
@ -126,10 +126,10 @@ typedef struct SQueryCostInfo {
uint64_t computTime;
} SQueryCostInfo;
typedef struct SGroupItem {
STableId id;
STableQueryInfo* info;
} SGroupItem;
//typedef struct SGroupItem {
// void *pTable;
// STableQueryInfo *info;
//} SGroupItem;
typedef struct SQuery {
int16_t numOfCols;
@ -187,8 +187,8 @@ typedef struct SQInfo {
void* tsdb;
int32_t vgId;
STableGroupInfo tableIdGroupInfo; // table id list < only includes the STableId list>
STableGroupInfo groupInfo; //
STableGroupInfo tableGroupInfo; // table id list < only includes the STable list>
STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray<STableQueryInfo*> structure
SQueryRuntimeEnv runtimeEnv;
int32_t groupIndex;
int32_t offset; // offset in group result set of subgroup, todo refactor

374
src/query/src/qExecutor.c
View File

@ -113,7 +113,7 @@ static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
static bool hasMainOutput(SQuery *pQuery);
static void buildTagQueryResult(SQInfo *pQInfo);
static int32_t setAdditionalInfo(SQInfo *pQInfo, STableId *pTableId, STableQueryInfo *pTableQueryInfo);
static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo);
static int32_t flushFromResultBuf(SQInfo *pQInfo);
bool doFilterData(SQuery *pQuery, int32_t elemPos) {
@ -1495,6 +1495,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
}
static bool isQueryKilled(SQInfo *pQInfo) {
return false;
return (pQInfo->code == TSDB_CODE_TSC_QUERY_CANCELLED);
}
@ -1781,7 +1782,7 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) {
if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
num = 128;
} else if (isIntervalQuery(pQuery)) { // time window query, allocate one page for each table
size_t s = pQInfo->groupInfo.numOfTables;
size_t s = pQInfo->tableqinfoGroupInfo.numOfTables;
num = MAX(s, INITIAL_RESULT_ROWS_VALUE);
} else { // for super table query, one page for each subset
num = 1; // pQInfo->pSidSet->numOfSubSet;
@ -1815,18 +1816,18 @@ char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWi
*/
UNUSED_FUNC void vnodeDecMeterRefcnt(SQInfo *pQInfo) {
if (pQInfo != NULL) {
// assert(taosHashGetSize(pQInfo->groupInfo) >= 1);
// assert(taosHashGetSize(pQInfo->tableqinfoGroupInfo) >= 1);
}
#if 0
if (pQInfo == NULL || pQInfo->groupInfo.numOfTables == 1) {
if (pQInfo == NULL || pQInfo->tableqinfoGroupInfo.numOfTables == 1) {
atomic_fetch_sub_32(&pQInfo->pObj->numOfQueries, 1);
qTrace("QInfo:%p vid:%d sid:%d meterId:%s, query is over, numOfQueries:%d", pQInfo, pQInfo->pObj->vnode,
pQInfo->pObj->sid, pQInfo->pObj->meterId, pQInfo->pObj->numOfQueries);
} else {
int32_t num = 0;
for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) {
SMeterObj *pMeter = getMeterObj(pQInfo->groupInfo, pQInfo->pSidSet->pTableIdList[i]->sid);
for (int32_t i = 0; i < pQInfo->tableqinfoGroupInfo.numOfTables; ++i) {
SMeterObj *pMeter = getMeterObj(pQInfo->tableqinfoGroupInfo, pQInfo->pSidSet->pTableIdList[i]->sid);
atomic_fetch_sub_32(&(pMeter->numOfQueries), 1);
if (pMeter->numOfQueries > 0) {
@ -1840,9 +1841,9 @@ UNUSED_FUNC void vnodeDecMeterRefcnt(SQInfo *pQInfo) {
* in order to reduce log output, for all meters of which numOfQueries count are 0,
* we do not output corresponding information
*/
num = pQInfo->groupInfo.numOfTables - num;
num = pQInfo->tableqinfoGroupInfo.numOfTables - num;
qTrace("QInfo:%p metric query is over, dec query ref for %d meters, numOfQueries on %d meters are 0", pQInfo,
pQInfo->groupInfo.numOfTables, num);
pQInfo->tableqinfoGroupInfo.numOfTables, num);
}
#endif
}
@ -2188,16 +2189,16 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
* set tag value in SQLFunctionCtx
* e.g.,tag information into input buffer
*/
static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) {
static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) {
tVariantDestroy(tag);
if (tagColId == TSDB_TBNAME_COLUMN_INDEX) {
char* val = tsdbGetTableName(tsdb, pTableId);
char* val = tsdbGetTableName(pTable);
assert(val != NULL);
tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), TSDB_DATA_TYPE_BINARY);
} else {
char* val = tsdbGetTableTagVal(tsdb, pTableId, tagColId, type, bytes);
char* val = tsdbGetTableTagVal(pTable, tagColId, type, bytes);
if (val == NULL) {
tag->nType = TSDB_DATA_TYPE_NULL;
return;
@ -2221,7 +2222,7 @@ static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColI
}
}
void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId* pTableId, void *tsdb) {
void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) {
SQuery *pQuery = pRuntimeEnv->pQuery;
SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
@ -2238,7 +2239,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId* pTableId, void *tsdb) {
}
}
doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
doSetTagValueInParam(tsdb, pTable, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
} else {
// set tag value, by which the results are aggregated.
for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) {
@ -2250,7 +2251,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId* pTableId, void *tsdb) {
}
// todo use tag column index to optimize performance
doSetTagValueInParam(tsdb, pTableId, pLocalExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag,
doSetTagValueInParam(tsdb, pTable, pLocalExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag,
pLocalExprInfo->type, pLocalExprInfo->bytes);
}
@ -2269,7 +2270,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId* pTableId, void *tsdb) {
}
}
doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
doSetTagValueInParam(tsdb, pTable, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
qTrace("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64,
pRuntimeEnv->pCtx[0].tag.i64Key)
}
@ -2474,10 +2475,10 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) {
int64_t st = taosGetTimestampMs();
int32_t ret = TSDB_CODE_SUCCESS;
int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
int32_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
while (pQInfo->groupIndex < numOfGroups) {
SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, pQInfo->groupIndex);
ret = mergeIntoGroupResultImpl(pQInfo, group);
if (ret < 0) { // not enough disk space to save the data into disk
return -1;
@ -2510,9 +2511,9 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
}
// check if all results has been sent to client
int32_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
int32_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == numOfGroup) {
pQInfo->tableIndex = pQInfo->groupInfo.numOfTables; // set query completed
pQInfo->tableIndex = pQInfo->tableqinfoGroupInfo.numOfTables; // set query completed
return;
}
}
@ -2599,12 +2600,11 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
// todo opt for the case of one table per group
int32_t numOfTables = 0;
for (int32_t i = 0; i < size; ++i) {
SGroupItem *item = taosArrayGet(pGroup, i);
STableQueryInfo *pInfo = item->info;
STableQueryInfo *item = taosArrayGetP(pGroup, i);
SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, pInfo->id.tid);
if (list.size > 0 && pInfo->windowResInfo.size > 0) {
pTableList[numOfTables] = pInfo;
SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, tsdbGetTableId(item->pTable).tid);
if (list.size > 0 && item->windowResInfo.size > 0) {
pTableList[numOfTables] = item;
numOfTables += 1;
}
}
@ -2845,15 +2845,15 @@ void disableFuncInReverseScan(SQInfo *pQInfo) {
}
}
int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
int32_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
for(int32_t i = 0; i < numOfGroups; ++i) {
SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i);
SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i);
size_t t = taosArrayGetSize(group);
for (int32_t j = 0; j < t; ++j) {
SGroupItem *item = taosArrayGet(group, j);
updateTableQueryInfoForReverseScan(pQuery, item->info);
STableQueryInfo *pCheckInfo = taosArrayGetP(group, j);
updateTableQueryInfoForReverseScan(pQuery, pCheckInfo);
}
}
}
@ -3088,7 +3088,8 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI
tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
}
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
// add ref for table
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
switchCtxOrder(pRuntimeEnv);
@ -3160,7 +3161,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
}
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex;
setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
@ -3236,17 +3237,13 @@ static bool hasMainOutput(SQuery *pQuery) {
return false;
}
static STableQueryInfo *createTableQueryInfo(
SQueryRuntimeEnv *pRuntimeEnv,
STableId tableId,
STimeWindow win
) {
static STableQueryInfo *createTableQueryInfo( SQueryRuntimeEnv *pRuntimeEnv, void* pTable, STimeWindow win) {
STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo));
pTableQueryInfo->win = win;
pTableQueryInfo->lastKey = win.skey;
pTableQueryInfo->id = tableId;
pTableQueryInfo->pTable = pTable;
pTableQueryInfo->cur.vgroupIndex = -1;
initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, 100, 100, TSDB_DATA_TYPE_INT);
@ -3275,7 +3272,7 @@ void setCurrentQueryTable(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTable
* @param pRuntimeEnv
* @param pDataBlockInfo
*/
void setExecutionContext(SQInfo *pQInfo, STableId* pTableId, int32_t groupIndex, TSKEY nextKey) {
void setExecutionContext(SQInfo *pQInfo, void* pTable, int32_t groupIndex, TSKEY nextKey) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current;
@ -3302,7 +3299,7 @@ void setExecutionContext(SQInfo *pQInfo, STableId* pTableId, int32_t groupIndex,
initCtxOutputBuf(pRuntimeEnv);
pTableQueryInfo->lastKey = nextKey;
setAdditionalInfo(pQInfo, pTableId, pTableQueryInfo);
setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo);
}
void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) {
@ -3362,11 +3359,10 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
}
}
int32_t setAdditionalInfo(SQInfo *pQInfo, STableId* pTableId, STableQueryInfo *pTableQueryInfo) {
int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQueryInfo) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
//assert(pTableQueryInfo->lastKey >= TSKEY_INITIAL_VAL);
setTagVal(pRuntimeEnv, pTableId, pQInfo->tsdb);
setTagVal(pRuntimeEnv, pTable, pQInfo->tsdb);
// both the master and supplement scan needs to set the correct ts comp start position
if (pRuntimeEnv->pTSBuf != NULL) {
@ -3469,7 +3465,7 @@ static int32_t getNumOfSubset(SQInfo *pQInfo) {
if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (isIntervalQuery(pQuery))) {
totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo);
} else {
totalSubset = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
totalSubset = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
}
return totalSubset;
@ -3669,7 +3665,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
// all data returned, set query over
if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
if (pQInfo->runtimeEnv.stableQuery) {
if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) {
if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) {
setQueryStatus(pQuery, QUERY_OVER);
}
} else {
@ -3979,23 +3975,23 @@ static void setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) {
};
if (!isSTableQuery
&& (pQInfo->groupInfo.numOfTables == 1)
&& (pQInfo->tableqinfoGroupInfo.numOfTables == 1)
&& (cond.order == TSDB_ORDER_ASC)
&& (!isIntervalQuery(pQuery))
&& (!isGroupbyNormalCol(pQuery->pGroupbyExpr))
&& (!isFixedOutputQuery(pQuery))
) {
SArray* pa = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
SGroupItem* pItem = taosArrayGet(pa, 0);
cond.twindow = pItem->info->win;
SArray* pa = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
STableQueryInfo* pCheckInfo = taosArrayGetP(pa, 0);
cond.twindow = pCheckInfo->win;
}
if (isFirstLastRowQuery(pQuery)) {
pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
} else if (isPointInterpoQuery(pQuery)) {
pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
} else {
pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
}
}
@ -4132,18 +4128,18 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
STableQueryInfo *pTableQueryInfo = NULL;
// todo opt performance using hash table
size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
size_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
for (int32_t i = 0; i < numOfGroup; ++i) {
SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i);
SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i);
size_t num = taosArrayGetSize(group);
for (int32_t j = 0; j < num; ++j) {
SGroupItem *item = taosArrayGet(group, j);
STableQueryInfo *pInfo = item->info;
STableQueryInfo *p = taosArrayGetP(group, j);
if (pInfo->id.tid == blockInfo.tid) {
assert(pInfo->id.uid == blockInfo.uid);
pTableQueryInfo = item->info;
STableId id = tsdbGetTableId(p->pTable);
if (id.tid == blockInfo.tid) {
assert(id.uid == blockInfo.uid);
pTableQueryInfo = p;
break;
}
@ -4163,11 +4159,11 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
if (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
if (!isIntervalQuery(pQuery)) {
int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1;
setExecutionContext(pQInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIndex, blockInfo.window.ekey + step);
setExecutionContext(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo->groupIndex, blockInfo.window.ekey + step);
} else { // interval query
TSKEY nextKey = blockInfo.window.skey;
setIntervalQueryRange(pQInfo, nextKey);
/*int32_t ret = */setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo);
/*int32_t ret = */setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo);
}
}
@ -4187,16 +4183,17 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
SQuery * pQuery = pRuntimeEnv->pQuery;
setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
SGroupItem* item = taosArrayGet(group, index);
SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
STableQueryInfo* pCheckInfo = taosArrayGetP(group, index);
setTagVal(pRuntimeEnv, &item->id, pQInfo->tsdb);
setTagVal(pRuntimeEnv, pCheckInfo->pTable, pQInfo->tsdb);
STableId id = tsdbGetTableId(pCheckInfo->pTable);
qTrace("QInfo:%p query on (%d): uid:%" PRIu64 ", tid:%d, qrange:%" PRId64 "-%" PRId64, pQInfo, index,
item->id.uid, item->id.tid, item->info->lastKey, item->info->win.ekey);
id.uid, id.tid, pCheckInfo->lastKey, pCheckInfo->win.ekey);
STsdbQueryCond cond = {
.twindow = {item->info->lastKey, item->info->win.ekey},
.twindow = {pCheckInfo->lastKey, pCheckInfo->win.ekey},
.order = pQuery->order.order,
.colList = pQuery->colList,
.numOfCols = pQuery->numOfCols,
@ -4204,9 +4201,9 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
// todo refactor
SArray *g1 = taosArrayInit(1, POINTER_BYTES);
SArray *tx = taosArrayInit(1, sizeof(STableId));
SArray *tx = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(tx, &item->info->id);
taosArrayPush(tx, &pCheckInfo->pTable);
taosArrayPush(g1, &tx);
STableGroupInfo gp = {.numOfTables = 1, .pGroupList = g1};
@ -4250,14 +4247,14 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
SQuery * pQuery = pRuntimeEnv->pQuery;
setQueryStatus(pQuery, QUERY_COMPLETED);
size_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
size_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
if (isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery)) {
resetCtxOutputBuf(pRuntimeEnv);
assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0);
while (pQInfo->groupIndex < numOfGroups) {
SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex);
qTrace("QInfo:%p last_row query on group:%d, total group:%zu, current group:%p", pQInfo, pQInfo->groupIndex,
numOfGroups, group);
@ -4289,10 +4286,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
initCtxOutputBuf(pRuntimeEnv);
SArray* s = tsdbGetQueriedTableIdList(pRuntimeEnv->pQueryHandle);
SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle);
assert(taosArrayGetSize(s) >= 1);
setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(s, 0), pQInfo->tsdb);
setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb);
if (isFirstLastRowQuery(pQuery)) {
assert(taosArrayGetSize(s) == 1);
}
@ -4300,7 +4297,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
taosArrayDestroy(s);
// here we simply set the first table as current table
pQuery->current = ((SGroupItem*) taosArrayGet(group, 0))->info;
pQuery->current = (STableQueryInfo*) taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
int64_t numOfRes = getNumOfResult(pRuntimeEnv);
@ -4322,7 +4319,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
}
} else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group-by on normal columns query
while (pQInfo->groupIndex < numOfGroups) {
SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex);
qTrace("QInfo:%p group by normal columns group:%d, total group:%zu", pQInfo, pQInfo->groupIndex, numOfGroups);
@ -4347,10 +4344,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
SArray* s = tsdbGetQueriedTableIdList(pRuntimeEnv->pQueryHandle);
SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle);
assert(taosArrayGetSize(s) >= 1);
setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(s, 0), pQInfo->tsdb);
setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb);
// here we simply set the first table as current table
scanMultiTableDataBlocks(pQInfo);
@ -4406,25 +4403,23 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
}
// all data have returned already
if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) {
if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) {
return;
}
resetCtxOutputBuf(pRuntimeEnv);
resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo);
SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
assert(taosArrayGetSize(group) == pQInfo->groupInfo.numOfTables &&
1 == taosArrayGetSize(pQInfo->groupInfo.pGroupList));
SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables &&
1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList));
while (pQInfo->tableIndex < pQInfo->groupInfo.numOfTables) {
while (pQInfo->tableIndex < pQInfo->tableqinfoGroupInfo.numOfTables) {
if (isQueryKilled(pQInfo)) {
return;
}
SGroupItem *item = taosArrayGet(group, pQInfo->tableIndex);
pQuery->current = item->info;
pQuery->current = taosArrayGetP(group, pQInfo->tableIndex);
if (!multiTableMultioutputHelper(pQInfo, pQInfo->tableIndex)) {
pQInfo->tableIndex++;
continue;
@ -4444,7 +4439,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
// the limitation of output result is reached, set the query completed
if (limitResults(pRuntimeEnv)) {
pQInfo->tableIndex = pQInfo->groupInfo.numOfTables;
pQInfo->tableIndex = pQInfo->tableqinfoGroupInfo.numOfTables;
break;
}
@ -4461,8 +4456,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
pQInfo->tableIndex++;
STableIdInfo tidInfo;
tidInfo.uid = item->id.uid;
tidInfo.tid = item->id.tid;
STableId id = tsdbGetTableId(pQuery->current->pTable);
tidInfo.uid = id.uid;
tidInfo.tid = id.tid;
tidInfo.key = pQuery->current->lastKey;
taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo);
@ -4485,7 +4482,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
}
}
if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) {
if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) {
setQueryStatus(pQuery, QUERY_COMPLETED);
}
}
@ -4511,7 +4508,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
qTrace(
"QInfo %p numOfTables:%"PRIu64", index:%d, numOfGroups:%zu, %"PRId64" points returned, total:%"PRId64", offset:%" PRId64,
pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total,
pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total,
pQuery->limit.offset);
}
@ -4539,7 +4536,7 @@ static void doSaveContext(SQInfo *pQInfo) {
tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
}
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
switchCtxOrder(pRuntimeEnv);
@ -4564,14 +4561,14 @@ static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) {
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
if (isIntervalQuery(pQuery)) {
size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
size_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
for (int32_t i = 0; i < numOfGroup; ++i) {
SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i);
SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i);
size_t num = taosArrayGetSize(group);
for (int32_t j = 0; j < num; ++j) {
SGroupItem* item = taosArrayGet(group, j);
closeAllTimeWindow(&item->info->windowResInfo);
STableQueryInfo* item = taosArrayGetP(group, j);
closeAllTimeWindow(&item->windowResInfo);
}
}
} else { // close results for group result
@ -4732,8 +4729,10 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
pQuery->current->lastKey, pQuery->window.ekey);
} else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
STableIdInfo tidInfo;
tidInfo.uid = pQuery->current->id.uid;
tidInfo.tid = pQuery->current->id.tid;
STableId id = tsdbGetTableId(pQuery->current);
tidInfo.uid = id.uid;
tidInfo.tid = id.tid;
tidInfo.key = pQuery->current->lastKey;
taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo);
}
@ -4883,23 +4882,23 @@ static void tableQueryImpl(SQInfo *pQInfo) {
pQuery->rec.rows = 0;
int64_t st = taosGetTimestampUs();
assert(pQInfo->groupInfo.numOfTables == 1);
SArray* g = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
SGroupItem* item = taosArrayGet(g, 0);
assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1);
SArray* g = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
STableQueryInfo* item = taosArrayGetP(g, 0);
// group by normal column, sliding window query, interval query are handled by interval query processor
if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation)
tableIntervalProcess(pQInfo, item->info);
tableIntervalProcess(pQInfo, item);
} else if (isFixedOutputQuery(pQuery)) {
tableFixedOutputProcess(pQInfo, item->info);
tableFixedOutputProcess(pQInfo, item);
} else { // diff/add/multiply/subtract/division
assert(pQuery->checkBuffer == 1);
tableMultiOutputProcess(pQInfo, item->info);
tableMultiOutputProcess(pQInfo, item);
}
// record the total elapsed time
pRuntimeEnv->summary.elapsedTime += (taosGetTimestampUs() - st);
assert(pQInfo->groupInfo.numOfTables == 1);
assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1);
/* check if query is killed or not */
if (isQueryKilled(pQInfo)) {
@ -4931,7 +4930,7 @@ static void stableQueryImpl(SQInfo *pQInfo) {
pQInfo->runtimeEnv.summary.elapsedTime += (taosGetTimestampUs() - st);
if (pQuery->rec.rows == 0) {
qTrace("QInfo:%p over, %zu tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->groupInfo.numOfTables, pQuery->rec.total);
qTrace("QInfo:%p over, %zu tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQuery->rec.total);
}
}
@ -5154,9 +5153,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
}
if (!validateQuerySourceCols(pQueryMsg, *pExpr)) {
tfree(*pExpr);
return TSDB_CODE_QRY_INVALID_MSG;
goto _cleanup;
}
pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList);
@ -5228,8 +5225,17 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols,
pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->intervalTime,
pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset);
return 0;
_cleanup:
tfree(*pExpr);
taosArrayDestroy(*pTableIdList);
*pTableIdList = NULL;
tfree(*tbnameCond);
tfree(*groupbyCols);
tfree(*tagCols);
tfree(*tagCond);
return TSDB_CODE_QRY_INVALID_MSG;
}
static int32_t buildAirthmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) {
@ -5487,7 +5493,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) {
}
static int compareTableIdInfo( const void* a, const void* b ) {
static int compareTableIdInfo(const void* a, const void* b) {
const STableIdInfo* x = (const STableIdInfo*)a;
const STableIdInfo* y = (const STableIdInfo*)b;
if (x->uid > y->uid) return 1;
@ -5495,8 +5501,10 @@ static int compareTableIdInfo( const void* a, const void* b ) {
return 0;
}
static void freeQInfo(SQInfo *pQInfo);
static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
STableGroupInfo *groupInfo, SColumnInfo* pTagCols) {
STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols) {
SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo));
if (pQInfo == NULL) {
return NULL;
@ -5582,39 +5590,41 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
// to make sure third party won't overwrite this structure
pQInfo->signature = pQInfo;
pQInfo->tableIdGroupInfo = *groupInfo;
size_t numOfGroups = taosArrayGetSize(groupInfo->pGroupList);
pQInfo->tableGroupInfo = *pTableGroupInfo;
size_t numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList);
pQInfo->groupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES);
pQInfo->groupInfo.numOfTables = groupInfo->numOfTables;
pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES);
pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables;
int tableIndex = 0;
STimeWindow window = pQueryMsg->window;
taosArraySort(pTableIdList, compareTableIdInfo);
for(int32_t i = 0; i < numOfGroups; ++i) {
SArray* pa = taosArrayGetP(groupInfo->pGroupList, i);
SArray* pa = taosArrayGetP(pTableGroupInfo->pGroupList, i);
size_t s = taosArrayGetSize(pa);
SArray* p1 = taosArrayInit(s, sizeof(SGroupItem));
SArray* p1 = taosArrayInit(s, POINTER_BYTES);
for(int32_t j = 0; j < s; ++j) {
STableId id = *(STableId*) taosArrayGet(pa, j);
SGroupItem item = { .id = id };
void* pTable = taosArrayGetP(pa, j);
// NOTE: compare STableIdInfo with STableId
STableIdInfo* pTableId = taosArraySearch( pTableIdList, &id, compareTableIdInfo);
STableId id = tsdbGetTableId(pTable);
STableIdInfo* pTableId = taosArraySearch(pTableIdList, &id, compareTableIdInfo);
if (pTableId != NULL ) {
window.skey = pTableId->key;
} else {
window.skey = pQueryMsg->window.skey;
}
item.info = createTableQueryInfo(&pQInfo->runtimeEnv, item.id, window);
item.info->groupIndex = i;
item.info->tableIndex = tableIndex++;
STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, pTable, window);
item->groupIndex = i;
item->tableIndex = tableIndex++;
taosArrayPush(p1, &item);
}
taosArrayPush(pQInfo->groupInfo.pGroupList, &p1);
taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1);
}
pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo));
@ -5633,22 +5643,27 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
return pQInfo;
_cleanup:
tfree(pQuery->fillVal);
//tfree(pQuery->fillVal);
if (pQuery->sdata != NULL) {
for (int16_t col = 0; col < pQuery->numOfOutput; ++col) {
tfree(pQuery->sdata[col]);
}
}
//if (pQuery->sdata != NULL) {
// for (int16_t col = 0; col < pQuery->numOfOutput; ++col) {
// tfree(pQuery->sdata[col]);
// }
//}
tfree(pQuery->sdata);
tfree(pQuery->pFilterInfo);
tfree(pQuery->colList);
//
//tfree(pQuery->sdata);
//tfree(pQuery->pFilterInfo);
//tfree(pQuery->colList);
tfree(pExprs);
tfree(pGroupbyExpr);
//tfree(pExprs);
//tfree(pGroupbyExpr);
tfree(pQInfo);
//taosArrayDestroy(pQInfo->arrTableIdInfo);
//tsdbDestoryTableGroup(&pQInfo->tableGroupInfo);
//
//tfree(pQInfo);
freeQInfo(pQInfo);
return NULL;
}
@ -5667,7 +5682,6 @@ static bool isValidQInfo(void *param) {
return (sig == (uint64_t)pQInfo);
}
static void freeQInfo(SQInfo *pQInfo);
static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) {
int32_t code = TSDB_CODE_SUCCESS;
@ -5693,7 +5707,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
return TSDB_CODE_SUCCESS;
}
if (pQInfo->groupInfo.numOfTables == 0) {
if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) {
qTrace("QInfo:%p no table qualified for tag filter, abort query", pQInfo);
setQueryStatus(pQuery, QUERY_COMPLETED);
@ -5754,29 +5768,24 @@ static void freeQInfo(SQInfo *pQInfo) {
}
// todo refactor, extract method to destroytableDataInfo
int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
int32_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
for (int32_t i = 0; i < numOfGroups; ++i) {
SArray *p = taosArrayGetP(pQInfo->groupInfo.pGroupList, i);
SArray *p = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i);
size_t num = taosArrayGetSize(p);
for(int32_t j = 0; j < num; ++j) {
SGroupItem* item = taosArrayGet(p, j);
if (item->info != NULL) {
destroyTableQueryInfo(item->info, pQuery->numOfOutput);
STableQueryInfo* item = taosArrayGetP(p, j);
if (item != NULL) {
destroyTableQueryInfo(item, pQuery->numOfOutput);
}
}
taosArrayDestroy(p);
}
taosArrayDestroy(pQInfo->groupInfo.pGroupList);
taosArrayDestroy(pQInfo->tableqinfoGroupInfo.pGroupList);
for(int32_t i = 0; i < numOfGroups; ++i) {
SArray* p = taosArrayGetP(pQInfo->tableIdGroupInfo.pGroupList, i);
taosArrayDestroy(p);
}
taosArrayDestroy(pQInfo->tableIdGroupInfo.pGroupList);
tsdbDestoryTableGroup(&pQInfo->tableGroupInfo);
taosArrayDestroy(pQInfo->arrTableIdInfo);
if (pQuery->pGroupbyExpr != NULL) {
@ -5861,8 +5870,6 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) {
}
return TSDB_CODE_SUCCESS;
// todo if interpolation exists, the result may be dump to client by several rounds
}
int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) {
@ -5875,6 +5882,8 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
SSqlFuncMsg **pExprMsg = NULL;
SColIndex * pGroupColIndex = NULL;
SColumnInfo* pTagColumnInfo = NULL;
SExprInfo *pExprs = NULL;
SSqlGroupbyExpr *pGroupbyExpr = NULL;
if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo)) !=
TSDB_CODE_SUCCESS) {
@ -5893,24 +5902,24 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
goto _over;
}
SExprInfo *pExprs = NULL;
if ((code = createQFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) {
free(pExprMsg);
goto _over;
}
SSqlGroupbyExpr *pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, pGroupColIndex, &code);
pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, pGroupColIndex, &code);
if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) {
goto _over;
}
bool isSTableQuery = false;
STableGroupInfo groupInfo = {0};
STableGroupInfo tableGroupInfo = {0};
if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) {
STableIdInfo *id = taosArrayGet(pTableIdList, 0);
qTrace("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) {
if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &tableGroupInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
} else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) {
@ -5927,25 +5936,28 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
numOfGroupByCols = 0;
}
code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex,
code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex,
numOfGroupByCols);
if (code != TSDB_CODE_SUCCESS) {
goto _over;
}
} else {
groupInfo.pGroupList = taosArrayInit(1, POINTER_BYTES);
groupInfo.numOfTables = taosArrayGetSize(pTableIdList);
code = tsdbGetTableGroupFromIdList(tsdb, pTableIdList, &tableGroupInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _over;
}
SArray* p = taosArrayClone(pTableIdList);
taosArrayPush(groupInfo.pGroupList, &p);
qTrace("qmsg:%p query on %zu tables in one group from client", pQueryMsg, groupInfo.numOfTables);
qTrace("qmsg:%p query on %zu tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables);
}
} else {
assert(0);
}
(*pQInfo) = createQInfoImpl(pQueryMsg, pTableIdList, pGroupbyExpr, pExprs, &groupInfo, pTagColumnInfo);
(*pQInfo) = createQInfoImpl(pQueryMsg, pTableIdList, pGroupbyExpr, pExprs, &tableGroupInfo, pTagColumnInfo);
pExprs = NULL;
pGroupbyExpr = NULL;
pTagColumnInfo = NULL;
if ((*pQInfo) == NULL) {
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _over;
@ -5954,9 +5966,15 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery);
_over:
tfree(tagCond);
tfree(tbnameCond);
tfree(pGroupColIndex);
free(tagCond);
free(tbnameCond);
free(pGroupColIndex);
if (pGroupbyExpr != NULL) {
taosArrayDestroy(pGroupbyExpr->columnInfo);
free(pGroupbyExpr);
}
free(pTagColumnInfo);
free(pExprs);
taosArrayDestroy(pTableIdList);
//pQInfo already freed in initQInfo, but *pQInfo may not pointer to null;
@ -6134,17 +6152,17 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
SQuery * pQuery = pRuntimeEnv->pQuery;
size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
size_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
assert(numOfGroup == 0 || numOfGroup == 1);
if (numOfGroup == 0) {
return;
}
SArray* pa = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
SArray* pa = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
size_t num = taosArrayGetSize(pa);
assert(num == pQInfo->groupInfo.numOfTables);
assert(num == pQInfo->tableqinfoGroupInfo.numOfTables);
int32_t count = 0;
int32_t functionId = pQuery->pSelectExpr[0].base.functionId;
@ -6168,26 +6186,28 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
while(pQInfo->tableIndex < num && count < pQuery->rec.capacity) {
int32_t i = pQInfo->tableIndex++;
SGroupItem *item = taosArrayGet(pa, i);
STableQueryInfo *item = taosArrayGetP(pa, i);
char *output = pQuery->sdata[0]->data + i * rsize;
varDataSetLen(output, rsize - VARSTR_HEADER_SIZE);
output = varDataVal(output);
*(int64_t *)output = item->id.uid; // memory align problem, todo serialize
output += sizeof(item->id.uid);
STableId id = tsdbGetTableId(item->pTable);
*(int32_t *)output = item->id.tid;
output += sizeof(item->id.tid);
*(int64_t *)output = id.uid; // memory align problem, todo serialize
output += sizeof(id.uid);
*(int32_t *)output = id.tid;
output += sizeof(id.tid);
*(int32_t *)output = pQInfo->vgId;
output += sizeof(pQInfo->vgId);
if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
char *data = tsdbGetTableName(pQInfo->tsdb, &item->id);
char *data = tsdbGetTableName(item->pTable);
memcpy(output, data, varDataTLen(data));
} else {
char *val = tsdbGetTableTagVal(pQInfo->tsdb, &item->id, pExprInfo->base.colInfo.colId, type, bytes);
char *val = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes);
// todo refactor
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
@ -6223,18 +6243,18 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
int32_t i = pQInfo->tableIndex++;
SExprInfo* pExprInfo = pQuery->pSelectExpr;
SGroupItem* item = taosArrayGet(pa, i);
STableQueryInfo* item = taosArrayGetP(pa, i);
for(int32_t j = 0; j < pQuery->numOfOutput; ++j) {
if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
char* data = tsdbGetTableName(pQInfo->tsdb, &item->id);
char* data = tsdbGetTableName(item->pTable);
char* dst = pQuery->sdata[j]->data + count * tbnameSchema.bytes;
memcpy(dst, data, varDataTLen(data));
} else {// todo refactor
int16_t type = pExprInfo[j].type;
int16_t bytes = pExprInfo[j].bytes;
char* data = tsdbGetTableTagVal(pQInfo->tsdb, &item->id, pExprInfo[j].base.colInfo.colId, type, bytes);
char* data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes);
char* dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {

4
src/query/src/qUtil.c
View File

@ -114,8 +114,6 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
SWindowResult *pResult = &pWindowResInfo->pResult[i];
if (pResult->status.closed) { // remove the window slot from hash table
taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, pWindowResInfo->type);
printf("remove ============>%ld, remain size:%ld\n", pResult->window.skey, pWindowResInfo->hashList->size);
} else {
break;
}
@ -140,7 +138,7 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
SWindowResult *pResult = &pWindowResInfo->pResult[k];
int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey,
tDataTypeDesc[pWindowResInfo->type].nSize);
assert(p != NULL);
int32_t v = (*p - num);
assert(v >= 0 && v <= pWindowResInfo->size);
taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, tDataTypeDesc[pWindowResInfo->type].nSize,

11
src/query/src/qpercentile.c
View File

@ -879,8 +879,8 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
UNUSED(ret);
for (uint32_t jx = 0; jx < pFlushInfo->numOfPages; ++jx) {
ret = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
UNUSED(ret);
size_t sz = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
UNUSED(sz);
tMemBucketPut(pMemBucket, pPage->data, pPage->num);
}
@ -965,10 +965,11 @@ char *getFirstElemOfMemBuffer(tMemBucketSegment *pSeg, int32_t slotIdx, tFilePag
*/
tFlushoutInfo *pFlushInfo = &pMemBuffer->fileMeta.flushoutData.pFlushoutInfo[0];
assert(pFlushInfo->numOfPages == pMemBuffer->fileMeta.nFileSize);
fseek(pMemBuffer->file, pFlushInfo->startPageId * pMemBuffer->pageSize, SEEK_SET);
size_t ret = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
int32_t ret;
ret = fseek(pMemBuffer->file, pFlushInfo->startPageId * pMemBuffer->pageSize, SEEK_SET);
UNUSED(ret);
size_t sz = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
UNUSED(sz);
thisVal = pPage->data;
}
return thisVal;

4
src/query/src/qresultBuf.c
View File

@ -72,7 +72,7 @@ static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t numOf
if (ret != 0) {
// dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile,
// strerror(errno));
return -TSDB_CODE_QRY_NO_DISKSPACE;
return TSDB_CODE_QRY_NO_DISKSPACE;
}
pResultBuf->totalBufSize = pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE;
@ -80,7 +80,7 @@ static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t numOf
if (pResultBuf->pBuf == MAP_FAILED) {
// dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno));
return -TSDB_CODE_QRY_OUT_OF_MEMORY;
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;

2
src/query/src/qtokenizer.c
View File

@ -25,7 +25,7 @@
// All the keywords of the SQL language are stored in a hash table
typedef struct SKeyword {
const char* name; // The keyword name
uint8_t type; // type
uint16_t type; // type
uint8_t len; // length
} SKeyword;

63
src/query/src/qtsbuf.c
View File

@ -65,8 +65,10 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
// validate the file magic number
STSBufFileHeader header = {0};
fseek(pTSBuf->f, 0, SEEK_SET);
fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f);
int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET);
UNUSED(ret);
size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f);
UNUSED(sz);
// invalid file
if (header.magic != TS_COMP_FILE_MAGIC) {
@ -97,22 +99,30 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
size_t infoSize = sizeof(STSVnodeBlockInfo) * pTSBuf->numOfVnodes;
STSVnodeBlockInfo* buf = (STSVnodeBlockInfo*)calloc(1, infoSize);
if (buf == NULL) {
tsBufDestory(pTSBuf);
return NULL;
}
//int64_t pos = ftell(pTSBuf->f); //pos not used
fread(buf, infoSize, 1, pTSBuf->f);
sz = fread(buf, infoSize, 1, pTSBuf->f);
UNUSED(sz);
// the length value for each vnode is not kept in file, so does not set the length value
for (int32_t i = 0; i < pTSBuf->numOfVnodes; ++i) {
STSVnodeBlockInfoEx* pBlockList = &pTSBuf->pData[i];
memcpy(&pBlockList->info, &buf[i], sizeof(STSVnodeBlockInfo));
}
free(buf);
fseek(pTSBuf->f, 0, SEEK_END);
ret = fseek(pTSBuf->f, 0, SEEK_END);
UNUSED(ret);
struct stat fileStat;
fstat(fileno(pTSBuf->f), &fileStat);
if (fstat(fileno(pTSBuf->f), &fileStat) != 0) {
tsBufDestory(pTSBuf);
return NULL;
}
pTSBuf->fileSize = (uint32_t)fileStat.st_size;
tsBufResetPos(pTSBuf);
@ -278,19 +288,24 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
* set the right position for the reversed traverse, the reversed traverse is started from
* the end of each comp data block
*/
fseek(pTSBuf->f, -sizeof(pBlock->padding), SEEK_CUR);
fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
int32_t ret = fseek(pTSBuf->f, -sizeof(pBlock->padding), SEEK_CUR);
size_t sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
UNUSED(sz);
pBlock->compLen = pBlock->padding;
int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
fseek(pTSBuf->f, -offset, SEEK_CUR);
ret = fseek(pTSBuf->f, -offset, SEEK_CUR);
UNUSED(ret);
}
fread(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f);
fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
size_t sz = fread(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f);
UNUSED(sz);
sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
UNUSED(sz);
sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
UNUSED(sz);
sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
UNUSED(sz);
if (decomp) {
pTSBuf->tsData.len =
@ -299,12 +314,13 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
}
// read the comp length at the length of comp block
fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
UNUSED(sz);
// for backwards traverse, set the start position at the end of previous block
if (order == TSDB_ORDER_DESC) {
int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
int64_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
UNUSED(r);
}
@ -441,7 +457,8 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int
STSBlock* pBlock = &pTSBuf->block;
int32_t compBlockSize =
pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
UNUSED(ret);
}
return 0;
@ -538,7 +555,7 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC);
int64_t r = fseek(pTSBuf->f, 0, SEEK_SET);
int32_t r = fseek(pTSBuf->f, 0, SEEK_SET);
if (r != 0) {
return -1;
}
@ -743,7 +760,9 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) {
int32_t oldSize = pDestBuf->fileSize;
struct stat fileStat;
fstat(fileno(pDestBuf->f), &fileStat);
if (fstat(fileno(pDestBuf->f), &fileStat) != 0) {
return -1;
}
pDestBuf->fileSize = (uint32_t)fileStat.st_size;
assert(pDestBuf->fileSize == oldSize + size);
@ -766,8 +785,10 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
// update prev vnode length info in file
TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, pBlockInfo);
fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET);
fwrite((void*)pData, 1, len, pTSBuf->f);
int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET);
UNUSED(ret);
size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f);
UNUSED(sz);
pTSBuf->fileSize += len;
pTSBuf->tsOrder = order;

2
src/rpc/inc/rpcTcp.h
View File

@ -21,9 +21,11 @@ extern "C" {
#endif
void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
void taosStopTcpServer(void *param);
void taosCleanUpTcpServer(void *param);
void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle);
void taosStopTcpClient(void *chandle);
void taosCleanUpTcpClient(void *chandle);
void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);

1
src/rpc/inc/rpcUdp.h
View File

@ -23,6 +23,7 @@ extern "C" {
#include "taosdef.h"
void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int, void *fp, void *shandle);
void taosStopUdpConnection(void *handle);
void taosCleanUpUdpConnection(void *handle);
int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle);
void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);

45
src/rpc/src/rpcMain.c
View File

@ -58,7 +58,7 @@ typedef struct {
void (*cfp)(SRpcMsg *, SRpcIpSet *);
int (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey);
int refCount;
int32_t refCount;
void *idPool; // handle to ID pool
void *tmrCtrl; // handle to timer
SHashObj *hash; // handle returned by hash utility
@ -153,6 +153,13 @@ void (*taosCleanUpConn[])(void *thandle) = {
taosCleanUpTcpClient
};
void (*taosStopConn[])(void *thandle) = {
taosStopUdpConnection,
taosStopUdpConnection,
taosStopTcpServer,
taosStopTcpClient,
};
int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = {
taosSendUdpData,
taosSendUdpData,
@ -289,12 +296,18 @@ void *rpcOpen(const SRpcInit *pInit) {
void rpcClose(void *param) {
SRpcInfo *pRpc = (SRpcInfo *)param;
// stop connection to outside first
(*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
(*taosStopConn[pRpc->connType])(pRpc->udphandle);
// close all connections
for (int i = 0; i < pRpc->sessions; ++i) {
if (pRpc->connList && pRpc->connList[i].user[0]) {
rpcCloseConn((void *)(pRpc->connList + i));
}
}
// clean up
(*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
(*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
@ -421,6 +434,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
pConn->rspMsgLen = msgLen;
if (pMsg->code == TSDB_CODE_RPC_ACTION_IN_PROGRESS) pConn->inTranId--;
// stop the progress timer
taosTmrStopA(&pConn->pTimer);
// set the idle timer to monitor the activity
@ -588,6 +602,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
pConn->inTranId = 0;
pConn->outTranId = 0;
pConn->secured = 0;
pConn->peerId = 0;
pConn->peerIp = 0;
pConn->peerPort = 0;
pConn->pReqMsg = NULL;
@ -627,6 +642,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) {
pConn->spi = pRpc->spi;
pConn->encrypt = pRpc->encrypt;
if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN);
tTrace("%s %p client connection is allocated", pRpc->label, pConn);
}
return pConn;
@ -681,6 +697,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
}
taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES);
tTrace("%s %p server connection is allocated", pRpc->label, pConn);
}
return pConn;
@ -948,11 +965,9 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
terrno = 0;
pConn = rpcProcessMsgHead(pRpc, pRecv);
if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) {
tTrace("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x",
tTrace("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x",
pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, terrno,
pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->code);
}
int32_t code = terrno;
if (code != TSDB_CODE_RPC_ALREADY_PROCESSED) {
@ -1007,7 +1022,11 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
if ( rpcIsReq(pHead->msgType) ) {
rpcMsg.handle = pConn;
rpcAddRef(pRpc); // add the refCount for requests
// start the progress timer to monitor the response from server app
pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pRpc->tmrCtrl);
// notify the server app
(*(pRpc->cfp))(&rpcMsg, NULL);
} else {
// it's a response
@ -1180,16 +1199,14 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
msgLen = rpcAddAuthPart(pConn, msg, msgLen);
if ( rpcIsReq(pHead->msgType)) {
if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
tTrace("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort,
msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
tTrace("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort,
msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
} else {
if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured
if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
tTrace("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort,
htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
tTrace("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort,
htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
}
//tTrace("connection type is: %d", pConn->connType);
@ -1480,12 +1497,12 @@ static void rpcUnlockConn(SRpcConn *pConn) {
static void rpcAddRef(SRpcInfo *pRpc)
{
atomic_add_fetch_8(&pRpc->refCount, 1);
atomic_add_fetch_32(&pRpc->refCount, 1);
}
static void rpcDecRef(SRpcInfo *pRpc)
{
if (atomic_sub_fetch_8(&pRpc->refCount, 1) == 0) {
if (atomic_sub_fetch_32(&pRpc->refCount, 1) == 0) {
taosHashCleanup(pRpc->hash);
taosTmrCleanUp(pRpc->tmrCtrl);
taosIdPoolCleanUp(pRpc->idPool);

68
src/rpc/src/rpcTcp.c
View File

@ -190,22 +190,28 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
}
}
void taosCleanUpTcpServer(void *handle) {
void taosStopTcpServer(void *handle) {
SServerObj *pServerObj = handle;
SThreadObj *pThreadObj;
if (pServerObj == NULL) return;
if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD);
if(pServerObj->thread) pthread_join(pServerObj->thread, NULL);
tTrace("%s TCP server is stopped", pServerObj->label);
}
void taosCleanUpTcpServer(void *handle) {
SServerObj *pServerObj = handle;
SThreadObj *pThreadObj;
if (pServerObj == NULL) return;
for (int i = 0; i < pServerObj->numOfThreads; ++i) {
pThreadObj = pServerObj->pThreadObj + i;
taosStopTcpThread(pThreadObj);
pthread_mutex_destroy(&(pThreadObj->mutex));
}
tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label);
tTrace("%s TCP server is cleaned up", pServerObj->label);
tfree(pServerObj->pThreadObj);
tfree(pServerObj);
@ -226,7 +232,7 @@ static void *taosAcceptTcpConnection(void *arg) {
connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen);
if (connFd == -1) {
if (errno == EINVAL) {
tTrace("%s TCP server socket was shutdown, exiting...", pServerObj->label);
tTrace("%s TCP server stop accepting new connections, exiting", pServerObj->label);
break;
}
@ -242,13 +248,13 @@ static void *taosAcceptTcpConnection(void *arg) {
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd);
if (pFdObj) {
pFdObj->ip = caddr.sin_addr.s_addr;
pFdObj->port = caddr.sin_port;
tTrace("%s new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label,
inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds);
pFdObj->port = htons(caddr.sin_port);
tTrace("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label,
inet_ntoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds);
} else {
close(connFd);
tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno),
inet_ntoa(caddr.sin_addr), caddr.sin_port);
inet_ntoa(caddr.sin_addr), htons(caddr.sin_port));
}
// pick up next thread for next connection
@ -304,12 +310,19 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *
return pThreadObj;
}
void taosStopTcpClient(void *chandle) {
SThreadObj *pThreadObj = chandle;
if (pThreadObj == NULL) return;
tTrace ("%s TCP client is stopped", pThreadObj->label);
}
void taosCleanUpTcpClient(void *chandle) {
SThreadObj *pThreadObj = chandle;
if (pThreadObj == NULL) return;
taosStopTcpThread(pThreadObj);
tTrace ("%s, all connections are cleaned up", pThreadObj->label);
tTrace ("%s TCP client is cleaned up", pThreadObj->label);
tfree(pThreadObj);
}
@ -320,14 +333,22 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip);
if (fd < 0) return NULL;
struct sockaddr_in sin;
uint16_t localPort = 0;
unsigned int addrlen = sizeof(sin);
if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 &&
sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
localPort = (uint16_t)ntohs(sin.sin_port);
}
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd);
if (pFdObj) {
pFdObj->thandle = thandle;
pFdObj->port = port;
pFdObj->ip = ip;
tTrace("%s %p, TCP connection to 0x%x:%hu is created, FD:%p numOfFds:%d",
pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds);
tTrace("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d",
pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds);
} else {
close(fd);
tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
@ -340,7 +361,10 @@ void taosCloseTcpConnection(void *chandle) {
SFdObj *pFdObj = chandle;
if (pFdObj == NULL) return;
pFdObj->thandle = NULL;
SThreadObj *pThreadObj = pFdObj->pThreadObj;
tTrace("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj);
// pFdObj->thandle = NULL;
pFdObj->closedByApp = 1;
shutdown(pFdObj->fd, SHUT_WR);
}
@ -385,14 +409,14 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
if (headLen != sizeof(SRpcHead)) {
tTrace("%s %p, read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen);
tTrace("%s %p read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen);
return -1;
}
msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
buffer = malloc(msgLen + tsRpcOverhead);
if ( NULL == buffer) {
tError("%s %p, TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
return -1;
}
@ -401,8 +425,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
if (leftLen != retLen) {
tError("%s %p, read error, leftLen:%d retLen:%d",
pThreadObj->label, pFdObj->thandle, leftLen, retLen);
tError("%s %p read error, leftLen:%d retLen:%d FD:%p",
pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj);
free(buffer);
return -1;
}
@ -446,19 +470,19 @@ static void *taosProcessTcpData(void *param) {
pFdObj = events[i].data.ptr;
if (events[i].events & EPOLLERR) {
tTrace("%s %p, error happened on FD", pThreadObj->label, pFdObj->thandle);
tTrace("%s %p FD:%p epoll errors", pThreadObj->label, pFdObj->thandle, pFdObj);
taosReportBrokenLink(pFdObj);
continue;
}
if (events[i].events & EPOLLRDHUP) {
tTrace("%s %p, FD RD hang up", pThreadObj->label, pFdObj->thandle);
tTrace("%s %p FD:%p RD hang up", pThreadObj->label, pFdObj->thandle, pFdObj);
taosReportBrokenLink(pFdObj);
continue;
}
if (events[i].events & EPOLLHUP) {
tTrace("%s %p, FD hang up", pThreadObj->label, pFdObj->thandle);
tTrace("%s %p FD:%p hang up", pThreadObj->label, pFdObj->thandle, pFdObj);
taosReportBrokenLink(pFdObj);
continue;
}
@ -527,7 +551,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pThreadObj->numOfFds--;
if (pThreadObj->numOfFds < 0)
tError("%s %p, TCP thread:%d, number of FDs is negative!!!",
tError("%s %p TCP thread:%d, number of FDs is negative!!!",
pThreadObj->label, pFdObj->thandle, pThreadObj->threadId);
if (pFdObj->prev) {
@ -542,7 +566,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pthread_mutex_unlock(&pThreadObj->mutex);
tTrace("%s %p, FD:%p is cleaned, numOfFds:%d",
tTrace("%s %p TCP connection is closed, FD:%p numOfFds:%d",
pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
tfree(pFdObj);

29
src/rpc/src/rpcUdp.c
View File

@ -30,7 +30,6 @@
#define RPC_MAX_UDP_SIZE 65480
typedef struct {
void *signature;
int index;
int fd;
uint16_t port; // peer port
@ -111,7 +110,6 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
pConn->processData = fp;
pConn->index = i;
pConn->pSet = pSet;
pConn->signature = pConn;
int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn);
if (code != 0) {
@ -132,7 +130,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
return pSet;
}
void taosCleanUpUdpConnection(void *handle) {
void taosStopUdpConnection(void *handle) {
SUdpConnSet *pSet = (SUdpConnSet *)handle;
SUdpConn *pConn;
@ -140,8 +138,6 @@ void taosCleanUpUdpConnection(void *handle) {
for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i;
pConn->signature = NULL;
if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR);
if (pConn->fd >=0) taosCloseSocket(pConn->fd);
}
@ -150,9 +146,24 @@ void taosCleanUpUdpConnection(void *handle) {
pConn = pSet->udpConn + i;
if (pConn->thread) pthread_join(pConn->thread, NULL);
tfree(pConn->buffer);
tTrace("%s UDP thread is closed, inedx:%d", pConn->label, i);
// tTrace("%s UDP thread is closed, index:%d", pConn->label, i);
}
tTrace("%s UDP is stopped", pSet->label);
}
void taosCleanUpUdpConnection(void *handle) {
SUdpConnSet *pSet = (SUdpConnSet *)handle;
SUdpConn *pConn;
if (pSet == NULL) return;
for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i;
if (pConn->fd >=0) taosCloseSocket(pConn->fd);
}
tTrace("%s UDP is cleaned up", pSet->label);
tfree(pSet);
}
@ -164,7 +175,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
SUdpConn *pConn = pSet->udpConn + pSet->index;
pConn->port = port;
tTrace("%s UDP connection is setup, ip:%x:%hu", pConn->label, ip, port);
tTrace("%s UDP connection is setup, ip:%x:%hu localPort:%hu", pConn->label, ip, port, pConn->localPort);
return pConn;
}
@ -185,7 +196,7 @@ static void *taosRecvUdpData(void *param) {
while (1) {
dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen);
if(dataLen <= 0) {
tTrace("%s UDP socket was closed, exiting", pConn->label);
tTrace("%s UDP socket was closed, exiting(%s)", pConn->label, strerror(errno));
break;
}
@ -221,7 +232,7 @@ static void *taosRecvUdpData(void *param) {
int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) {
SUdpConn *pConn = (SUdpConn *)chandle;
if (pConn == NULL || pConn->signature != pConn) return -1;
if (pConn == NULL) return -1;
struct sockaddr_in destAdd;
memset(&destAdd, 0, sizeof(destAdd));

2
src/tsdb/inc/tsdbMain.h
View File

@ -25,6 +25,7 @@
#include "tsdb.h"
#include "tskiplist.h"
#include "tutil.h"
#include "trwlatch.h"
#ifdef __cplusplus
extern "C" {
@ -59,6 +60,7 @@ typedef struct STable {
TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure
char* sql;
void* cqhandle;
SRWLatch latch; // TODO: implementa latch functions
T_REF_DECLARE();
} STable;

3
src/tsdb/src/tsdbBuffer.c
View File

@ -120,10 +120,13 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) {
STsdbBufPool *pBufPool = pRepo->pPool;
while (POOL_IS_EMPTY(pBufPool)) {
pRepo->repoLocked = false;
pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex));
pRepo->repoLocked = true;
}
SListNode * pNode = tdListPopHead(pBufPool->bufBlockList);
ASSERT(pNode != NULL);
STsdbBufBlock *pBufBlock = NULL;
tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock));

4
src/tsdb/src/tsdbFile.c
View File

@ -358,7 +358,9 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) {
ASSERT(pFileH->nFGroups >= 0);
for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
remove(fileGroup.files[type].fname);
if (remove(fileGroup.files[type].fname) < 0) {
tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname);
}
tsdbDestroyFile(&fileGroup.files[type]);
}
}

15
src/tsdb/src/tsdbMain.c
View File

@ -179,7 +179,7 @@ int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *
return -1;
}
}
pRsp->affectedRows = htonl(affectedrows);
if (pRsp != NULL) pRsp->affectedRows = htonl(affectedrows);
return 0;
}
@ -213,10 +213,10 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
SFileGroup *pFGroup =
taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE);
if (pFGroup->fileId == fid) {
strcpy(fname, pFGroup->files[(*index) % 3].fname);
fname = strdup(pFGroup->files[(*index) % 3].fname);
} else {
if (pFGroup->fileId * 3 + 2 < eindex) {
strcpy(fname, pFGroup->files[0].fname);
fname = strdup(pFGroup->files[0].fname);
*index = pFGroup->fileId * 3;
} else {
tfree(sdup);
@ -237,12 +237,13 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
}
SFile *pFile = &pFGroup->files[(*index) % 3];
strcpy(fname, pFile->fname);
fname = strdup(pFile->fname);
}
}
if (stat(fname, &fState) < 0) {
tfree(sdup);
tfree(fname);
return 0;
}
@ -566,7 +567,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
_err:
tfree(fname);
if (fd > 0) close(fd);
if (fd >= 0) close(fd);
return -1;
}
@ -609,7 +610,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
_err:
tfree(fname);
if (fd > 0) close(fd);
if (fd >= 0) close(fd);
return -1;
}
@ -647,7 +648,7 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) {
}
pRepo->config = *pCfg;
pRepo->appH = *pAppH;
if (pAppH) pRepo->appH = *pAppH;
pRepo->tsdbMeta = tsdbNewMeta(pCfg);
if (pRepo->tsdbMeta == NULL) {

12
src/tsdb/src/tsdbMemTable.c
View File

@ -99,6 +99,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
if (tSkipListPut(pTableData->pData, pNode) == NULL) {
tsdbFreeBytes(pRepo, (void *)pNode, bytes);
} else {
if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
if (pMemTable->keyLast < key) pMemTable->keyLast = key;
pMemTable->numOfRows++;
@ -222,11 +223,12 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
pRepo->commit = 0;
}
ASSERT(pRepo->commit == 0);
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
if (pRepo->mem != NULL) {
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
if (tsdbLockRepo(pRepo) < 0) return -1;
pRepo->imem = pRepo->mem;
pRepo->mem = NULL;
@ -468,9 +470,6 @@ _err:
static void tsdbEndCommit(STsdbRepo *pRepo) {
ASSERT(pRepo->commit == 1);
tsdbLockRepo(pRepo);
pRepo->commit = 0;
tsdbUnlockRepo(pRepo);
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER);
}
@ -526,8 +525,6 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
goto _err;
}
free(dataDir);
// Open files for write/read
if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno));
@ -590,6 +587,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
goto _err;
}
tfree(dataDir);
tsdbCloseHelperFile(pHelper, 0);
pthread_rwlock_wrlock(&(pFileH->fhlock));
@ -601,7 +599,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
return 0;
_err:
// ASSERT(false);
tfree(dataDir);
tsdbCloseHelperFile(pHelper, 1);
return -1;
}

27
src/tsdb/src/tsdbMeta.c
View File

@ -147,6 +147,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) {
tsdbInsertTableAct(pRepo, TSDB_DROP_META, buf, tTable);
tsdbRemoveTableFromMeta(pRepo, tTable, false, true);
}
tSkipListDestroyIter(pIter);
}
tsdbRemoveTableFromMeta(pRepo, pTable, true, true);
@ -157,18 +158,16 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) {
return 0;
}
void *tsdbGetTableTagVal(TSDB_REPO_T *repo, const STableId *id, int32_t colId, int16_t type, int16_t bytes) {
void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes) {
// TODO: this function should be changed also
STsdbMeta *pMeta = tsdbGetMeta(repo);
STable * pTable = tsdbGetTableByUid(pMeta, id->uid);
STSchema *pSchema = tsdbGetTableTagSchema(pTable);
STSchema *pSchema = tsdbGetTableTagSchema((STable*) pTable);
STColumn *pCol = tdGetColOfID(pSchema, colId);
if (pCol == NULL) {
return NULL; // No matched tag volumn
}
char *val = tdGetKVRowValOfCol(pTable->tagVal, colId);
char *val = tdGetKVRowValOfCol(((STable*)pTable)->tagVal, colId);
assert(type == pCol->type && bytes == pCol->bytes);
if (val != NULL && IS_VAR_DATA_TYPE(type)) {
@ -178,20 +177,21 @@ void *tsdbGetTableTagVal(TSDB_REPO_T *repo, const STableId *id, int32_t colId, i
return val;
}
char *tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id) {
char *tsdbGetTableName(void* pTable) {
// TODO: need to change as thread-safe
STsdbRepo *pRepo = (STsdbRepo *)repo;
STsdbMeta *pMeta = pRepo->tsdbMeta;
STable *pTable = tsdbGetTableByUid(pMeta, id->uid);
if (pTable == NULL) {
return NULL;
} else {
return (char *)pTable->name;
return (char*) (((STable *)pTable)->name);
}
}
STableId tsdbGetTableId(void *pTable) {
assert(pTable);
return ((STable*)pTable)->tableId;
}
STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) {
if (pMsg == NULL) return NULL;
@ -270,7 +270,6 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) {
_err:
tdDestroyTSchemaBuilder(&schemaBuilder);
tsdbClearTableCfg(pCfg);
tfree(pCfg);
return NULL;
}
@ -309,6 +308,7 @@ int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) {
int32_t code = tsdbUpdateTable(pRepo, super, pTableCfg);
if (code != TSDB_CODE_SUCCESS) {
tsdbClearTableCfg(pTableCfg);
return code;
}
tsdbClearTableCfg(pTableCfg);
@ -788,6 +788,9 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) {
}
if (addIdx && tsdbUnlockRepoMeta(pRepo) < 0) return -1;
if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE && addIdx) {
pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql, tsdbGetTableSchema(pTable));
}
tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
TABLE_TID(pTable), TABLE_UID(pTable));

90
src/tsdb/src/tsdbRWHelper.c
View File

@ -173,9 +173,14 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
close(pHelper->files.nHeadF.fd);
pHelper->files.nHeadF.fd = -1;
if (hasError) {
remove(pHelper->files.nHeadF.fname);
(void)remove(pHelper->files.nHeadF.fname);
} else {
rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname);
if (rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname) < 0) {
tsdbError("failed to rename file from %s to %s since %s", pHelper->files.nHeadF.fname,
pHelper->files.headF.fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
pHelper->files.headF.info = pHelper->files.nHeadF.info;
}
}
@ -186,9 +191,14 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
close(pHelper->files.nLastF.fd);
pHelper->files.nLastF.fd = -1;
if (hasError) {
remove(pHelper->files.nLastF.fname);
(void)remove(pHelper->files.nLastF.fname);
} else {
rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname);
if (rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname) < 0) {
tsdbError("failed to rename file from %s to %s since %s", pHelper->files.nLastF.fname,
pHelper->files.lastF.fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
pHelper->files.lastF.info = pHelper->files.nLastF.info;
}
}
@ -306,8 +316,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
if (pCompBlock->numOfSubBlocks > 1) {
if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1), NULL) < 0) return -1;
ASSERT(pHelper->pDataCols[0]->numOfRows > 0 &&
pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock);
ASSERT(pHelper->pDataCols[0]->numOfRows > 0 && pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock);
if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0],
pHelper->pDataCols[0]->numOfRows, &compBlock, true, true) < 0)
return -1;
@ -330,14 +339,27 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
}
int tsdbWriteCompInfo(SRWHelper *pHelper) {
off_t offset = 0;
SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) {
if (pIdx->offset > 0) {
pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
if (pIdx->offset < 0) return -1;
offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
if (offset < 0) {
tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
pIdx->offset = offset;
ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) return -1;
if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) {
tsdbError("vgId:%d failed to send %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
pHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
} else {
pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER;
@ -345,12 +367,23 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
pHelper->pCompInfo->checksum = 0;
ASSERT((pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0);
taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len);
pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
if (offset < 0) {
tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
pIdx->offset = offset;
pIdx->uid = pHelper->tableInfo.uid;
if (pIdx->offset < 0) return -1;
ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1;
if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
pHelper->files.nHeadF.fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
return 0;
@ -567,24 +600,24 @@ _err:
static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) {
ASSERT(pHelper->files.lastF.fd > 0);
struct stat st;
fstat(pHelper->files.lastF.fd, &st);
if (fstat(pHelper->files.lastF.fd, &st) < 0) return true;
if (st.st_size > 32 * 1024 + TSDB_FILE_HEAD_SIZE) return true;
return false;
}
static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite,
SCompBlock *pCompBlock, bool isLast, bool isSuperBlock) {
STsdbCfg *pCfg = &(pHelper->pRepo->config);
STsdbCfg * pCfg = &(pHelper->pRepo->config);
SCompData *pCompData = (SCompData *)(pHelper->pBuffer);
int64_t offset = 0;
ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfRows && rowsToWrite <= pCfg->maxRowsPerFileBlock);
ASSERT(isLast ? rowsToWrite < pCfg->minRowsPerFileBlock : true);
offset = lseek(pFile->fd, 0, SEEK_END);
if (offset < 0) {
tsdbError("vgId:%d failed to write block to file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno));
tsdbError("vgId:%d failed to write block to file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
@ -639,9 +672,9 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
}
}
pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))(
(char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->pBuffer) - lsize, pCfg->compression,
pHelper->compBuffer, tsizeof(pHelper->compBuffer));
pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr,
tsizeof(pHelper->pBuffer) - lsize, pCfg->compression,
pHelper->compBuffer, tsizeof(pHelper->compBuffer));
} else {
pCompCol->len = tlen;
memcpy(tptr, pDataCol->pData, pCompCol->len);
@ -725,8 +758,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
// ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0);
if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append
ASSERT(blockAtIdx(pHelper, blkIdx)->numOfRows < pCfg->minRowsPerFileBlock &&
blkIdx == pIdx->numOfBlocks - 1);
ASSERT(blockAtIdx(pHelper, blkIdx)->numOfRows < pCfg->minRowsPerFileBlock && blkIdx == pIdx->numOfBlocks - 1);
int defaultRowsToWrite = pCfg->maxRowsPerFileBlock * 4 / 5; // TODO: make a interface
rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfRows), pDataCols->numOfRows);
@ -1051,7 +1083,7 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
static int tsdbInitHelperFile(SRWHelper *pHelper) {
STsdbCfg *pCfg = &pHelper->pRepo->config;
size_t tsize = sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM);
size_t tsize = sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM);
pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize);
if (pHelper->pCompIdx == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@ -1099,10 +1131,8 @@ static int tsdbInitHelperBlock(SRWHelper *pHelper) {
STsdbRepo *pRepo = helperRepo(pHelper);
STsdbMeta *pMeta = pHelper->pRepo->tsdbMeta;
pHelper->pDataCols[0] =
tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
pHelper->pDataCols[1] =
tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
pHelper->pDataCols[0] = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
pHelper->pDataCols[1] = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
@ -1222,12 +1252,16 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32
static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) {
ASSERT(pCompBlock->numOfSubBlocks <= 1);
ASSERT(tsizeof(pHelper->pBuffer) >= pCompBlock->len);
SCompData *pCompData = (SCompData *)pHelper->pBuffer;
SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF);
pHelper->pBuffer = trealloc(pHelper->pBuffer, pCompBlock->len);
if (pHelper->pBuffer == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
}
int fd = pFile->fd;
if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) {
tsdbError("vgId:%d tid:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,

227
src/tsdb/src/tsdbRead.c
View File

@ -184,15 +184,17 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
assert(gsize > 0);
for (int32_t j = 0; j < gsize; ++j) {
STableId* id = (STableId*) taosArrayGet(group, j);
STable* pTable = (STable*) taosArrayGetP(group, j);
STableCheckInfo info = {
.lastKey = pQueryHandle->window.skey,
.tableId = *id,
.pTableObj = tsdbGetTableByUid(pMeta, id->uid),
.tableId = pTable->tableId,
.pTableObj = pTable,
};
assert(info.pTableObj != NULL && info.pTableObj->tableId.tid == id->tid);
assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE ||
info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE));
taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
}
}
@ -215,17 +217,17 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
return pQueryHandle;
}
SArray* tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle) {
SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
assert(pHandle != NULL);
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) pHandle;
size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
SArray* res = taosArrayInit(size, sizeof(STableId));
SArray* res = taosArrayInit(size, POINTER_BYTES);
for(int32_t i = 0; i < size; ++i) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
taosArrayPush(res, &pCheckInfo->tableId);
taosArrayPush(res, &pCheckInfo->pTableObj);
}
return res;
@ -1052,7 +1054,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order);
if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
tSkipListIterNext(pCheckInfo->iter);
moveToNextRow(pCheckInfo);
}
int32_t start = -1;
@ -1607,6 +1609,7 @@ void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
}
if (index == -1) {
// todo add failure test cases
return;
}
@ -1856,42 +1859,19 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
SArray* tsdbRetrieveDataRow(TsdbQueryHandleT* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; }
TsdbQueryHandleT* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, const char* pTagFilterStr) {
return NULL;
}
SArray* tsdbGetTableList(TsdbQueryHandleT* pQueryHandle) { return NULL; }
static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) {
static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex);
while (tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter);
STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode);
taosArrayPush(list, &(*pTable)->tableId);
taosArrayPush(list, pTable);
}
tSkipListDestroyIter(iter);
return TSDB_CODE_SUCCESS;
}
/**
* convert the result pointer to table id instead of table object pointer
* todo remove it by using callback function to change the final result in-time.
* @param pRes
*/
static void convertQueryResult(SArray* pRes, SArray* pTableList) {
if (pTableList == NULL || taosArrayGetSize(pTableList) == 0) {
return;
}
size_t size = taosArrayGetSize(pTableList);
for (int32_t i = 0; i < size; ++i) { // todo speedup by using reserve space.
STable* pTable = taosArrayGetP(pTableList, i);
taosArrayPush(pRes, &pTable->tableId);
}
}
static void destroyHelper(void* param) {
if (param == NULL) {
return;
@ -1960,16 +1940,13 @@ typedef struct STableGroupSupporter {
int32_t numOfCols;
SColIndex* pCols;
STSchema* pTagSchema;
void* tsdbMeta;
// void* tsdbMeta;
} STableGroupSupporter;
int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param;
STableId* id1 = (STableId*) p1;
STableId* id2 = (STableId*) p2;
STable *pTable1 = tsdbGetTableByUid(pTableGroupSupp->tsdbMeta, id1->uid);
STable *pTable2 = tsdbGetTableByUid(pTableGroupSupp->tsdbMeta, id2->uid);
STable* pTable1 = *(STable**) p1;
STable* pTable2 = *(STable**) p2;
for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) {
SColIndex* pColIndex = &pTableGroupSupp->pCols[i];
@ -2019,26 +1996,29 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
return 0;
}
void createTableGroupImpl(SArray* pGroups, SArray* pTableIdList, size_t numOfTables, STableGroupSupporter* pSupp,
void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, STableGroupSupporter* pSupp,
__ext_compar_fn_t compareFn) {
STableId* pId = taosArrayGet(pTableIdList, 0);
SArray* g = taosArrayInit(16, sizeof(STableId));
taosArrayPush(g, pId);
STable* pTable = taosArrayGetP(pTableList, 0);
SArray* g = taosArrayInit(16, POINTER_BYTES);
taosArrayPush(g, &pTable);
tsdbRefTable(pTable);
for (int32_t i = 1; i < numOfTables; ++i) {
STableId* prev = taosArrayGet(pTableIdList, i - 1);
STableId* p = taosArrayGet(pTableIdList, i);
STable** prev = taosArrayGet(pTableList, i - 1);
STable** p = taosArrayGet(pTableList, i);
int32_t ret = compareFn(prev, p, pSupp);
assert(ret == 0 || ret == -1);
tsdbRefTable(*p);
assert((*p)->type == TSDB_CHILD_TABLE);
if (ret == 0) {
taosArrayPush(g, p);
} else {
taosArrayPush(pGroups, &g); // current group is ended, start a new group
g = taosArrayInit(16, sizeof(STableId));
g = taosArrayInit(16, POINTER_BYTES);
taosArrayPush(g, p);
}
}
@ -2046,8 +2026,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableIdList, size_t numOfTab
taosArrayPush(pGroups, &g);
}
SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols,
TSDB_REPO_T* tsdb) {
SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols) {
assert(pTableList != NULL);
SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES);
@ -2058,22 +2037,24 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
}
if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table
SArray* sa = taosArrayInit(size, sizeof(STableId));
SArray* sa = taosArrayInit(size, POINTER_BYTES);
for(int32_t i = 0; i < size; ++i) {
STableId* tableId = taosArrayGet(pTableList, i);
taosArrayPush(sa, tableId);
STable** pTable = taosArrayGet(pTableList, i);
assert((*pTable)->type == TSDB_CHILD_TABLE);
tsdbRefTable(*pTable);
taosArrayPush(sa, pTable);
}
taosArrayPush(pTableGroup, &sa);
tsdbTrace("all %zu tables belong to one group", size);
} else {
STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter));
pSupp->tsdbMeta = tsdbGetMeta(tsdb);
pSupp->numOfCols = numOfOrderCols;
pSupp->pTagSchema = pTagSchema;
pSupp->pCols = pCols;
taosqsort(pTableList->pData, size, sizeof(STableId), pSupp, tableGroupComparFn);
taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn);
createTableGroupImpl(pTableGroup, pTableList, size, pSupp, tableGroupComparFn);
tfree(pSupp);
}
@ -2149,48 +2130,53 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
.pExtInfo = pSTable->tagSchema,
};
SArray* pTableList = taosArrayInit(8, POINTER_BYTES);
tExprTreeTraverse(pExpr, pSTable->pIndex, pTableList, &supp);
tExprTreeTraverse(pExpr, pSTable->pIndex, pRes, &supp);
tExprTreeDestroy(&pExpr, destroyHelper);
convertQueryResult(pRes, pTableList);
taosArrayDestroy(pTableList);
return TSDB_CODE_SUCCESS;
}
int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pTagCond, size_t len,
int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
SColIndex* pColIndex, int32_t numOfCols) {
if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
if (pTable == NULL) {
tsdbError("%p failed to get stable, uid:%" PRIu64, tsdb, uid);
return TSDB_CODE_TDB_INVALID_TABLE_ID;
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
tsdbUnlockRepoMeta(tsdb);
goto _error;
}
if (pTable->type != TSDB_SUPER_TABLE) {
tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid,
pTable->name->data);
return TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client
terrno = TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client
tsdbUnlockRepoMeta(tsdb);
goto _error;
}
SArray* res = taosArrayInit(8, sizeof(STableId));
//NOTE: not add ref count for super table
SArray* res = taosArrayInit(8, POINTER_BYTES);
STSchema* pTagSchema = tsdbGetTableTagSchema(pTable);
// no tags and tbname condition, all child tables of this stable are involved
if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) {
int32_t ret = getAllTableIdList(pTable, res);
if (ret == TSDB_CODE_SUCCESS) {
pGroupInfo->numOfTables = taosArrayGetSize(res);
pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, tsdb);
tsdbTrace("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables);
} else {
// todo add error
int32_t ret = getAllTableList(pTable, res);
if (ret != TSDB_CODE_SUCCESS) {
tsdbUnlockRepoMeta(tsdb);
goto _error;
}
pGroupInfo->numOfTables = taosArrayGetSize(res);
pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols);
tsdbTrace("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables);
taosArrayDestroy(res);
if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error;
return ret;
}
@ -2227,31 +2213,84 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
doQueryTableList(pTable, res, expr);
pGroupInfo->numOfTables = taosArrayGetSize(res);
pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, tsdb);
pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols);
tsdbTrace("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%zu, belong to %zu groups", tsdb, pTable->tableId.tid,
pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList));
taosArrayDestroy(res);
if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error;
return ret;
_error:
return terrno;
}
int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* pGroupInfo) {
if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
if (pTable == NULL) {
return TSDB_CODE_TDB_INVALID_TABLE_ID;
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
tsdbUnlockRepoMeta(tsdb);
goto _error;
}
//todo assert table type, add the table ref count
assert(pTable->type == TSDB_CHILD_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE);
tsdbRefTable(pTable);
if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error;
pGroupInfo->numOfTables = 1;
pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES);
SArray* group = taosArrayInit(1, sizeof(STableId));
SArray* group = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(group, &pTable->tableId);
taosArrayPush(group, &pTable);
taosArrayPush(pGroupInfo->pGroupList, &group);
return TSDB_CODE_SUCCESS;
_error:
return terrno;
}
int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) {
if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
assert(pTableIdList != NULL);
size_t size = taosArrayGetSize(pTableIdList);
pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES);
SArray* group = taosArrayInit(1, POINTER_BYTES);
int32_t i = 0;
for(; i < size; ++i) {
STableIdInfo *id = taosArrayGet(pTableIdList, i);
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), id->uid);
if (pTable == NULL) {
tsdbWarn("table uid:%"PRIu64", tid:%d has been drop already", id->uid, id->tid);
continue;
}
if (pTable->type == TSDB_SUPER_TABLE) {
tsdbError("direct query on super tale is not allowed, table uid:%"PRIu64", tid:%d", id->uid, id->tid);
terrno = TSDB_CODE_QRY_INVALID_MSG;
}
tsdbRefTable(pTable);
taosArrayPush(group, &pTable);
}
if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error;
pGroupInfo->numOfTables = i;
taosArrayPush(pGroupInfo->pGroupList, &group);
return TSDB_CODE_SUCCESS;
_error:
return terrno;
}
void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
@ -2263,12 +2302,11 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
for (int32_t i = 0; i < size; ++i) {
STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
tSkipListDestroyIter(pTableCheckInfo->iter);
tsdbUnRefMemTable(pQueryHandle->pTsdb, pTableCheckInfo->mem);
tsdbUnRefMemTable(pQueryHandle->pTsdb, pTableCheckInfo->imem);
tSkipListDestroyIter(pTableCheckInfo->iter);
if (pTableCheckInfo->pDataCols != NULL) {
tfree(pTableCheckInfo->pDataCols->buf);
}
@ -2293,3 +2331,26 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
tfree(pQueryHandle);
}
void tsdbDestoryTableGroup(STableGroupInfo *pGroupList) {
assert(pGroupList != NULL);
size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList);
for(int32_t i = 0; i < numOfGroup; ++i) {
SArray* p = taosArrayGetP(pGroupList->pGroupList, i);
size_t numOfTables = taosArrayGetSize(p);
for(int32_t j = 0; j < numOfTables; ++j) {
STable* pTable = taosArrayGetP(p, j);
assert(pTable != NULL);
tsdbUnRefTable(pTable);
}
taosArrayDestroy(p);
}
taosArrayDestroy(pGroupList->pGroupList);
}

2
src/tsdb/tests/CMakeLists.txt
View File

@ -1,6 +1,6 @@
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
add_executable(tsdbTests ${SOURCE_LIST})
target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb)
target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb tutil trpc)
add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests)

255
src/tsdb/tests/tsdbTests.cpp
View File

@ -2,9 +2,8 @@
#include <stdlib.h>
#include <sys/time.h>
#include "tdataformat.h"
#include "tsdb.h"
#include "tsdbMain.h"
#include "tskiplist.h"
static double getCurTime() {
struct timeval tv;
@ -77,7 +76,7 @@ static int insertData(SInsertInfo *pInfo) {
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
pMsg->compressed = htonl(pMsg->numOfBlocks);
if (tsdbInsertData(pInfo->pRepo, pMsg) < 0) {
if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) {
tfree(pMsg);
return -1;
}
@ -90,222 +89,72 @@ static int insertData(SInsertInfo *pInfo) {
return 0;
}
TEST(TsdbTest, DISABLED_tableEncodeDecode) {
// TEST(TsdbTest, tableEncodeDecode) {
STable *pTable = (STable *)malloc(sizeof(STable));
static void tsdbSetCfg(STsdbCfg *pCfg, int32_t tsdbId, int32_t cacheBlockSize, int32_t totalBlocks, int32_t maxTables,
int32_t daysPerFile, int32_t keep, int32_t minRows, int32_t maxRows, int8_t precision,
int8_t compression) {
pCfg->tsdbId = tsdbId;
pCfg->cacheBlockSize = cacheBlockSize;
pCfg->totalBlocks = totalBlocks;
pCfg->maxTables = maxTables;
pCfg->daysPerFile = daysPerFile;
pCfg->keep = keep;
pCfg->minRowsPerFileBlock = minRows;
pCfg->maxRowsPerFileBlock = maxRows;
pCfg->precision = precision;
pCfg->compression = compression;
}
pTable->type = TSDB_NORMAL_TABLE;
pTable->tableId.uid = 987607499877672L;
pTable->tableId.tid = 0;
pTable->superUid = -1;
pTable->sversion = 0;
pTable->tagSchema = NULL;
pTable->tagVal = NULL;
int nCols = 5;
STSchema *schema = tdNewSchema(nCols);
static void tsdbSetTableCfg(STableCfg *pCfg) {
STSchemaBuilder schemaBuilder = {0};
for (int i = 0; i < nCols; i++) {
if (i == 0) {
tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
} else {
tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
}
pCfg->type = TSDB_NORMAL_TABLE;
pCfg->superUid = TSDB_INVALID_SUPER_TABLE_ID;
pCfg->tableId.tid = 1;
pCfg->tableId.uid = 5849583783847394;
tdInitTSchemaBuilder(&schemaBuilder, 0);
int colId = 0;
for (int i = 0; i < 5; i++) {
tdAddColToSchema(&schemaBuilder, (colId == 0) ? TSDB_DATA_TYPE_TIMESTAMP : TSDB_DATA_TYPE_INT, colId, 0);
colId++;
}
pTable->schema = schema;
pCfg->schema = tdGetSchemaFromBuilder(&schemaBuilder);
pCfg->name = strdup("t1");
int bufLen = 0;
void *buf = tsdbEncodeTable(pTable, &bufLen);
STable *tTable = tsdbDecodeTable(buf, bufLen);
ASSERT_EQ(pTable->type, tTable->type);
ASSERT_EQ(pTable->tableId.uid, tTable->tableId.uid);
ASSERT_EQ(pTable->tableId.tid, tTable->tableId.tid);
ASSERT_EQ(pTable->superUid, tTable->superUid);
ASSERT_EQ(pTable->sversion, tTable->sversion);
ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0);
tdDestroyTSchemaBuilder(&schemaBuilder);
}
// TEST(TsdbTest, DISABLED_createRepo) {
TEST(TsdbTest, createRepo) {
STsdbCfg config;
STsdbRepo *repo;
TEST(TsdbTest, testInsertSpeed) {
int vnode = 1;
int ret = 0;
STsdbCfg tsdbCfg;
STableCfg tableCfg;
std::string testDir = "./test";
char * rootDir = strdup((testDir + "/vnode" + std::to_string(vnode)).c_str());
// 1. Create a tsdb repository
tsdbSetDefaultCfg(&config);
ASSERT_EQ(tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL), 0);
tsdbDebugFlag = 131; //NOTE: you must set the flag
TSDB_REPO_T *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
ASSERT_NE(pRepo, nullptr);
taosRemoveDir(rootDir);
// 2. Create a normal table
STableCfg tCfg;
ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0);
tsdbTableSetName(&tCfg, "test", false);
// Create and open repository
tsdbSetCfg(&tsdbCfg, 1, 16, 4, -1, -1, -1, -1, -1, -1, -1);
tsdbCreateRepo(rootDir, &tsdbCfg);
TSDB_REPO_T *repo = tsdbOpenRepo(rootDir, NULL);
ASSERT_NE(repo, nullptr);
int nCols = 5;
STSchema *schema = tdNewSchema(nCols);
// Create table
tsdbSetTableCfg(&tableCfg);
tsdbCreateTable(repo, &tableCfg);
for (int i = 0; i < nCols; i++) {
if (i == 0) {
tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
} else {
tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
}
}
// Insert data
SInsertInfo iInfo = {repo, true, 1, 5849583783847394, 0, 1590000000000, 10, 10000000, 100, tableCfg.schema};
tsdbTableSetSchema(&tCfg, schema, true);
insertData(&iInfo);
tsdbCreateTable(pRepo, &tCfg);
// Insert Some Data
SInsertInfo iInfo = {
.pRepo = pRepo,
// .isAscend = true,
.isAscend = false,
.tid = tCfg.tableId.tid,
.uid = tCfg.tableId.uid,
.sversion = tCfg.sversion,
.startTime = 1584081000000,
.interval = 1000,
.totalRows = 10000000,
.rowsPerSubmit = 1,
.pSchema = schema
};
ASSERT_EQ(insertData(&iInfo), 0);
// Close the repository
tsdbCloseRepo(pRepo);
// Open the repository again
pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
repo = (STsdbRepo *)pRepo;
ASSERT_NE(pRepo, nullptr);
// // Insert more data
// iInfo.startTime = iInfo.startTime + iInfo.interval * iInfo.totalRows;
// iInfo.totalRows = 10;
// iInfo.pRepo = pRepo;
// ASSERT_EQ(insertData(&iInfo), 0);
// // Close the repository
// tsdbCloseRepo(pRepo);
// // Open the repository again
// pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
// repo = (STsdbRepo *)pRepo;
// ASSERT_NE(pRepo, nullptr);
// // Read from file
// SRWHelper rhelper;
// tsdbInitReadHelper(&rhelper, repo);
// SFileGroup *pFGroup = tsdbSearchFGroup(repo->tsdbFileH, 1833);
// ASSERT_NE(pFGroup, nullptr);
// ASSERT_GE(tsdbSetAndOpenHelperFile(&rhelper, pFGroup), 0);
// STable *pTable = tsdbGetTableByUid(repo->tsdbMeta, tCfg.tableId.uid);
// ASSERT_NE(pTable, nullptr);
// tsdbSetHelperTable(&rhelper, pTable, repo);
// ASSERT_EQ(tsdbLoadCompInfo(&rhelper, NULL), 0);
// ASSERT_EQ(tsdbLoadBlockData(&rhelper, blockAtIdx(&rhelper, 0), NULL), 0);
int k = 0;
}
TEST(TsdbTest, DISABLED_openRepo) {
// TEST(TsdbTest, openRepo) {
// tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL);
// ASSERT_NE(repo, nullptr);
// STsdbRepo *pRepo = (STsdbRepo *)repo;
// SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1655);
// for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
// tsdbOpenFile(&pGroup->files[type], O_RDONLY);
// }
// SCompIdx *pIdx = (SCompIdx *)calloc(pRepo->config.maxTables, sizeof(SCompIdx));
// tsdbLoadCompIdx(pGroup, (void *)pIdx, pRepo->config.maxTables);
// SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len);
// tsdbLoadCompBlocks(pGroup, &pIdx[1], (void *)pCompInfo);
// int blockIdx = 0;
// SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]);
// SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols);
// tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData);
// STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid);
// SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5);
// tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable));
// tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData);
// tdResetDataCols(pDataCols);
// tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock + 1, 1, pDataCols, pCompData);
// int k = 0;
}
TEST(TsdbTest, DISABLED_createFileGroup) {
SFileGroup fGroup;
// ASSERT_EQ(tsdbCreateFileGroup("/home/ubuntu/work/ttest/vnode0/data", 1820, &fGroup, 1000), 0);
int k = 0;
tsdbCloseRepo(repo, 1);
}
static char *getTKey(const void *data) {
return (char *)data;
}
static void insertSkipList(bool isAscend) {
TSKEY start_time = 1587393453000;
TSKEY interval = 1000;
SSkipList *pList = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(TSKEY), 0, 0, 1, getTKey);
ASSERT_NE(pList, nullptr);
for (size_t i = 0; i < 20000000; i++)
{
TSKEY time = isAscend ? (start_time + i * interval) : (start_time - i * interval);
int32_t level = 0;
int32_t headSize = 0;
tSkipListNewNodeInfo(pList, &level, &headSize);
SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(TSKEY));
ASSERT_NE(pNode, nullptr);
pNode->level = level;
*(TSKEY *)((char *)pNode + headSize) = time;
tSkipListPut(pList, pNode);
}
tSkipListDestroy(pList);
}
TEST(TsdbTest, DISABLED_testSkipList) {
// TEST(TsdbTest, testSkipList) {
double stime = getCurTime();
insertSkipList(true);
double etime = getCurTime();
printf("Time used to insert 100000000 records takes %f seconds\n", etime-stime);
stime = getCurTime();
insertSkipList(false);
etime = getCurTime();
printf("Time used to insert 100000000 records takes %f seconds\n", etime-stime);
}

36
src/util/inc/trwlatch.h Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __TD_RWLATCH_H__
#define __TD_RWLATCH_H__
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
typedef int32_t SRWLatch;
void taosInitRWLatch(SRWLatch *pLatch);
void taosWLockLatch(SRWLatch *pLatch);
void taosWUnLockLatch(SRWLatch *pLatch);
void taosRLockLatch(SRWLatch *pLatch);
void taosRUnLockLatch(SRWLatch *pLatch);
#ifdef __cplusplus
}
#endif
#endif

9
src/util/inc/tstoken.h
View File

@ -24,14 +24,7 @@ extern "C" {
#include "tutil.h"
#include "ttokendef.h"
#define TK_SPACE 200
#define TK_COMMENT 201
#define TK_ILLEGAL 202
#define TK_HEX 203 // hex number 0x123
#define TK_OCT 204 // oct number
#define TK_BIN 205 // bin format data 0b111
#define TK_FILE 206
#define TK_QUESTION 207 // denoting the placeholder of "?",when invoking statement bind query
#define TSQL_TBNAME "TBNAME"
#define TSQL_TBNAME_L "tbname"

2
src/util/inc/ttimer.h
View File

@ -24,7 +24,7 @@ typedef void *tmr_h;
typedef void (*TAOS_TMR_CALLBACK)(void *, void *);
extern int taosTmrThreads;
extern uint32_t taosMaxTmrCtrl;
extern uint32_t tsMaxTmrCtrl;
#define MSECONDS_PER_TICK 5

5
src/util/inc/tutil.h
View File

@ -119,6 +119,8 @@ extern "C" {
uint32_t taosRand(void);
uint32_t trand(void);
size_t twcslen(const wchar_t *wcs);
int32_t strdequote(char *src);
@ -133,7 +135,7 @@ char* strtolower(char *dst, const char *src);
int64_t strnatoi(char *num, int32_t len);
char* strreplace(const char* str, const char* pattern, const char* rep);
//char* strreplace(const char* str, const char* pattern, const char* rep);
char *strbetween(char *string, char *begin, char *end);
@ -181,6 +183,7 @@ char *taosIpStr(uint32_t ipInt);
uint32_t ip2uint(const char *const ip_addr);
void taosRemoveDir(char *rootDir);
int tmkdir(const char *pathname, mode_t mode);
#define TAOS_ALLOC_MODE_DEFAULT 0
#define TAOS_ALLOC_MODE_RANDOM_FAIL 1

3
src/util/src/hash.c
View File

@ -468,7 +468,8 @@ void taosHashTableResize(SHashObj *pHashObj) {
return;
}
void *pNewEntry = realloc(pHashObj->hashList, POINTER_BYTES * newSize);
int32_t pointerSize = POINTER_BYTES;
void *pNewEntry = realloc(pHashObj->hashList, pointerSize * newSize);
if (pNewEntry == NULL) {// todo handle error
// uTrace("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity);
return;

17
src/util/src/tconfig.c
View File

@ -111,21 +111,18 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
wordfree(&full_path);
return;
}
if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
strcpy(option, full_path.we_wordv[0]);
}
wordfree(&full_path);
struct stat dirstat;
if (stat(option, &dirstat) < 0) {
int code = mkdir(option, 0755);
if (code < 0) {
uError("config option:%s, input value:%s, directory not exist, create fail with return code:%d",
cfg->option, input_value, code);
} else {
uPrint("config option:%s, input value:%s, directory not exist, create with return code:%d",
cfg->option, input_value, code);
}
int code = tmkdir(option, 0755);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("config option:%s, input value:%s, directory not exist, create fail:%s",
cfg->option, input_value, strerror(errno));
}
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {

20
src/util/src/tkvstore.c
View File

@ -78,8 +78,8 @@ int tdCreateKVStore(char *fname) {
return 0;
_err:
if (fd > 0) close(fd);
remove(fname);
if (fd >= 0) close(fd);
(void)remove(fname);
return -1;
}
@ -106,15 +106,15 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH
goto _err;
}
if (access(pStore->fsnap, F_OK) == 0) { // .snap file exists
uTrace("file %s exists, try to recover the KV store", pStore->fsnap);
pStore->sfd = open(pStore->fsnap, O_RDONLY);
if (pStore->sfd < 0) {
pStore->sfd = open(pStore->fsnap, O_RDONLY);
if (pStore->sfd < 0) {
if (errno != ENOENT) {
uError("failed to open file %s since %s", pStore->fsnap, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
} else {
uTrace("file %s exists, try to recover the KV store", pStore->fsnap);
if (tdLoadKVStoreHeader(pStore->sfd, pStore->fsnap, &info) < 0) {
if (terrno != TSDB_CODE_COM_FILE_CORRUPTED) goto _err;
} else {
@ -133,7 +133,7 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH
close(pStore->sfd);
pStore->sfd = -1;
remove(pStore->fsnap);
(void)remove(pStore->fsnap);
}
if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err;
@ -212,7 +212,7 @@ _err:
if (pStore->sfd > 0) {
close(pStore->sfd);
pStore->sfd = -1;
remove(pStore->fsnap);
(void)remove(pStore->fsnap);
}
if (pStore->fd > 0) {
close(pStore->fd);
@ -314,7 +314,7 @@ int tdKVStoreEndCommit(SKVStore *pStore) {
}
pStore->fd = -1;
remove(pStore->fsnap);
(void)remove(pStore->fsnap);
return 0;
}

11
src/util/src/tlog.c
View File

@ -147,7 +147,7 @@ static void *taosThreadToOpenNewFile(void *param) {
return NULL;
}
taosLockFile(fd);
lseek(fd, 0, SEEK_SET);
(void)lseek(fd, 0, SEEK_SET);
int32_t oldFd = tsLogObj.logHandle->fd;
tsLogObj.logHandle->fd = fd;
@ -276,14 +276,15 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
}
}
sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag);
char fileName[LOG_FILE_NAME_LEN + 50] = "\0";
sprintf(fileName, "%s.%d", tsLogObj.logName, tsLogObj.flag);
pthread_mutex_init(&tsLogObj.logMutex, NULL);
umask(0);
tsLogObj.logHandle->fd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
tsLogObj.logHandle->fd = open(fileName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (tsLogObj.logHandle->fd < 0) {
printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno));
printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno));
return -1;
}
taosLockFile(tsLogObj.logHandle->fd);
@ -291,7 +292,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
// only an estimate for number of lines
struct stat filestat;
if (fstat(tsLogObj.logHandle->fd, &filestat) < 0) {
printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno));
printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno));
return -1;
}
size = (int32_t)filestat.st_size;

13
src/util/src/tnote.c
View File

@ -92,7 +92,7 @@ void *taosThreadToOpenNewNote(void *param)
}
taosLockNote(fd, pNote);
lseek(fd, 0, SEEK_SET);
(void)lseek(fd, 0, SEEK_SET);
int oldFd = pNote->taosNoteFd;
pNote->taosNoteFd = fd;
@ -128,10 +128,12 @@ int taosOpenNewNote(taosNoteInfo * pNote)
bool taosCheckNoteIsOpen(char *noteName, taosNoteInfo * pNote)
{
/*
int exist = access(noteName, F_OK);
if (exist != 0) {
return false;
}
*/
int fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (fd < 0) {
@ -206,14 +208,15 @@ int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInf
}
}
sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
char noteName[NOTE_FILE_NAME_LEN * 2] = "\0";
sprintf(noteName, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
pthread_mutex_init(&pNote->taosNoteMutex, NULL);
umask(0);
pNote->taosNoteFd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
pNote->taosNoteFd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (pNote->taosNoteFd < 0) {
fprintf(stderr, "failed to open note file:%s reason:%s\n", name, strerror(errno));
fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno));
return -1;
}
taosLockNote(pNote->taosNoteFd, pNote);
@ -221,7 +224,7 @@ int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInf
// only an estimate for number of lines
struct stat filestat;
if (fstat(pNote->taosNoteFd, &filestat) < 0) {
fprintf(stderr, "failed to fstat note file:%s reason:%s\n", name, strerror(errno));
fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno));
return -1;
}
size = (int)filestat.st_size;

80
src/util/src/trwlatch.c Normal file
View File

@ -0,0 +1,80 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// #define _GNU_SOURCE
// #include <pthread.h>
#include "trwlatch.h"
#include "os.h"
#define TD_RWLATCH_WRITE_FLAG 0x40000000
void taosInitRWLatch(SRWLatch *pLatch) { *pLatch = 0; }
void taosWLockLatch(SRWLatch *pLatch) {
SRWLatch oLatch, nLatch;
int nLoops = 0;
// Set write flag
while (1) {
oLatch = atomic_load_32(pLatch);
if (oLatch & TD_RWLATCH_WRITE_FLAG) {
nLoops++;
if (nLoops > 1000) {
sched_yield();
nLoops = 0;
}
continue;
}
nLatch = oLatch | TD_RWLATCH_WRITE_FLAG;
if (atomic_val_compare_exchange_32(pLatch, oLatch, nLatch) == oLatch) break;
}
// wait for all reads end
nLoops = 0;
while (1) {
oLatch = atomic_load_32(pLatch);
if (oLatch == TD_RWLATCH_WRITE_FLAG) break;
nLoops++;
if (nLoops > 1000) {
sched_yield();
nLoops = 0;
}
}
}
void taosWUnLockLatch(SRWLatch *pLatch) { atomic_store_32(pLatch, 0); }
void taosRLockLatch(SRWLatch *pLatch) {
SRWLatch oLatch, nLatch;
int nLoops = 0;
while (1) {
oLatch = atomic_load_32(pLatch);
if (oLatch & TD_RWLATCH_WRITE_FLAG) {
nLoops++;
if (nLoops > 1000) {
sched_yield();
nLoops = 0;
}
continue;
}
nLatch = oLatch + 1;
if (atomic_val_compare_exchange_32(pLatch, oLatch, nLatch) == oLatch) break;
}
}
void taosRUnLockLatch(SRWLatch *pLatch) { atomic_fetch_sub_32(pLatch, 1); }

11
src/util/src/tskiplist.c
View File

@ -194,13 +194,12 @@ void *tSkipListDestroy(SSkipList *pSkipList) {
pthread_rwlock_wrlock(pSkipList->lock);
}
SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0);
if (pSkipList->keyInfo.freeNode) {
SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0);
while (pNode != pSkipList->pTail) {
SSkipListNode *pTemp = pNode;
pNode = SL_GET_FORWARD_POINTER(pNode, 0);
if (pSkipList->keyInfo.freeNode) {
while (pNode != pSkipList->pTail) {
SSkipListNode *pTemp = pNode;
pNode = SL_GET_FORWARD_POINTER(pNode, 0);
tfree(pTemp);
}
}

29
src/util/src/tsocket.c
View File

@ -222,9 +222,7 @@ int taosReadn(int fd, char *ptr, int nbytes) {
int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
struct sockaddr_in localAddr;
int sockFd;
int ttl = 128;
int reuse, nocheck;
int bufSize = 8192000;
int bufSize = 1024000;
uTrace("open udp socket:0x%x:%hu", ip, port);
@ -238,31 +236,6 @@ int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
return -1;
}
reuse = 1;
if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
uError("setsockopt SO_REUSEADDR failed): %d (%s)", errno, strerror(errno));
close(sockFd);
return -1;
};
nocheck = 1;
if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_NO_CHECK, (void *)&nocheck, sizeof(nocheck)) < 0) {
if (!taosSkipSocketCheck()) {
uError("setsockopt SO_NO_CHECK failed: %d (%s)", errno, strerror(errno));
close(sockFd);
return -1;
} else {
uPrint("Skipping setsockopt SO_NO_CHECK error: %d (%s)", errno, strerror(errno));
}
}
ttl = 128;
if (taosSetSockOpt(sockFd, IPPROTO_IP, IP_TTL, (void *)&ttl, sizeof(ttl)) < 0) {
uError("setsockopt IP_TTL failed: %d (%s)", errno, strerror(errno));
close(sockFd);
return -1;
}
if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
uError("failed to set the send buffer size for UDP socket\n");
close(sockFd);

9
src/util/src/ttime.c
View File

@ -56,8 +56,13 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0,
year -= 1;
}
int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) +
year*365 - 719499)*24 + hour)*60 + min)*60 + sec);
//int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) +
// year*365 - 719499)*24 + hour)*60 + min)*60 + sec);
int64_t res;
res = 367*((int64_t)mon)/12;
res += year/4 - year/100 + year/400 + day + year*365 - 719499;
res = res*24;
res = ((res + hour) * 60 + min) * 60 + sec;
return (res + timezone);
}

13
src/util/src/ttimer.c
View File

@ -90,7 +90,7 @@ typedef struct time_wheel_t {
} time_wheel_t;
int32_t tmrDebugFlag = 131;
uint32_t taosMaxTmrCtrl = 512;
uint32_t tsMaxTmrCtrl = 512;
static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT;
static pthread_mutex_t tmrCtrlMutex;
@ -385,8 +385,8 @@ static void taosTimerLoopFunc(int signo) {
timer = next;
}
pthread_mutex_unlock(&wheel->mutex);
wheel->nextScanAt += wheel->resolution;
pthread_mutex_unlock(&wheel->mutex);
}
addToExpired(expired);
@ -492,17 +492,17 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle,
}
static void taosTmrModuleInit(void) {
tmrCtrls = malloc(sizeof(tmr_ctrl_t) * taosMaxTmrCtrl);
tmrCtrls = malloc(sizeof(tmr_ctrl_t) * tsMaxTmrCtrl);
if (tmrCtrls == NULL) {
tmrError("failed to allocate memory for timer controllers.");
return;
}
for (int i = 0; i < taosMaxTmrCtrl - 1; ++i) {
for (int i = 0; i < tsMaxTmrCtrl - 1; ++i) {
tmr_ctrl_t* ctrl = tmrCtrls + i;
ctrl->next = ctrl + 1;
}
(tmrCtrls + taosMaxTmrCtrl - 1)->next = NULL;
(tmrCtrls + tsMaxTmrCtrl - 1)->next = NULL;
unusedTmrCtrl = tmrCtrls;
pthread_mutex_init(&tmrCtrlMutex, NULL);
@ -514,17 +514,14 @@ static void taosTmrModuleInit(void) {
tmrError("failed to create the mutex for wheel, reason:%s", strerror(errno));
return;
}
pthread_mutex_lock(&wheel->mutex);
wheel->nextScanAt = now + wheel->resolution;
wheel->index = 0;
wheel->slots = (tmr_obj_t**)calloc(wheel->size, sizeof(tmr_obj_t*));
if (wheel->slots == NULL) {
tmrError("failed to allocate wheel slots");
pthread_mutex_unlock(&wheel->mutex);
return;
}
timerMap.size += wheel->size;
pthread_mutex_unlock(&wheel->mutex);
}
timerMap.count = 0;

27
src/util/src/tutil.c
View File

@ -55,6 +55,25 @@ uint32_t taosRand(void)
*/
return rand();
}
uint32_t trand(void)
{
int fd;
int seed;
fd = open("/dev/urandom", 0);
if (fd < 0) {
seed = time(0);
} else {
int len = read(fd, &seed, sizeof(seed));
if (len < 0) {
seed = time(0);
}
close(fd);
}
return (uint32_t)seed;
}
#endif
size_t twcslen(const wchar_t *wcs) {
@ -295,6 +314,7 @@ int64_t strnatoi(char *num, int32_t len) {
return ret;
}
#if 0
FORCE_INLINE size_t getLen(size_t old, size_t size) {
if (old == 1) {
old = 2;
@ -382,6 +402,7 @@ char *strreplace(const char *str, const char *pattern, const char *rep) {
return dest;
}
#endif
char *strbetween(char *string, char *begin, char *end) {
char *result = NULL;
@ -772,3 +793,9 @@ void taosRemoveDir(char *rootDir) {
uPrint("dir:%s is removed", rootDir);
}
int tmkdir(const char *path, mode_t mode) {
int code = mkdir(path, 0755);
if (code < 0 && errno == EEXIST) code = 0;
return code;
}

2
src/util/tests/stringTest.cpp
View File

@ -26,6 +26,7 @@ TEST(testCase, string_dequote_test) {
EXPECT_EQ(3, lx);
}
#if 0
TEST(testCase, string_replace_test) {
char t3[] = "abc01abc02abc";
char* ret = strreplace(t3, "abc", "7");
@ -87,6 +88,7 @@ TEST(testCase, string_replace_test) {
EXPECT_STREQ("abcdef", ret);
free(ret);
}
#endif
TEST(testCase, string_tolower_test) {
char t[1024] = {1};

46
src/wal/src/walMain.c
View File

@ -71,14 +71,12 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
tstrncpy(pWal->path, path, sizeof(pWal->path));
pthread_mutex_init(&pWal->mutex, NULL);
if (access(path, F_OK) != 0) {
if (mkdir(path, 0755) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("wal:%s, failed to create directory(%s)", path, strerror(errno));
pthread_mutex_destroy(&pWal->mutex);
free(pWal);
pWal = NULL;
}
if (tmkdir(path, 0755) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("wal:%s, failed to create directory(%s)", path, strerror(errno));
pthread_mutex_destroy(&pWal->mutex);
free(pWal);
pWal = NULL;
}
if (pCfg->keep == 1) return pWal;
@ -86,16 +84,15 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
if (walHandleExistingFiles(path) == 0)
walRenew(pWal);
if (pWal->fd <0) {
if (pWal && pWal->fd <0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("wal:%s, failed to open(%s)", path, strerror(errno));
pthread_mutex_destroy(&pWal->mutex);
free(pWal);
pWal = NULL;
} else {
wTrace("wal:%s, it is open, level:%d", path, pWal->level);
}
}
if (pWal) wTrace("wal:%s, it is open, level:%d", path, pWal->level);
return pWal;
}
@ -218,10 +215,13 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
if ( pWal->keep == 0)
strcpy(opath+slen, "/old");
// is there old directory?
if (access(opath, F_OK)) return 0;
DIR *dir = opendir(opath);
if (dir == NULL && errno == ENOENT) return 0;
if (dir == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
while ((ent = readdir(dir))!= NULL) {
if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
index = atol(ent->d_name + plen);
@ -379,12 +379,10 @@ int walHandleExistingFiles(const char *path) {
if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name);
if (access(opath, F_OK) != 0) {
if (mkdir(opath, 0755) != 0) {
wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
break;
}
if (tmkdir(opath, 0755) != 0) {
wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
break;
}
if (rename(oname, nname) < 0) {
@ -409,10 +407,14 @@ static int walRemoveWalFiles(const char *path) {
char name[TSDB_FILENAME_LEN * 3];
terrno = 0;
if (access(path, F_OK) != 0) return 0;
struct dirent *ent;
DIR *dir = opendir(path);
if (dir == NULL && errno == ENOENT) return 0;
if (dir == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
while ((ent = readdir(dir))!= NULL) {
if ( strncmp(ent->d_name, walPrefix, plen) == 0) {

16
tests/examples/c/demo.c
View File

@ -44,7 +44,9 @@ int main(int argc, char *argv[]) {
taos_query(taos, "drop database demo");
if (taos_query(taos, "create database demo") != 0) {
result = taos_query(taos, "create database demo");
if (result == NULL) {
printf("failed to create database, reason:%s\n", taos_errstr(taos));
exit(1);
}
@ -53,7 +55,7 @@ int main(int argc, char *argv[]) {
taos_query(taos, "use demo");
// create table
if (taos_query(taos, "create table m1 (ts timestamp, speed int)") != 0) {
if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) {
printf("failed to create table, reason:%s\n", taos_errstr(taos));
exit(1);
}
@ -65,9 +67,10 @@ int main(int argc, char *argv[]) {
// insert 10 records
int i = 0;
for (i = 0; i < 10; ++i) {
sprintf(qstr, "insert into m1 values (%ld, %d)", 1546300800000 + i * 1000, i * 10);
sprintf(qstr, "insert into m1 values (%ld, %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello");
printf("qstr: %s\n", qstr);
if (taos_query(taos, qstr)) {
printf("failed to insert row: %i, reason:%s\n", i, taos_errstr(taos));
printf("insert row: %i, reason:%s\n", i, taos_errstr(taos));
}
//sleep(1);
}
@ -83,10 +86,11 @@ int main(int argc, char *argv[]) {
TAOS_ROW row;
int rows = 0;
int num_fields = taos_field_count(taos);
int num_fields = taos_field_count(result);
TAOS_FIELD *fields = taos_fetch_fields(result);
char temp[256];
char temp[1024];
printf("num_fields = %d\n", num_fields);
printf("select * from table, result:\n");
// fetch the records row by row
while ((row = taos_fetch_row(result))) {

43
tests/perftest-scripts/perftest-consistent-inserting-data.sh Executable file
View File

@ -0,0 +1,43 @@
#!/bin/bash
ulimit -c unlimited
function buildTDengine {
cd /root/TDengine
git remote update
REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
LOCAL_COMMIT=`git rev-parse --short @`
echo " LOCAL: $LOCAL_COMMIT"
echo "REMOTE: $REMOTE_COMMIT"
if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
echo "repo up-to-date"
else
echo "repo need to pull"
git pull
LOCAL_COMMIT=`git rev-parse --short @`
cd debug
rm -rf *
cmake ..
make > /dev/null
make install
fi
}
function restartTaosd {
systemctl stop taosd
pkill -KILL -x taosd
sleep 10
rm -rf /var/lib/taos/data/*
rm -rf /var/lib/taos/log/*
taosd 2>&1 > /dev/null &
sleep 10
}
buildTDengine
restartTaosd
cd /root/TDengine/tests/pytest/insert
python3 writeDBNonStop.py

3
tests/pytest/alter/alter_table_crash.py
View File

@ -47,7 +47,6 @@ class TDTestCase:
tdSql.query("select * from st")
tdSql.checkRows(1)
print("==============Case 2: keep adding columns, restart taosd")
tdSql.execute(
"create table dt(ts timestamp, tbcol1 tinyint) tags(tgcol1 tinyint)")
@ -72,7 +71,7 @@ class TDTestCase:
tdDnodes.forcestop(1)
tdDnodes.start(1)
tdSql.query("select * from st")
tdSql.query("select * from dt")
tdSql.checkRows(0)
def stop(self):

49
tests/pytest/client/client.py Normal file
View File

@ -0,0 +1,49 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
tdSql.prepare()
ret = tdSql.query('select database()')
tdSql.checkData(0, 0, "db")
ret = tdSql.query('select server_version()')
tdSql.checkData(0, 0, "2.0.0.0")
ret = tdSql.query('select client_version()')
tdSql.checkData(0, 0, "2.0.0.0")
ret = tdSql.query('select server_status()')
tdSql.checkData(0, 0, 1)
ret = tdSql.query('select server_status() as result')
tdSql.checkData(0, 0, 1)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

4
tests/pytest/fulltest.sh
View File

@ -1,6 +1,7 @@
#!/bin/bash
ulimit -c unlimited
python3 ./test.py -f client/client.py
python3 ./test.py -f insert/basic.py
python3 ./test.py -f insert/int.py
python3 ./test.py -f insert/float.py
@ -142,10 +143,11 @@ python3 ./test.py -f query/filterFloatAndDouble.py
python3 ./test.py -f query/filterOtherTypes.py
python3 ./test.py -f query/querySort.py
python3 ./test.py -f query/queryJoin.py
python3 ./test.py -f query/select_last_crash.py
#stream
python3 ./test.py -f stream/stream1.py
python3 ./test.py -f stream/stream2.py
#alter table
python3 ./test.py -f alter/alter_table_crash.py
python3 ./test.py -f alter/alter_table_crash.py

82
tests/pytest/insert/writeDBNonStop.py Normal file
View File

@ -0,0 +1,82 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
import time
from datetime import datetime
import csv
class DBWriteNonStop:
def __init__(self):
self.host = "127.0.0.1"
self.user = "root"
self.password = "taosdata"
self.config = "/etc/taos"
def connectDB(self):
self.conn = taos.connect(
self.host,
self.user,
self.password,
self.config)
self.cursor = self.conn.cursor()
def createTable(self):
self.cursor.execute("drop database if exists dbwrite")
self.cursor.execute("create database dbwrite")
self.cursor.execute("use dbwrite")
self.cursor.execute(
"create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))")
def writeDataToCSVFile(self, data, duration):
csvFile = open('/root/csvFile.csv', 'a', newline='')
writer = csv.writer(csvFile)
writer.writerow([data[0][0], data[0][1], data[0][2],
data[0][3], data[0][4], data[0][5], duration])
csvFile.close()
def insertData(self):
print("===============inserting data===============")
i = 1
startTime = datetime.now()
while True:
self.cursor.execute(
"insert into st1 using st tags('dev_001') values(now, 'taosdata%d', %d)" %
(i %
10000,
i %
100000))
i += 1
i = i % 32000000
endTime = datetime.now()
if (endTime - startTime).seconds >= 3600 * 1:
startTime = endTime
start = datetime.now()
self.cursor.execute(
"select first(ts), last(ts), min(speed), max(speed), avg(speed), count(*) from st")
data = self.cursor.fetchall()
end = datetime.now()
self.writeDataToCSVFile(data, (end - start).seconds)
time.sleep(.001)
def closeConn(self):
self.cursor.close()
self.conn.close()
test = DBWriteNonStop()
test.connectDB()
test.createTable()
test.insertData()
test.closeConn()

54
tests/pytest/query/select_last_crash.py Normal file
View File

@ -0,0 +1,54 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.rowNum = 5000
self.ts = 1537146000000
def run(self):
tdSql.prepare()
tdSql.execute(
"create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))")
tdSql.execute(
"create table t1 using st tags('dev_001')")
for i in range(self.rowNum):
tdSql.execute("insert into t1 values(%d, 'taosdata%d', %d)" % (self.ts + i, i + 1, i + 1))
tdSql.query("select last(*) from st")
tdSql.checkRows(1)
print(
"======= Verify filter for %s type finished =========" %
curType)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

125
tests/pytest/random-test/random-test-multi-threading-3.py
View File

@ -24,19 +24,23 @@ last_tb = ""
last_stb = ""
written = 0
last_timestamp = 0
colAdded = False
killed = False
class Test (Thread):
def __init__(self, threadId, name, events):
def __init__(self, threadId, name, events, q):
Thread.__init__(self)
self.threadId = threadId
self.name = name
self.dataEvent, self.dbEvent, self.queryEvent = events
self.q = q
def create_table(self):
tdLog.info("create_table")
global last_tb
global written
global killed
current_tb = "tb%d" % int(round(time.time() * 1000))
@ -51,8 +55,14 @@ class Test (Thread):
current_tb)
last_tb = current_tb
written = 0
killed = False
except Exception as e:
tdLog.info(repr(e))
tdLog.info("killed: %d error: %s" % (killed, e.args[0]))
if killed and (e.args[0] == 'network unavailable'):
tdLog.info("database killed, expect failed")
return 0
return -1
return 0
def insert_data(self):
tdLog.info("insert_data")
@ -74,22 +84,33 @@ class Test (Thread):
for j in range(0, insertRows):
if (last_tb == ""):
tdLog.info("no table, return")
return
tdSql.execute(
'insert into %s values (%d + %da, %d, "test")' %
(last_tb, start_time, last_timestamp, last_timestamp))
written = written + 1
last_timestamp = last_timestamp + 1
return 0
try:
tdSql.execute(
'insert into %s values (%d + %da, %d, "test")' %
(last_tb, start_time, last_timestamp, last_timestamp))
written = written + 1
last_timestamp = last_timestamp + 1
except Exception as e:
if killed:
tdLog.info(
"database killed, expect failed %s" %
e.args[0])
return 0
tdLog.info(repr(e))
return -1
return 0
def query_data(self):
tdLog.info("query_data")
global last_tb
global written
global killed
if (written > 0):
if not killed and last_tb != "":
tdLog.info("query data from table")
tdSql.query("select * from %s" % last_tb)
tdSql.checkRows(written)
return 0
def create_stable(self):
tdLog.info("create_stable")
@ -123,6 +144,7 @@ class Test (Thread):
(last_tb, start_time, last_timestamp))
written = written + 1
last_timestamp = last_timestamp + 1
return 0
def drop_stable(self):
tdLog.info("drop_stable")
@ -139,22 +161,57 @@ class Test (Thread):
last_stb = ""
last_tb = ""
written = 0
return 0
def alter_table_to_add_col(self):
tdLog.info("alter_table_to_add_col")
global last_stb
global colAdded
if last_stb != "" and colAdded == False:
tdSql.execute(
"alter table %s add column col binary(20)" %
last_stb)
colAdded = True
return 0
def alter_table_to_drop_col(self):
tdLog.info("alter_table_to_drop_col")
global last_stb
global colAdded
if last_stb != "" and colAdded:
tdSql.execute("alter table %s drop column col" % last_stb)
colAdded = False
return 0
def restart_database(self):
tdLog.info("restart_database")
global last_tb
global written
global killed
tdDnodes.stop(1)
killed = True
tdDnodes.start(1)
tdLog.sleep(10)
killed = False
return 0
def force_restart_database(self):
tdLog.info("force_restart_database")
global last_tb
global written
global killed
tdDnodes.forcestop(1)
last_tb = ""
written = 0
killed = True
tdDnodes.start(1)
# tdLog.sleep(10)
killed = False
return 0
def drop_table(self):
tdLog.info("drop_table")
@ -167,6 +224,7 @@ class Test (Thread):
tdSql.execute("drop table %s" % last_tb)
last_tb = ""
written = 0
return 0
def query_data_from_stable(self):
tdLog.info("query_data_from_stable")
@ -178,6 +236,7 @@ class Test (Thread):
else:
tdLog.info("will query data from super table")
tdSql.execute('select * from %s' % last_stb)
return 0
def reset_query_cache(self):
tdLog.info("reset_query_cache")
@ -187,39 +246,45 @@ class Test (Thread):
tdLog.info("reset query cache")
tdSql.execute("reset query cache")
tdLog.sleep(1)
return 0
def reset_database(self):
tdLog.info("reset_database")
global last_tb
global last_stb
global written
global killed
tdDnodes.forcestop(1)
killed = True
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.prepare()
last_tb = ""
last_stb = ""
written = 0
killed = False
return 0
def delete_datafiles(self):
tdLog.info("delete_data_files")
global last_tb
global last_stb
global written
global killed
dnodesDir = tdDnodes.getDnodesRootDir()
tdDnodes.forcestop(1)
killed = True
dataDir = dnodesDir + '/dnode1/data/*'
deleteCmd = 'rm -rf %s' % dataDir
os.system(deleteCmd)
tdDnodes.start(1)
tdSql.prepare()
last_tb = ""
last_stb = ""
written = 0
tdDnodes.start(1)
tdSql.prepare()
killed = False
return 0
def run(self):
dataOp = {
1: self.insert_data,
@ -235,6 +300,8 @@ class Test (Thread):
7: self.reset_database,
8: self.delete_datafiles,
9: self.drop_stable,
10: self.alter_table_to_add_col,
11: self.alter_table_to_drop_col,
}
queryOp = {
@ -247,16 +314,28 @@ class Test (Thread):
self.dataEvent.wait()
tdLog.notice("first thread")
randDataOp = random.randint(1, 1)
dataOp.get(randDataOp, lambda: "ERROR")()
self.dataEvent.clear()
self.queryEvent.clear()
self.dbEvent.set()
ret1 = dataOp.get(randDataOp, lambda: "ERROR")()
if ret1 == -1:
self.q.put(-1)
tdLog.exit("first thread failed")
else:
self.q.put(1)
if (self.q.get() != -2):
self.dataEvent.clear()
self.queryEvent.clear()
self.dbEvent.set()
else:
self.q.put(-1)
tdLog.exit("second thread failed, first thread exit too")
elif (self.threadId == 2):
while True:
self.dbEvent.wait()
tdLog.notice("second thread")
randDbOp = random.randint(1, 9)
randDbOp = random.randint(1, 11)
dbOp.get(randDbOp, lambda: "ERROR")()
self.dbEvent.clear()
self.dataEvent.clear()
@ -298,6 +377,10 @@ class TDTestCase:
test2.join()
test3.join()
while not q.empty():
if (q.get() != 0):
tdLog.exit("failed to end of test")
tdLog.info("end of test")
def stop(self):

155
tests/pytest/random-test/random-test-multi-threading.py
View File

@ -14,6 +14,7 @@
import sys
import random
import threading
import queue
from util.log import *
from util.cases import *
@ -24,13 +25,16 @@ last_tb = ""
last_stb = ""
written = 0
last_timestamp = 0
colAdded = False
killed = False
class Test (threading.Thread):
def __init__(self, threadId, name):
def __init__(self, threadId, name, q):
threading.Thread.__init__(self)
self.threadId = threadId
self.name = name
self.q = q
self.threadLock = threading.Lock()
@ -38,11 +42,12 @@ class Test (threading.Thread):
tdLog.info("create_table")
global last_tb
global written
global killed
current_tb = "tb%d" % int(round(time.time() * 1000))
if (current_tb == last_tb):
return
return 0
else:
tdLog.info("will create table %s" % current_tb)
@ -52,8 +57,14 @@ class Test (threading.Thread):
current_tb)
last_tb = current_tb
written = 0
killed = False
except Exception as e:
tdLog.info(repr(e))
tdLog.info("killed: %d error: %s" % (killed, e.args[0]))
if killed and (e.args[0] == 'network unavailable'):
tdLog.info("database killed, expect failed")
return 0
return -1
return 0
def insert_data(self):
tdLog.info("insert_data")
@ -75,22 +86,34 @@ class Test (threading.Thread):
for j in range(0, insertRows):
if (last_tb == ""):
tdLog.info("no table, return")
return
tdSql.execute(
'insert into %s values (%d + %da, %d, "test")' %
(last_tb, start_time, last_timestamp, last_timestamp))
written = written + 1
last_timestamp = last_timestamp + 1
return 0
try:
tdSql.execute(
'insert into %s values (%d + %da, %d, "test")' %
(last_tb, start_time, last_timestamp, last_timestamp))
written = written + 1
last_timestamp = last_timestamp + 1
except Exception as e:
if killed:
tdLog.info(
"database killed, expect failed %s" %
e.args[0])
return 0
tdLog.info(repr(e))
return -1
return 0
def query_data(self):
tdLog.info("query_data")
global last_tb
global written
global killed
if (written > 0):
if not killed and last_tb != "":
tdLog.info("query data from table")
tdSql.query("select * from %s" % last_tb)
tdSql.checkRows(written)
return 0
def create_stable(self):
tdLog.info("create_stable")
@ -101,9 +124,7 @@ class Test (threading.Thread):
current_stb = "stb%d" % int(round(time.time() * 1000))
if (current_stb == last_stb):
return
else:
if (current_stb != last_stb):
tdLog.info("will create stable %s" % current_stb)
tdLog.info(
'create table %s(ts timestamp, c1 int, c2 nchar(10)) tags (t1 int, t2 nchar(10))' %
@ -131,6 +152,8 @@ class Test (threading.Thread):
written = written + 1
last_timestamp = last_timestamp + 1
return 0
def drop_stable(self):
tdLog.info("drop_stable")
global last_stb
@ -139,31 +162,63 @@ class Test (threading.Thread):
if (last_stb == ""):
tdLog.info("no super table")
return
else:
tdLog.info("will drop last super table")
tdLog.info("will drop last super table %s" % last_stb)
tdSql.execute('drop table %s' % last_stb)
last_stb = ""
last_tb = ""
written = 0
return 0
def alter_table_to_add_col(self):
tdLog.info("alter_table_to_add_col")
global last_stb
global colAdded
if last_stb != "" and not colAdded:
tdSql.execute(
"alter table %s add column col binary(20)" %
last_stb)
colAdded = True
return 0
def alter_table_to_drop_col(self):
tdLog.info("alter_table_to_drop_col")
global last_stb
global colAdded
if last_stb != "" and colAdded:
tdSql.execute("alter table %s drop column col" % last_stb)
colAdded = False
return 0
def restart_database(self):
tdLog.info("restart_database")
global last_tb
global written
global killed
tdDnodes.stop(1)
killed = True
tdDnodes.start(1)
# tdLog.sleep(5)
tdLog.sleep(10)
killed = False
return 0
def force_restart_database(self):
tdLog.info("force_restart_database")
global last_tb
global written
global killed
tdDnodes.forcestop(1)
last_tb = ""
written = 0
killed = True
tdDnodes.start(1)
# tdLog.sleep(10)
killed = False
return 0
def drop_table(self):
tdLog.info("drop_table")
@ -176,6 +231,7 @@ class Test (threading.Thread):
tdSql.execute("drop table %s" % last_tb)
last_tb = ""
written = 0
return 0
def query_data_from_stable(self):
tdLog.info("query_data_from_stable")
@ -183,10 +239,10 @@ class Test (threading.Thread):
if (last_stb == ""):
tdLog.info("no super table")
return
else:
tdLog.info("will query data from super table")
tdSql.execute('select * from %s' % last_stb)
return 0
def reset_query_cache(self):
tdLog.info("reset_query_cache")
@ -196,39 +252,45 @@ class Test (threading.Thread):
tdLog.info("reset query cache")
tdSql.execute("reset query cache")
# tdLog.sleep(1)
return 0
def reset_database(self):
tdLog.info("reset_database")
global last_tb
global last_stb
global written
global killed
tdDnodes.forcestop(1)
killed = True
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.prepare()
last_tb = ""
last_stb = ""
written = 0
killed = False
return 0
def delete_datafiles(self):
tdLog.info("delete_data_files")
global last_tb
global last_stb
global written
global killed
dnodesDir = tdDnodes.getDnodesRootDir()
tdDnodes.forcestop(1)
killed = True
dataDir = dnodesDir + '/dnode1/data/*'
deleteCmd = 'rm -rf %s' % dataDir
os.system(deleteCmd)
tdDnodes.start(1)
tdSql.prepare()
last_tb = ""
last_stb = ""
written = 0
tdDnodes.start(1)
tdSql.prepare()
killed = False
return 0
def run(self):
dataOp = {
1: self.insert_data,
@ -246,6 +308,8 @@ class Test (threading.Thread):
7: self.reset_database,
8: self.delete_datafiles,
9: self.drop_stable,
10: self.alter_table_to_add_col,
11: self.alter_table_to_drop_col,
}
if (self.threadId == 1):
@ -253,16 +317,38 @@ class Test (threading.Thread):
self.threadLock.acquire()
tdLog.notice("first thread")
randDataOp = random.randint(1, 3)
dataOp.get(randDataOp, lambda: "ERROR")()
self.threadLock.release()
ret1 = dataOp.get(randDataOp, lambda: "ERROR")()
if ret1 == -1:
self.q.put(-1)
tdLog.exit("first thread failed")
else:
self.q.put(1)
if (self.q.get() != -2):
self.threadLock.release()
else:
self.q.put(-1)
tdLog.exit("second thread failed, first thread exit too")
elif (self.threadId == 2):
while True:
tdLog.notice("second thread")
self.threadLock.acquire()
randDbOp = random.randint(1, 9)
dbOp.get(randDbOp, lambda: "ERROR")()
self.threadLock.release()
tdLog.notice("second thread")
randDbOp = random.randint(1, 11)
ret2 = dbOp.get(randDbOp, lambda: "ERROR")()
if ret2 == -1:
self.q.put(-2)
tdLog.exit("second thread failed")
else:
self.q.put(2)
if (self.q.get() != -1):
self.threadLock.release()
else:
self.q.put(-2)
tdLog.exit("first thread failed, second exit too")
class TDTestCase:
@ -273,14 +359,19 @@ class TDTestCase:
def run(self):
tdSql.prepare()
test1 = Test(1, "data operation")
test2 = Test(2, "db operation")
q = queue.Queue()
test1 = Test(1, "data operation", q)
test2 = Test(2, "db operation", q)
test1.start()
test2.start()
test1.join()
test2.join()
while not q.empty():
if (q.get() != 0):
tdLog.exit("failed to end of test")
tdLog.info("end of test")
def stop(self):

Some files were not shown because too many files have changed in this diff Show More