xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/2.0tsdb

This commit is contained in:
Hongze Cheng 2020-06-23 17:32:15 +08:00
parent 38b8f4efca 9a6dd6d497
commit 90f44e04e6
10 changed files with 75 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/client/src/tscParseInsert.c
View File

@ -791,12 +791,12 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
sToken = tStrGetToken(sql, &index, false, 0, NULL);
sql += index;
STagData *pTag = (STagData *)pCmd->payload;
tscAllocPayload(pCmd, sizeof(STagData));
STagData *pTag = (STagData *) pCmd->payload;
memset(pTag, 0, sizeof(STagData));
/*
* the source super table is moved to the secondary position of the pTableMetaInfo list
*/
//the source super table is moved to the secondary position of the pTableMetaInfo list
if (pQueryInfo->numOfTables < 2) {
tscAddEmptyMetaInfo(pQueryInfo);
}
@ -897,9 +897,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
index = 0;
sToken = tStrGetToken(sql, &index, true, numOfIgnoreToken, &ignoreTokenTypes);
sql += index;
if (sToken.n == 0) {
break;
} else if (sToken.type == TK_RP) {
if (sToken.n == 0 || sToken.type == TK_RP) {
break;
}
@ -913,11 +912,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if ((pTagSchema[colIndex].type == TSDB_DATA_TYPE_BINARY || pTagSchema[colIndex].type == TSDB_DATA_TYPE_NCHAR) &&
sToken.n > pTagSchema[colIndex].bytes) {
return tscInvalidSQLErrMsg(pCmd->payload, "string too long", sToken.z);
}
}
index = 0;
@ -1042,9 +1036,9 @@ int tsParseInsertSql(SSqlObj *pSql) {
}
// TODO: 2048 is added because TSDB_MAX_TAGS_LEN now is 65536, but TSDB_PAYLOAD_SIZE is 65380
if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE + 2048)) != TSDB_CODE_SUCCESS) {
return code;
}
// if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE + 2048)) != TSDB_CODE_SUCCESS) {
// return code;
// }
if (NULL == pCmd->pTableList) {
pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);

29
src/client/src/tscUtil.c
View File

@ -608,6 +608,9 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
SSqlCmd* pCmd = &pSql->cmd;
// the expanded size when a row data is converted to SDataRow format
const int32_t MAX_EXPAND_SIZE = TD_DATA_ROW_HEAD_SIZE + TYPE_BYTES[TSDB_DATA_TYPE_BINARY];
void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
@ -627,7 +630,9 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
return ret;
}
int64_t destSize = dataBuf->size + pOneTableBlock->size + pOneTableBlock->size*sizeof(int32_t)*2;
SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * MAX_EXPAND_SIZE;
if (dataBuf->nAllocSize < destSize) {
while (dataBuf->nAllocSize < destSize) {
dataBuf->nAllocSize = dataBuf->nAllocSize * 1.5;
@ -648,26 +653,30 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
}
}
SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
tscSortRemoveDataBlockDupRows(pOneTableBlock);
char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1);
tscTrace("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId,
pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey));
int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + sizeof(int32_t) * 2);
int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + MAX_EXPAND_SIZE);
pBlocks->tid = htonl(pBlocks->tid);
pBlocks->uid = htobe64(pBlocks->uid);
pBlocks->sversion = htonl(pBlocks->sversion);
pBlocks->numOfRows = htons(pBlocks->numOfRows);
pBlocks->len = htonl(len);
// erase the empty space reserved for binary data
len = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
dataBuf->size += (len + sizeof(SSubmitBlk));
int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
assert(finalLen <= len);
dataBuf->size += (finalLen + sizeof(SSubmitBlk));
assert(dataBuf->size <= dataBuf->nAllocSize);
// the length does not include the SSubmitBlk structure
pBlocks->len = htonl(finalLen);
dataBuf->numOfTables += 1;
}

29
src/kit/taosdump/taosdump.c
View File

@ -644,14 +644,15 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) {
(void)lseek(fd, 0, SEEK_SET);
STableRecord tableInfo;
while (1) {
memset(&tableRecord, 0, sizeof(STableRecord));
ssize_t ret = read(fd, &tableRecord, sizeof(STableRecord));
memset(&tableInfo, 0, sizeof(STableRecord));
ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord));
if (ret <= 0) break;
tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
tableInfo.name[sizeof(tableInfo.name) - 1] = 0;
tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0;
taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp);
}
close(fd);
@ -910,14 +911,22 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
(void)lseek(fd, 0, SEEK_SET);
STableRecord tableInfo;
char tableName[TSDB_TABLE_NAME_LEN] ;
char metricName[TSDB_TABLE_NAME_LEN];
while (1) {
memset(&tableRecord, 0, sizeof(STableRecord));
ssize_t ret = read(fd, &tableRecord, sizeof(STableRecord));
memset(&tableInfo, 0, sizeof(STableRecord));
memset(tableName, 0, TSDB_TABLE_NAME_LEN);
memset(metricName, 0, TSDB_TABLE_NAME_LEN);
ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord));
if (ret <= 0) break;
tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
//tableInfo.name[sizeof(tableInfo.name) - 1] = 0;
//tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0;
//taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp);
tstrncpy(tableName, tableInfo.name, TSDB_TABLE_NAME_LEN-1);
tstrncpy(metricName, tableInfo.metric, TSDB_TABLE_NAME_LEN-1);
taosDumpTable(tableName, metricName, arguments, fp);
}
close(fd);

17
src/util/src/tconfig.c
View File

@ -111,21 +111,18 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
wordfree(&full_path);
return;
}
if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
strcpy(option, full_path.we_wordv[0]);
}
wordfree(&full_path);
struct stat dirstat;
if (stat(option, &dirstat) < 0) {
int code = mkdir(option, 0755);
if (code < 0) {
uError("config option:%s, input value:%s, directory not exist, create fail with return code:%d",
cfg->option, input_value, code);
} else {
uPrint("config option:%s, input value:%s, directory not exist, create with return code:%d",
cfg->option, input_value, code);
}
int code = tmkdir(option, 0755);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("config option:%s, input value:%s, directory not exist, create fail:%s",
cfg->option, input_value, strerror(errno));
}
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {

9
src/util/src/tlog.c
View File

@ -276,14 +276,15 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
}
}
sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag);
char fileName[LOG_FILE_NAME_LEN + 50] = "\0";
sprintf(fileName, "%s.%d", tsLogObj.logName, tsLogObj.flag);
pthread_mutex_init(&tsLogObj.logMutex, NULL);
umask(0);
tsLogObj.logHandle->fd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
tsLogObj.logHandle->fd = open(fileName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (tsLogObj.logHandle->fd < 0) {
printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno));
printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno));
return -1;
}
taosLockFile(tsLogObj.logHandle->fd);
@ -291,7 +292,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
// only an estimate for number of lines
struct stat filestat;
if (fstat(tsLogObj.logHandle->fd, &filestat) < 0) {
printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno));
printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno));
return -1;
}
size = (int32_t)filestat.st_size;

2
src/util/src/ttime.c
View File

@ -56,7 +56,7 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0,
year -= 1;
}
int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) +
int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + (int64_t)(367*mon)/12 + day) +
year*365 - 719499)*24 + hour)*60 + min)*60 + sec);
return (res + timezone);

16
tests/examples/c/demo.c
View File

@ -44,7 +44,9 @@ int main(int argc, char *argv[]) {
taos_query(taos, "drop database demo");
if (taos_query(taos, "create database demo") != 0) {
result = taos_query(taos, "create database demo");
if (result == NULL) {
printf("failed to create database, reason:%s\n", taos_errstr(taos));
exit(1);
}
@ -53,7 +55,7 @@ int main(int argc, char *argv[]) {
taos_query(taos, "use demo");
// create table
if (taos_query(taos, "create table m1 (ts timestamp, speed int)") != 0) {
if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) {
printf("failed to create table, reason:%s\n", taos_errstr(taos));
exit(1);
}
@ -65,9 +67,10 @@ int main(int argc, char *argv[]) {
// insert 10 records
int i = 0;
for (i = 0; i < 10; ++i) {
sprintf(qstr, "insert into m1 values (%ld, %d)", 1546300800000 + i * 1000, i * 10);
sprintf(qstr, "insert into m1 values (%ld, %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello");
printf("qstr: %s\n", qstr);
if (taos_query(taos, qstr)) {
printf("failed to insert row: %i, reason:%s\n", i, taos_errstr(taos));
printf("insert row: %i, reason:%s\n", i, taos_errstr(taos));
}
//sleep(1);
}
@ -83,10 +86,11 @@ int main(int argc, char *argv[]) {
TAOS_ROW row;
int rows = 0;
int num_fields = taos_field_count(taos);
int num_fields = taos_field_count(result);
TAOS_FIELD *fields = taos_fetch_fields(result);
char temp[256];
char temp[1024];
printf("num_fields = %d\n", num_fields);
printf("select * from table, result:\n");
// fetch the records row by row
while ((row = taos_fetch_row(result))) {

3
tests/pytest/alter/alter_table_crash.py
View File

@ -47,7 +47,6 @@ class TDTestCase:
tdSql.query("select * from st")
tdSql.checkRows(1)
print("==============Case 2: keep adding columns, restart taosd")
tdSql.execute(
"create table dt(ts timestamp, tbcol1 tinyint) tags(tgcol1 tinyint)")
@ -72,7 +71,7 @@ class TDTestCase:
tdDnodes.forcestop(1)
tdDnodes.start(1)
tdSql.query("select * from st")
tdSql.query("select * from dt")
tdSql.checkRows(0)
def stop(self):

2
tests/pytest/regressiontest.sh
View File

@ -141,5 +141,7 @@ python3 ./test.py -f query/querySort.py
python3 ./test.py -f stream/stream1.py
python3 ./test.py -f stream/stream2.py
#alter table
python3 ./test.py -f alter/alter_table_crash.py

4
tests/script/unique/http/opentsdb.sim
View File

@ -75,7 +75,7 @@ endi
system_content curl -u root:taosdata -d '[{"metric": "ab1234567890123456789012345678ab1234567890123456789012345678","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put
print $system_content
if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":-2147482101}}],"failed":1,"success":0,"affected_rows":0}@ then
if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":-2147482101,"desc":"tsdb timestamp is out of range"}}],"failed":1,"success":0,"affected_rows":0}@ then
return -1
endi
@ -125,7 +125,7 @@ endi
system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","group1": "1","group1": "1","group1": "1","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put
print $system_content
if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbbbbbb","table":"sys_cpu_d_bbbbbbb_lga_1_1_1_1_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","group1":"1","group1":"1","group1":"1","group1":"1","host":"web01"},"status":"error","code":-2147482782}}],"failed":1,"success":0,"affected_rows":0}@ then
if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbbbbbb","table":"sys_cpu_d_bbbbbbb_lga_1_1_1_1_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","group1":"1","group1":"1","group1":"1","group1":"1","host":"web01"},"status":"error","code":-2147482782,"desc":"failed to create table"}}],"failed":1,"success":0,"affected_rows":0}@ then
return -1
endi