From 6df8ed9f4eec85a66569b8247d89cfe156599431 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 7 May 2021 15:06:31 +0800 Subject: [PATCH 01/27] support bind multiple tables --- src/client/inc/tscSubquery.h | 2 + src/client/inc/tsclient.h | 3 +- src/client/src/tscParseInsert.c | 7 +- src/client/src/tscPrepare.c | 376 ++++++++++++++++++++++++++++++-- src/client/src/tscUtil.c | 116 +++++----- src/inc/taos.h | 2 + src/query/src/qTokenizer.c | 22 ++ src/util/inc/tstoken.h | 3 + tests/examples/c/makefile | 2 + 9 files changed, 457 insertions(+), 76 deletions(-) diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h index 15ef54b7b1..f0349c2b3d 100644 --- a/src/client/inc/tscSubquery.h +++ b/src/client/inc/tscSubquery.h @@ -48,6 +48,8 @@ void tscLockByThread(int64_t *lockedBy); void tscUnlockByThread(int64_t *lockedBy); +int tsInsertInitialCheck(SSqlObj *pSql); + #ifdef __cplusplus } #endif diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ec3b0c4421..73b0172e85 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -372,7 +372,8 @@ typedef struct SSqlObj { tsem_t rspSem; SSqlCmd cmd; SSqlRes res; - + bool isBind; + SSubqueryState subState; struct SSqlObj **pSubs; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6b88c90747..f964c5173e 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -386,7 +386,7 @@ int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, cha * The server time/client time should not be mixed up in one sql string * Do not employ sort operation is not involved if server time is used. */ -static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start) { +int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start) { // once the data block is disordered, we do NOT keep previous timestamp any more if (!pDataBlocks->ordered) { return TSDB_CODE_SUCCESS; @@ -411,6 +411,7 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { pDataBlocks->ordered = false; + tscWarn("NOT ordered input timestamp"); } pDataBlocks->prevTS = k; @@ -693,6 +694,8 @@ void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) { pBlocks->numOfRows = i + 1; dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows; } + + dataBuf->prevTS = INT64_MIN; } static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, STableDataBlocks* dataBuf, int32_t *totalNum) { @@ -1262,7 +1265,7 @@ int tsParseInsertSql(SSqlObj *pSql) { goto _clean; } - if (taosHashGetSize(pCmd->pTableBlockHashList) > 0) { // merge according to vgId + if ((pCmd->insertType != TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCmd->pTableBlockHashList) > 0) { // merge according to vgId if ((code = tscMergeTableDataBlocks(pSql, true)) != TSDB_CODE_SUCCESS) { goto _clean; } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index c3c8986e2f..baff65d6b9 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -24,6 +24,7 @@ #include "tscSubquery.h" int tsParseInsertSql(SSqlObj *pSql); +int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); //////////////////////////////////////////////////////////////////////////////// // functions for normal statement preparation @@ -43,10 +44,21 @@ typedef struct SNormalStmt { tVariant* params; } SNormalStmt; +typedef struct SMultiTbStmt { + bool nameSet; + uint64_t currentUid; + uint32_t tbNum; + SStrToken tbname; + SHashObj *pTableHash; +} SMultiTbStmt; + typedef struct STscStmt { bool isInsert; + bool multiTbInsert; + int64_t prevTs; STscObj* taos; SSqlObj* pSql; + SMultiTbStmt mtb; SNormalStmt normal; } STscStmt; @@ -255,7 +267,7 @@ static char* normalStmtBuildSql(STscStmt* stmt) { //////////////////////////////////////////////////////////////////////////////// // functions for insertion statement preparation -static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { +static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind) { if (bind->is_null != NULL && *(bind->is_null)) { setNull(data + param->offset, param->type, param->bytes); return TSDB_CODE_SUCCESS; @@ -690,29 +702,52 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { } memcpy(data + param->offset, bind->buffer, size); + if (param->offset == 0) { + if (tsCheckTimestamp(pBlock, data + param->offset) != TSDB_CODE_SUCCESS) { + tscError("invalid timestamp"); + return TSDB_CODE_TSC_INVALID_VALUE; + } + } + return TSDB_CODE_SUCCESS; } static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { SSqlCmd* pCmd = &stmt->pSql->cmd; - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->pTableBlockHashList == NULL) { - pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - + STscStmt* pStmt = (STscStmt*)stmt; + STableDataBlocks* pBlock = NULL; + + if (pStmt->multiTbInsert) { + if (pCmd->pTableBlockHashList == NULL) { + tscError("Table block hash list is empty"); + return TSDB_CODE_TSC_APP_ERROR; + } + + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); + if (t1 == NULL) { + tscError("no table data block in hash list, uid:%" PRId64 , pStmt->mtb.currentUid); + return TSDB_CODE_TSC_APP_ERROR; + } - int32_t ret = - tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - if (ret != 0) { - // todo handle error + pBlock = *t1; + } else { + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); + + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pCmd->pTableBlockHashList == NULL) { + pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + } + + int32_t ret = + tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); + if (ret != 0) { + return ret; + } } - uint32_t totalDataSize = sizeof(SSubmitBlk) + pCmd->batchSize * pBlock->rowSize; + uint32_t totalDataSize = sizeof(SSubmitBlk) + (pCmd->batchSize + 1) * pBlock->rowSize; if (totalDataSize > pBlock->nAllocSize) { const double factor = 1.5; @@ -729,7 +764,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { SParamInfo* param = &pBlock->params[j]; - int code = doBindParam(data, param, &bind[param->idx]); + int code = doBindParam(pBlock, data, param, &bind[param->idx]); if (code != TSDB_CODE_SUCCESS) { tscDebug("param %d: type mismatch or invalid", param->idx); return code; @@ -739,9 +774,98 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { return TSDB_CODE_SUCCESS; } + +static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_BIND* bind, int32_t num) { + SSqlCmd* pCmd = &stmt->pSql->cmd; + STscStmt* pStmt = (STscStmt*)stmt; + + STableDataBlocks* pBlock = NULL; + + if (pCmd->pTableBlockHashList == NULL) { + tscError("Table block hash list is empty"); + return TSDB_CODE_TSC_APP_ERROR; + } + + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); + if (t1 == NULL) { + tscError("no table data block in hash list, uid:%" PRId64 , pStmt->mtb.currentUid); + return TSDB_CODE_TSC_APP_ERROR; + } + + pBlock = *t1; + + uint32_t totalDataSize = sizeof(SSubmitBlk) + (pCmd->batchSize + num) * pBlock->rowSize; + if (totalDataSize > pBlock->nAllocSize) { + const double factor = 1.5; + + void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); + if (tmp == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pBlock->pData = (char*)tmp; + pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); + } + + for (uint32_t i = 0; i < num; ++i) { + char* data = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * (pCmd->batchSize + i); + TAOS_BIND* tbind = bind + pBlock->numOfParams * i; + + for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { + SParamInfo* param = &pBlock->params[j]; + + int code = doBindParam(pBlock, data, param, &tbind[param->idx]); + if (code != TSDB_CODE_SUCCESS) { + tscError("param %d: type mismatch or invalid", param->idx); + return code; + } + } + } + + pCmd->batchSize += num - 1; + + return TSDB_CODE_SUCCESS; +} + + +static int insertStmtUpdateBatch(STscStmt* stmt) { + SSqlObj* pSql = stmt->pSql; + SSqlCmd* pCmd = &pSql->cmd; + STableDataBlocks* pBlock = NULL; + + assert(pCmd->numOfClause == 1); + if (taosHashGetSize(pCmd->pTableBlockHashList) == 0) { + return TSDB_CODE_SUCCESS; + } + + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&stmt->mtb.currentUid, sizeof(stmt->mtb.currentUid)); + if (t1 == NULL) { + tscError("no table data block in hash list, uid:%" PRId64 , stmt->mtb.currentUid); + return TSDB_CODE_TSC_APP_ERROR; + } + + pBlock = *t1; + + STableMeta* pTableMeta = pBlock->pTableMeta; + + pBlock->size = sizeof(SSubmitBlk) + pCmd->batchSize * pBlock->rowSize; + SSubmitBlk* pBlk = (SSubmitBlk*) pBlock->pData; + pBlk->numOfRows = pCmd->batchSize; + pBlk->dataLen = 0; + pBlk->uid = pTableMeta->id.uid; + pBlk->tid = pTableMeta->id.tid; + + return TSDB_CODE_SUCCESS; +} + static int insertStmtAddBatch(STscStmt* stmt) { SSqlCmd* pCmd = &stmt->pSql->cmd; ++pCmd->batchSize; + + if (stmt->multiTbInsert) { + return insertStmtUpdateBatch(stmt); + } + return TSDB_CODE_SUCCESS; } @@ -835,6 +959,80 @@ static int insertStmtExecute(STscStmt* stmt) { return pSql->res.code; } +static void insertBatchClean(STscStmt* pStmt) { + SSqlCmd *pCmd = &pStmt->pSql->cmd; + SSqlObj *pSql = pStmt->pSql; + int32_t size = taosHashGetSize(pCmd->pTableBlockHashList); + + // data block reset + pCmd->batchSize = 0; + + for(int32_t i = 0; i < size; ++i) { + if (pCmd->pTableNameList && pCmd->pTableNameList[i]) { + tfree(pCmd->pTableNameList[i]); + } + } + + tfree(pCmd->pTableNameList); + + STableDataBlocks** p = taosHashIterate(pCmd->pTableBlockHashList, NULL); + + STableDataBlocks* pOneTableBlock = *p; + + while (1) { + SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; + + pOneTableBlock->size = sizeof(SSubmitBlk); + + pBlocks->numOfRows = 0; + + p = taosHashIterate(pCmd->pTableBlockHashList, p); + if (p == NULL) { + break; + } + + pOneTableBlock = *p; + } + + pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + pCmd->numOfTables = 0; + + tscFreeSqlResult(pSql); + tscFreeSubobj(pSql); + tfree(pSql->pSubs); + pSql->subState.numOfSub = 0; +} + +static int insertBatchStmtExecute(STscStmt* pStmt) { + int32_t code = 0; + + if(pStmt->mtb.nameSet == false) { + tscError("no table name set"); + return TSDB_CODE_TSC_APP_ERROR; + } + + pStmt->pSql->retry = pStmt->pSql->maxRetry + 1; //no retry + + if (taosHashGetSize(pStmt->pSql->cmd.pTableBlockHashList) > 0) { // merge according to vgId + if ((code = tscMergeTableDataBlocks(pStmt->pSql, false)) != TSDB_CODE_SUCCESS) { + return code; + } + } + + code = tscHandleMultivnodeInsert(pStmt->pSql); + + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // wait for the callback function to post the semaphore + tsem_wait(&pStmt->pSql->rspSem); + + insertBatchClean(pStmt); + + return pStmt->pSql->res.code; +} + //////////////////////////////////////////////////////////////////////////////// // interface functions @@ -866,6 +1064,7 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) { pSql->signature = pSql; pSql->pTscObj = pObj; pSql->maxRetry = TSDB_MAX_REPLICA; + pSql->isBind = true; pStmt->pSql = pSql; return pStmt; @@ -917,6 +1116,32 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { registerSqlObj(pSql); + int32_t ret = TSDB_CODE_SUCCESS; + + if ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS) { + return ret; + } + + int32_t index = 0; + SStrToken sToken = tStrGetToken(pCmd->curSql, &index, false); + + if (sToken.n == 0) { + return TSDB_CODE_TSC_INVALID_SQL; + } + + if (sToken.n == 1 && sToken.type == TK_QUESTION) { + pStmt->multiTbInsert = true; + pStmt->mtb.tbname = sToken; + pStmt->mtb.nameSet = false; + if (pStmt->mtb.pTableHash == NULL) { + pStmt->mtb.pTableHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + } + return TSDB_CODE_SUCCESS; + } + + pStmt->multiTbInsert = false; + memset(&pStmt->mtb, 0, sizeof(pStmt->mtb)); + int32_t code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { // wait for the callback function to post the semaphore @@ -931,9 +1156,100 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { return normalStmtPrepare(pStmt); } + +int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { + STscStmt* pStmt = (STscStmt*)stmt; + SSqlObj* pSql = pStmt->pSql; + SSqlCmd* pCmd = &pSql->cmd; + + if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + + if (name == NULL) { + terrno = TSDB_CODE_TSC_APP_ERROR; + tscError("name is NULL"); + return TSDB_CODE_TSC_APP_ERROR; + } + + if (pStmt->multiTbInsert == false || !tscIsInsertData(pSql->sqlstr)) { + terrno = TSDB_CODE_TSC_APP_ERROR; + tscError("not multi table insert"); + return TSDB_CODE_TSC_APP_ERROR; + } + + uint64_t* uid = (uint64_t*)taosHashGet(pStmt->mtb.pTableHash, name, strlen(name)); + if (uid != NULL) { + pStmt->mtb.currentUid = *uid; + + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); + if (t1 == NULL) { + tscError("no table data block in hash list, uid:%" PRId64 , pStmt->mtb.currentUid); + return TSDB_CODE_TSC_APP_ERROR; + } + + SSubmitBlk* pBlk = (SSubmitBlk*) (*t1)->pData; + pCmd->batchSize = pBlk->numOfRows; + + tscDebug("table:%s is already prepared, uid:%" PRIu64, name, pStmt->mtb.currentUid); + return TSDB_CODE_SUCCESS; + } + + pStmt->mtb.tbname = tscReplaceStrToken(&pSql->sqlstr, &pStmt->mtb.tbname, name); + + pStmt->mtb.nameSet = true; + + tscDebug("sqlstr set to %s", pSql->sqlstr); + + pSql->cmd.parseFinished = 0; + pSql->cmd.numOfParams = 0; + pSql->cmd.batchSize = 0; + + if (taosHashGetSize(pCmd->pTableBlockHashList) > 0) { + SHashObj* hashList = pCmd->pTableBlockHashList; + pCmd->pTableBlockHashList = NULL; + tscResetSqlCmd(pCmd, true); + pCmd->pTableBlockHashList = hashList; + } + + int32_t code = tsParseSql(pStmt->pSql, true); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + // wait for the callback function to post the semaphore + tsem_wait(&pStmt->pSql->rspSem); + + code = pStmt->pSql->res.code; + } + + if (code == TSDB_CODE_SUCCESS) { + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + STableDataBlocks* pBlock = NULL; + code = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + SSubmitBlk* blk = (SSubmitBlk*)pBlock->pData; + blk->numOfRows = 0; + + pStmt->mtb.currentUid = pTableMeta->id.uid; + pStmt->mtb.tbNum++; + + taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); + + tscDebug("table:%s is prepared, uid:%" PRIu64, name, pStmt->mtb.currentUid); + } + + return code; +} + int taos_stmt_close(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; if (!pStmt->isInsert) { + taosHashCleanup(pStmt->mtb.pTableHash); + SNormalStmt* normal = &pStmt->normal; if (normal->params != NULL) { for (uint16_t i = 0; i < normal->numParams; i++) { @@ -953,12 +1269,34 @@ int taos_stmt_close(TAOS_STMT* stmt) { int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { STscStmt* pStmt = (STscStmt*)stmt; if (pStmt->isInsert) { + if (pStmt->multiTbInsert && pStmt->mtb.nameSet == false) { + tscError("no table name set"); + return TSDB_CODE_TSC_APP_ERROR; + } + return insertStmtBindParam(pStmt, bind); } else { return normalStmtBindParam(pStmt, bind); } } + +int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_BIND* bind, int32_t num) { + STscStmt* pStmt = (STscStmt*)stmt; + if (num <= 0 || bind == NULL) { + tscError("invalid parameter"); + return TSDB_CODE_TSC_APP_ERROR; + } + + if (!pStmt->isInsert || !pStmt->multiTbInsert || !pStmt->mtb.nameSet) { + tscError("not or invalid batch insert"); + return TSDB_CODE_TSC_APP_ERROR; + } + + return insertStmtBindParamBatch(pStmt, bind, num); +} + + int taos_stmt_add_batch(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; if (pStmt->isInsert) { @@ -979,7 +1317,11 @@ int taos_stmt_execute(TAOS_STMT* stmt) { int ret = 0; STscStmt* pStmt = (STscStmt*)stmt; if (pStmt->isInsert) { - ret = insertStmtExecute(pStmt); + if (pStmt->multiTbInsert) { + ret = insertBatchStmtExecute(pStmt); + } else { + ret = insertStmtExecute(pStmt); + } } else { // normal stmt query char* sql = normalStmtBuildSql(pStmt); if (sql == NULL) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 420b78f64d..eb31048433 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1255,67 +1255,71 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap) { STableDataBlocks* pOneTableBlock = *p; while(pOneTableBlock) { - // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta); - STableDataBlocks* dataBuf = NULL; - - int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); - if (ret != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to prepare the data block buffer for merging table data, code:%d", pSql->self, ret); - taosHashCleanup(pVnodeDataBlockHashList); - tscDestroyBlockArrayList(pVnodeDataBlockList); - return ret; - } - SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); - - if (dataBuf->nAllocSize < destSize) { - while (dataBuf->nAllocSize < destSize) { - dataBuf->nAllocSize = (uint32_t)(dataBuf->nAllocSize * 1.5); - } - - char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize); - if (tmp != NULL) { - dataBuf->pData = tmp; - memset(dataBuf->pData + dataBuf->size, 0, dataBuf->nAllocSize - dataBuf->size); - } else { // failed to allocate memory, free already allocated memory and return error code - tscError("0x%"PRIx64" failed to allocate memory for merging submit block, size:%d", pSql->self, dataBuf->nAllocSize); - + if (pBlocks->numOfRows > 0) { + // the maximum expanded size in byte when a row-wise data is converted to SDataRow format + int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta); + STableDataBlocks* dataBuf = NULL; + + int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, + INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + if (ret != TSDB_CODE_SUCCESS) { + tscError("0x%"PRIx64" failed to prepare the data block buffer for merging table data, code:%d", pSql->self, ret); taosHashCleanup(pVnodeDataBlockHashList); tscDestroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return ret; } + + int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); + + if (dataBuf->nAllocSize < destSize) { + while (dataBuf->nAllocSize < destSize) { + dataBuf->nAllocSize = (uint32_t)(dataBuf->nAllocSize * 1.5); + } + + char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize); + if (tmp != NULL) { + dataBuf->pData = tmp; + memset(dataBuf->pData + dataBuf->size, 0, dataBuf->nAllocSize - dataBuf->size); + } else { // failed to allocate memory, free already allocated memory and return error code + tscError("0x%"PRIx64" failed to allocate memory for merging submit block, size:%d", pSql->self, dataBuf->nAllocSize); + + taosHashCleanup(pVnodeDataBlockHashList); + tscDestroyBlockArrayList(pVnodeDataBlockList); + tfree(dataBuf->pData); + + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + tscSortRemoveDataBlockDupRows(pOneTableBlock); + char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); + + tscDebug("0x%"PRIx64" name:%s, name:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql->self, tNameGetTableName(&pOneTableBlock->tableName), + pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); + + int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); + + pBlocks->tid = htonl(pBlocks->tid); + pBlocks->uid = htobe64(pBlocks->uid); + pBlocks->sversion = htonl(pBlocks->sversion); + pBlocks->numOfRows = htons(pBlocks->numOfRows); + pBlocks->schemaLen = 0; + + // erase the empty space reserved for binary data + int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pCmd->submitSchema); + assert(finalLen <= len); + + dataBuf->size += (finalLen + sizeof(SSubmitBlk)); + assert(dataBuf->size <= dataBuf->nAllocSize); + + // the length does not include the SSubmitBlk structure + pBlocks->dataLen = htonl(finalLen); + dataBuf->numOfTables += 1; + }else { + tscWarn("table %s data block is empty", pOneTableBlock->tableName.tname); } - - tscSortRemoveDataBlockDupRows(pOneTableBlock); - char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); - - tscDebug("0x%"PRIx64" name:%s, name:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql->self, tNameGetTableName(&pOneTableBlock->tableName), - pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); - - int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); - - pBlocks->tid = htonl(pBlocks->tid); - pBlocks->uid = htobe64(pBlocks->uid); - pBlocks->sversion = htonl(pBlocks->sversion); - pBlocks->numOfRows = htons(pBlocks->numOfRows); - pBlocks->schemaLen = 0; - - // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pCmd->submitSchema); - assert(finalLen <= len); - - dataBuf->size += (finalLen + sizeof(SSubmitBlk)); - assert(dataBuf->size <= dataBuf->nAllocSize); - - // the length does not include the SSubmitBlk structure - pBlocks->dataLen = htonl(finalLen); - dataBuf->numOfTables += 1; - + p = taosHashIterate(pCmd->pTableBlockHashList, p); if (p == NULL) { break; diff --git a/src/inc/taos.h b/src/inc/taos.h index cd8e116053..3f6609f6df 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -101,10 +101,12 @@ typedef struct TAOS_BIND { TAOS_STMT *taos_stmt_init(TAOS *taos); int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); +int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name); int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); +int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_BIND* bind, int32_t num); int taos_stmt_add_batch(TAOS_STMT *stmt); int taos_stmt_execute(TAOS_STMT *stmt); TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 7869e27707..00fcaf82f5 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -560,6 +560,28 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { return 0; } +SStrToken tscReplaceStrToken(char **str, SStrToken *token, const char* new) { + char *src = *str; + int32_t nsize = strlen(new); + int32_t size = strlen(*str) - token->n + nsize + 1; + int32_t bsize = (uint64_t)token->z - (uint64_t)src; + SStrToken ntoken; + + *str = calloc(1, size); + + strncpy(*str, src, bsize); + strcat(*str, new); + strcat(*str, token->z + token->n); + + ntoken.n = nsize; + ntoken.z = *str + bsize; + + tfree(src); + + return ntoken; +} + + SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { SStrToken t0 = {0}; diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index ab1ef7b279..93d48e01cb 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -182,6 +182,9 @@ static FORCE_INLINE int32_t tGetNumericStringType(const SStrToken* pToken) { void taosCleanupKeywordsTable(); +SStrToken tscReplaceStrToken(char **str, SStrToken *token, const char* new); + + #ifdef __cplusplus } #endif diff --git a/tests/examples/c/makefile b/tests/examples/c/makefile index b06fe551db..09682d35ef 100644 --- a/tests/examples/c/makefile +++ b/tests/examples/c/makefile @@ -14,6 +14,7 @@ exe: gcc $(CFLAGS) ./asyncdemo.c -o $(ROOT)asyncdemo $(LFLAGS) gcc $(CFLAGS) ./demo.c -o $(ROOT)demo $(LFLAGS) gcc $(CFLAGS) ./prepare.c -o $(ROOT)prepare $(LFLAGS) + gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) gcc $(CFLAGS) ./stream.c -o $(ROOT)stream $(LFLAGS) gcc $(CFLAGS) ./subscribe.c -o $(ROOT)subscribe $(LFLAGS) gcc $(CFLAGS) ./apitest.c -o $(ROOT)apitest $(LFLAGS) @@ -22,6 +23,7 @@ clean: rm $(ROOT)asyncdemo rm $(ROOT)demo rm $(ROOT)prepare + rm $(ROOT)batchprepare rm $(ROOT)stream rm $(ROOT)subscribe rm $(ROOT)apitest From 020fa3dd06751e50c96b11b5ec82c82d79dc9411 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 8 May 2021 11:52:24 +0800 Subject: [PATCH 02/27] performance issue --- src/client/src/tscPrepare.c | 16 ++++++++++++++-- src/client/src/tscUtil.c | 4 +++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index baff65d6b9..ec85dff359 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -49,7 +49,8 @@ typedef struct SMultiTbStmt { uint64_t currentUid; uint32_t tbNum; SStrToken tbname; - SHashObj *pTableHash; + SHashObj *pTableHash; + SHashObj *pTableBlockHashList; // data block for each table } SMultiTbStmt; typedef struct STscStmt { @@ -975,6 +976,7 @@ static void insertBatchClean(STscStmt* pStmt) { tfree(pCmd->pTableNameList); +/* STableDataBlocks** p = taosHashIterate(pCmd->pTableBlockHashList, NULL); STableDataBlocks* pOneTableBlock = *p; @@ -993,10 +995,12 @@ static void insertBatchClean(STscStmt* pStmt) { pOneTableBlock = *p; } +*/ pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); pCmd->numOfTables = 0; + taosHashEmpty(pCmd->pTableBlockHashList); tscFreeSqlResult(pSql); tscFreeSubobj(pSql); tfree(pSql->pSubs); @@ -1136,6 +1140,10 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { if (pStmt->mtb.pTableHash == NULL) { pStmt->mtb.pTableHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); } + if (pStmt->mtb.pTableBlockHashList == NULL) { + pStmt->mtb.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + } + return TSDB_CODE_SUCCESS; } @@ -1183,7 +1191,7 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { if (uid != NULL) { pStmt->mtb.currentUid = *uid; - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pStmt->mtb.pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); if (t1 == NULL) { tscError("no table data block in hash list, uid:%" PRId64 , pStmt->mtb.currentUid); return TSDB_CODE_TSC_APP_ERROR; @@ -1191,6 +1199,8 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { SSubmitBlk* pBlk = (SSubmitBlk*) (*t1)->pData; pCmd->batchSize = pBlk->numOfRows; + + taosHashPut(pCmd->pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)t1, POINTER_BYTES); tscDebug("table:%s is already prepared, uid:%" PRIu64, name, pStmt->mtb.currentUid); return TSDB_CODE_SUCCESS; @@ -1236,6 +1246,8 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { pStmt->mtb.currentUid = pTableMeta->id.uid; pStmt->mtb.tbNum++; + + taosHashPut(pStmt->mtb.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index eb31048433..7a6842869a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1316,8 +1316,10 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap) { // the length does not include the SSubmitBlk structure pBlocks->dataLen = htonl(finalLen); dataBuf->numOfTables += 1; + + pBlocks->numOfRows = 0; }else { - tscWarn("table %s data block is empty", pOneTableBlock->tableName.tname); + tscDebug("table %s data block is empty", pOneTableBlock->tableName.tname); } p = taosHashIterate(pCmd->pTableBlockHashList, p); From 003562bc78de37a5682c3c79000047cbac283337 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 8 May 2021 17:19:21 +0800 Subject: [PATCH 03/27] fix bug --- src/client/src/tscPrepare.c | 119 ++++++++++++++++++++++++++++++------ src/inc/taos.h | 14 ++++- 2 files changed, 113 insertions(+), 20 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index ec85dff359..1c263487df 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -268,12 +268,13 @@ static char* normalStmtBuildSql(STscStmt* stmt) { //////////////////////////////////////////////////////////////////////////////// // functions for insertion statement preparation -static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind) { +static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { if (bind->is_null != NULL && *(bind->is_null)) { setNull(data + param->offset, param->type, param->bytes); return TSDB_CODE_SUCCESS; } +#if 0 if (0) { // allow user bind param data with different type union { @@ -654,6 +655,7 @@ static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, } } } +#endif if (bind->buffer_type != param->type) { return TSDB_CODE_TSC_INVALID_VALUE; @@ -713,6 +715,85 @@ static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, return TSDB_CODE_SUCCESS; } + +static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MULTI_BIND* bind, int32_t rowNum) { + if (bind->buffer_type != param->type) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + short size = 0; + switch(param->type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + size = 1; + break; + + case TSDB_DATA_TYPE_SMALLINT: + size = 2; + break; + + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_FLOAT: + size = 4; + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + size = 8; + break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + if (bind->length == NULL) { + tscError("BINARY/NCHAR no length"); + return TSDB_CODE_TSC_INVALID_VALUE; + } + break; + + default: + assert(false); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + + for (int i = 0; i < bind->num; ++i) { + char* data = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * (rowNum + i); + + if (bind->is_null != NULL && bind->is_null[i]) { + setNull(data + param->offset, param->type, param->bytes); + return TSDB_CODE_SUCCESS; + } + + if (size > 0) { + memcpy(data + param->offset, bind->buffer + bind->buffer_length * i, size); + + if (param->offset == 0) { + if (tsCheckTimestamp(pBlock, data + param->offset) != TSDB_CODE_SUCCESS) { + tscError("invalid timestamp"); + return TSDB_CODE_TSC_INVALID_VALUE; + } + } + } else if (param->type == TSDB_DATA_TYPE_BINARY) { + if (bind->length[i] > (uintptr_t)param->bytes) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + size = (short)bind->length[i]; + STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer + bind->buffer_length * i, size); + } else if (param->type == TSDB_DATA_TYPE_NCHAR) { + int32_t output = 0; + if (!taosMbsToUcs4(bind->buffer + bind->buffer_length * i, bind->length[i], varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + varDataSetLen(data + param->offset, output); + } + } + + return TSDB_CODE_SUCCESS; +} + + + static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { SSqlCmd* pCmd = &stmt->pSql->cmd; STscStmt* pStmt = (STscStmt*)stmt; @@ -765,7 +846,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { SParamInfo* param = &pBlock->params[j]; - int code = doBindParam(pBlock, data, param, &bind[param->idx]); + int code = doBindParam(pBlock, data, param, &bind[param->idx], 1); if (code != TSDB_CODE_SUCCESS) { tscDebug("param %d: type mismatch or invalid", param->idx); return code; @@ -776,9 +857,10 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { } -static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_BIND* bind, int32_t num) { +static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind) { SSqlCmd* pCmd = &stmt->pSql->cmd; STscStmt* pStmt = (STscStmt*)stmt; + int rowNum = bind->num; STableDataBlocks* pBlock = NULL; @@ -795,7 +877,7 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_BIND* bind, int32_t num pBlock = *t1; - uint32_t totalDataSize = sizeof(SSubmitBlk) + (pCmd->batchSize + num) * pBlock->rowSize; + uint32_t totalDataSize = sizeof(SSubmitBlk) + (pCmd->batchSize + rowNum) * pBlock->rowSize; if (totalDataSize > pBlock->nAllocSize) { const double factor = 1.5; @@ -808,22 +890,21 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_BIND* bind, int32_t num pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); } - for (uint32_t i = 0; i < num; ++i) { - char* data = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * (pCmd->batchSize + i); - TAOS_BIND* tbind = bind + pBlock->numOfParams * i; + for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { + SParamInfo* param = &pBlock->params[j]; + if (bind[param->idx].num != rowNum) { + tscError("param %d: num[%d:%d] not match", param->idx, rowNum, bind[param->idx].num); + return TSDB_CODE_TSC_INVALID_VALUE; + } - for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { - SParamInfo* param = &pBlock->params[j]; - - int code = doBindParam(pBlock, data, param, &tbind[param->idx]); - if (code != TSDB_CODE_SUCCESS) { - tscError("param %d: type mismatch or invalid", param->idx); - return code; - } + int code = doBindBatchParam(pBlock, param, &bind[param->idx], pCmd->batchSize); + if (code != TSDB_CODE_SUCCESS) { + tscError("param %d: type mismatch or invalid", param->idx); + return code; } } - pCmd->batchSize += num - 1; + pCmd->batchSize += rowNum - 1; return TSDB_CODE_SUCCESS; } @@ -1293,9 +1374,9 @@ int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { } -int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_BIND* bind, int32_t num) { +int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { STscStmt* pStmt = (STscStmt*)stmt; - if (num <= 0 || bind == NULL) { + if (bind == NULL || bind->num <= 0) { tscError("invalid parameter"); return TSDB_CODE_TSC_APP_ERROR; } @@ -1305,7 +1386,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_BIND* bind, int32_t num) { return TSDB_CODE_TSC_APP_ERROR; } - return insertStmtBindParamBatch(pStmt, bind, num); + return insertStmtBindParamBatch(pStmt, bind); } diff --git a/src/inc/taos.h b/src/inc/taos.h index 3f6609f6df..ca87337800 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -82,6 +82,7 @@ typedef struct TAOS_BIND { uintptr_t buffer_length; // unused uintptr_t *length; int * is_null; + int is_unsigned; // unused int * error; // unused union { @@ -99,6 +100,17 @@ typedef struct TAOS_BIND { unsigned int allocated; } TAOS_BIND; +typedef struct TAOS_MULTI_BIND { + int buffer_type; + void * buffer; + uintptr_t buffer_length; // unused + uintptr_t *length; + int * is_null; + + int num; +} TAOS_MULTI_BIND; + + TAOS_STMT *taos_stmt_init(TAOS *taos); int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name); @@ -106,7 +118,7 @@ int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); -int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_BIND* bind, int32_t num); +int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind); int taos_stmt_add_batch(TAOS_STMT *stmt); int taos_stmt_execute(TAOS_STMT *stmt); TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); From 9cc8a36c1ffb38d8996ff2987c9d129a7fa8c957 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 8 May 2021 19:49:45 +0800 Subject: [PATCH 04/27] support bind single column --- src/client/src/tscPrepare.c | 62 ++++++++++++++++++++++++++++--------- src/inc/taos.h | 1 + 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 1c263487df..9af7b11615 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -762,7 +762,7 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU if (bind->is_null != NULL && bind->is_null[i]) { setNull(data + param->offset, param->type, param->bytes); - return TSDB_CODE_SUCCESS; + continue; } if (size > 0) { @@ -776,13 +776,15 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU } } else if (param->type == TSDB_DATA_TYPE_BINARY) { if (bind->length[i] > (uintptr_t)param->bytes) { + tscError("invalid binary length"); return TSDB_CODE_TSC_INVALID_VALUE; } - size = (short)bind->length[i]; - STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer + bind->buffer_length * i, size); + int16_t bsize = (short)bind->length[i]; + STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer + bind->buffer_length * i, bsize); } else if (param->type == TSDB_DATA_TYPE_NCHAR) { int32_t output = 0; if (!taosMbsToUcs4(bind->buffer + bind->buffer_length * i, bind->length[i], varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { + tscError("convert failed"); return TSDB_CODE_TSC_INVALID_VALUE; } varDataSetLen(data + param->offset, output); @@ -857,7 +859,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { } -static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind) { +static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int colIdx) { SSqlCmd* pCmd = &stmt->pSql->cmd; STscStmt* pStmt = (STscStmt*)stmt; int rowNum = bind->num; @@ -877,6 +879,8 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind) { pBlock = *t1; + assert(colIdx == -1 || (colIdx >= 0 && colIdx < pBlock->numOfParams)); + uint32_t totalDataSize = sizeof(SSubmitBlk) + (pCmd->batchSize + rowNum) * pBlock->rowSize; if (totalDataSize > pBlock->nAllocSize) { const double factor = 1.5; @@ -890,21 +894,35 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind) { pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); } - for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { - SParamInfo* param = &pBlock->params[j]; - if (bind[param->idx].num != rowNum) { - tscError("param %d: num[%d:%d] not match", param->idx, rowNum, bind[param->idx].num); - return TSDB_CODE_TSC_INVALID_VALUE; + if (colIdx == -1) { + for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { + SParamInfo* param = &pBlock->params[j]; + if (bind[param->idx].num != rowNum) { + tscError("param %d: num[%d:%d] not match", param->idx, rowNum, bind[param->idx].num); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int code = doBindBatchParam(pBlock, param, &bind[param->idx], pCmd->batchSize); + if (code != TSDB_CODE_SUCCESS) { + tscError("param %d: type mismatch or invalid", param->idx); + return code; + } } - - int code = doBindBatchParam(pBlock, param, &bind[param->idx], pCmd->batchSize); + + pCmd->batchSize += rowNum - 1; + } else { + SParamInfo* param = &pBlock->params[colIdx]; + + int code = doBindBatchParam(pBlock, param, bind, pCmd->batchSize); if (code != TSDB_CODE_SUCCESS) { tscError("param %d: type mismatch or invalid", param->idx); return code; } - } - pCmd->batchSize += rowNum - 1; + if (colIdx == (pBlock->numOfParams - 1)) { + pCmd->batchSize += rowNum - 1; + } + } return TSDB_CODE_SUCCESS; } @@ -1386,9 +1404,25 @@ int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { return TSDB_CODE_TSC_APP_ERROR; } - return insertStmtBindParamBatch(pStmt, bind); + return insertStmtBindParamBatch(pStmt, bind, -1); } +int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int colIdx) { + STscStmt* pStmt = (STscStmt*)stmt; + if (bind == NULL || bind->num <= 0) { + tscError("invalid parameter"); + return TSDB_CODE_TSC_APP_ERROR; + } + + if (!pStmt->isInsert || !pStmt->multiTbInsert || !pStmt->mtb.nameSet) { + tscError("not or invalid batch insert"); + return TSDB_CODE_TSC_APP_ERROR; + } + + return insertStmtBindParamBatch(pStmt, bind, colIdx); +} + + int taos_stmt_add_batch(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; diff --git a/src/inc/taos.h b/src/inc/taos.h index ca87337800..788502b45a 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -119,6 +119,7 @@ int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind); +int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int colIdx); int taos_stmt_add_batch(TAOS_STMT *stmt); int taos_stmt_execute(TAOS_STMT *stmt); TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); From a1fe6660fbde06bede72f55ba2e10f3fdbf04d7e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 8 May 2021 22:12:44 +0800 Subject: [PATCH 05/27] [td-4030] --- .../jni/com_taosdata_jdbc_TSDBJNIConnector.h | 40 ++++- src/client/src/TSDBJNIConnector.c | 138 +++++++++++++- .../com/taosdata/jdbc/TSDBJNIConnector.java | 58 +++++- .../taosdata/jdbc/TSDBPreparedStatement.java | 169 ++++++++++++++++-- .../taosdata/jdbc/TSDBResultSetBlockData.java | 56 +----- .../jdbc/rs/RestfulPreparedStatement.java | 11 -- src/query/src/qTokenizer.c | 6 +- src/util/inc/tstoken.h | 2 +- 8 files changed, 393 insertions(+), 87 deletions(-) diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index b3060e2c82..bcd0f63818 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -100,7 +100,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: isUpdateQueryImp - * Signature: (J)J + * Signature: (JJ)I */ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp (JNIEnv *env, jobject jobj, jlong con, jlong tres); @@ -185,6 +185,44 @@ JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTableSqlImp (JNIEnv *, jobject, jlong, jbyteArray); +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: prepareStmtImp + * Signature: ([BJ)I + */ +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp + (JNIEnv *, jobject, jbyteArray, jlong); + +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: setBindTableNameImp + * Signature: (JLjava/lang/String;J)I + */ +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameImp + (JNIEnv *, jobject, jlong, jstring, jlong); + +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: bindColDataImp + * Signature: (J[BIIIJ)J + */ +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp +(JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jint, jlong); + +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: executeBatchImp + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(JNIEnv *env, jobject jobj, jlong stmt, jlong con); + +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: executeBatchImp + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv *env, jobject jobj, jlong stmt, jlong con); + #ifdef __cplusplus } #endif diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 7447e36ac9..c28766b860 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -687,4 +687,140 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TDDBJNIConnector_getResultTimePrec } return taos_result_precision(result); -} \ No newline at end of file +} + +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp(JNIEnv *env, jobject jobj, jbyteArray jsql, jlong con) { + TAOS *tscon = (TAOS *)con; + if (tscon == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + + if (jsql == NULL) { + jniError("jobj:%p, conn:%p, empty sql string", jobj, tscon); + return JNI_SQL_NULL; + } + + jsize len = (*env)->GetArrayLength(env, jsql); + + char *str = (char *) calloc(1, sizeof(char) * (len + 1)); + if (str == NULL) { + jniError("jobj:%p, conn:%p, alloc memory failed", jobj, tscon); + return JNI_OUT_OF_MEMORY; + } + + (*env)->GetByteArrayRegion(env, jsql, 0, len, (jbyte *)str); + if ((*env)->ExceptionCheck(env)) { + // todo handle error + } + + TAOS_STMT* pStmt = taos_stmt_init(tscon); + int32_t code = taos_stmt_prepare(pStmt, str, len); + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + return JNI_TDENGINE_ERROR; + } + + free(str); + return (jlong) pStmt; +} + +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameImp(JNIEnv *env, jobject jobj, jlong stmt, jstring jname, jlong conn) { + TAOS *tsconn = (TAOS *)conn; + if (tsconn == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + + TAOS_STMT* pStmt = (TAOS_STMT*) stmt; + if (pStmt == NULL) { + jniError("jobj:%p, conn:%p, invalid stmt handle", jobj, tsconn); + return JNI_SQL_NULL; + } + + const char *name = (*env)->GetStringUTFChars(env, jname, NULL); + + int32_t code = taos_stmt_set_tbname((void*)stmt, name); + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); + return JNI_TDENGINE_ERROR; + } + + jniDebug("jobj:%p, conn:%p, set stmt bind table name", jobj, tsconn); + + (*env)->ReleaseStringUTFChars(env, jname, name); + return JNI_SUCCESS; +} + +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp(JNIEnv *env, jobject jobj, jlong stmt, + jbyteArray data, jint dataType, jint numOfRows, jint colIndex, jlong con) { + TAOS *tscon = (TAOS *)con; + if (tscon == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + + TAOS_STMT* pStmt = (TAOS_STMT*) stmt; + if (pStmt == NULL) { + jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); + return JNI_SQL_NULL; + } + +#if 0 + TAOS_BIND* b = malloc(20); + b.num= jrows; + int32_t code = taos_stmt_bind_param_batch(stmt, b, colInex); + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + return JNI_TDENGINE_ERROR; + } +#endif + + return JNI_SUCCESS; +} + +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(JNIEnv *env, jobject jobj, jlong stmt, jlong con) { + TAOS *tscon = (TAOS *)con; + if (tscon == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + + TAOS_STMT *pStmt = (TAOS_STMT*) stmt; + if (pStmt == NULL) { + jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); + return JNI_SQL_NULL; + } + + int32_t code = taos_stmt_execute(pStmt); + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + return JNI_TDENGINE_ERROR; + } + + jniDebug("jobj:%p, conn:%p, batch execute", jobj, tscon); + return JNI_SUCCESS; +} + +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv *env, jobject jobj, jlong stmt, jlong con) { + TAOS *tscon = (TAOS *)con; + if (tscon == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + + TAOS_STMT *pStmt = (TAOS_STMT*) stmt; + if (pStmt == NULL) { + jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); + return JNI_SQL_NULL; + } + + int32_t code = taos_stmt_close(pStmt); + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + return JNI_TDENGINE_ERROR; + } + + jniDebug("jobj:%p, conn:%p, stmt closed", jobj, tscon); + return JNI_SUCCESS; +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 5e3ffffa4f..05e28578f1 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -18,6 +18,7 @@ package com.taosdata.jdbc; import com.taosdata.jdbc.utils.TaosInfo; +import java.nio.ByteBuffer; import java.sql.SQLException; import java.sql.SQLWarning; import java.util.List; @@ -75,7 +76,6 @@ public class TSDBJNIConnector { public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException { if (this.taos != TSDBConstants.JNI_NULL_POINTER) { -// this.closeConnectionImp(this.taos); closeConnection(); this.taos = TSDBConstants.JNI_NULL_POINTER; } @@ -97,12 +97,6 @@ public class TSDBJNIConnector { * @throws SQLException */ public long executeQuery(String sql) throws SQLException { - // close previous result set if the user forgets to invoke the - // free method to close previous result set. -// if (!this.isResultsetClosed) { -// freeResultSet(taosResultSetPointer); -// } - Long pSql = 0l; try { pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos); @@ -297,4 +291,54 @@ public class TSDBJNIConnector { } private native int validateCreateTableSqlImp(long connection, byte[] sqlBytes); + + public long prepareStmt(String sql) throws SQLException { + Long stmt = 0L; + try { + stmt = prepareStmtImp(sql, this.taos); + } catch (Exception e) { + e.printStackTrace(); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_ENCODING); + } + + if (stmt == TSDBConstants.JNI_CONNECTION_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + } + + if (stmt == TSDBConstants.JNI_SQL_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_SQL_NULL); + } + + if (stmt == TSDBConstants.JNI_OUT_OF_MEMORY) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_OUT_OF_MEMORY); + } + + return stmt; + } + + private native long prepareStmtImp(String sql, long con); + + public int setBindTableName(long stmt, String tableName) { + return setBindTableNameImp(stmt, tableName, this.taos); + } + + private native int setBindTableNameImp(long stmt, String name, long conn); + + public int bindColumnDataArray(long stmt, byte[] data, int type, int numOfRows, int columnIndex) { + return bindColDataImp(stmt, data, type, numOfRows, columnIndex, this.taos); + } + + private native int bindColDataImp(long stmt, byte[] data, int type, int numOfRows, int columnIndex, long conn); + + public int executeBatch(long stmt) { + return executeBatchImp(stmt, this.taos); + } + + private native int executeBatchImp(long stmt, long con); + + public int closeBatch(long stmt) { + return closeStmt(stmt, this.taos); + } + + private native int closeStmt(long stmt, long con); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index 56f971a35e..be6c2361a1 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -14,37 +14,47 @@ *****************************************************************************/ package com.taosdata.jdbc; +import com.sun.tools.javac.util.Assert; import com.taosdata.jdbc.utils.Utils; import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.DoubleBuffer; +import java.nio.IntBuffer; import java.sql.*; import java.util.ArrayList; import java.util.Calendar; +import java.util.Collections; import java.util.regex.Matcher; import java.util.regex.Pattern; /* - * TDengine only supports a subset of the standard SQL, thus this implemetation of the + * TDengine only supports a subset of the standard SQL, thus this implementation of the * standard JDBC API contains more or less some adjustments customized for certain * compatibility needs. */ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStatement { - private String rawSql; private Object[] parameters; private boolean isPrepared; - + + private ArrayList colData; + private int type; + + private String tableName; + private long nativeStmtPtr = 0; + private volatile TSDBParameterMetaData parameterMetaData; - TSDBPreparedStatement(TSDBConnection connection, TSDBJNIConnector connecter, String sql) { - super(connection, connecter); + TSDBPreparedStatement(TSDBConnection connection, TSDBJNIConnector connector, String sql) { + super(connection, connector); init(sql); + int parameterCnt = 0; if (sql.contains("?")) { - int parameterCnt = 0; for (int i = 0; i < sql.length(); i++) { if ('?' == sql.charAt(i)) { parameterCnt++; @@ -53,6 +63,9 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat parameters = new Object[parameterCnt]; this.isPrepared = true; } + + this.colData = new ArrayList(parameterCnt); + this.colData.addAll(Collections.nCopies(parameterCnt, null)); } private void init(String sql) { @@ -260,10 +273,14 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public void setObject(int parameterIndex, Object x) throws SQLException { - if (isClosed()) + if (isClosed()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - if (parameterIndex < 1 && parameterIndex >= parameters.length) + } + + if (parameterIndex < 1 && parameterIndex >= parameters.length) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + } + parameters[parameterIndex - 1] = x; } @@ -300,9 +317,10 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public void setRef(int parameterIndex, Ref x) throws SQLException { - if (isClosed()) + if (isClosed()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @@ -515,4 +533,135 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } + + /////////////////////////////////////////////////////////////////////// + // NOTE: the following APIs are not JDBC compatible + // set the bind table name + private static class ColumnInfo { + @SuppressWarnings("rawtypes") + private ArrayList data; + private int type; + private boolean typeIsSet; + + public void ClumnInfo() { + this.typeIsSet = false; + } + + public void setType(int type) { + Assert.check(!this.typeIsSet); + this.typeIsSet = true; + this.type = type; + } + + public boolean isTypeSet() { + return this.typeIsSet; + } + }; + + public void setTableName(String name) { + this.tableName = name; + } + + @SuppressWarnings("unchecked") + public void setInt(int columnIndex, ArrayList list) throws SQLException { + ColumnInfo col = (ColumnInfo) this.colData.get(columnIndex); + if (col == null) { + ColumnInfo p = new ColumnInfo(); + p.setType(TSDBConstants.TSDB_DATA_TYPE_INT); + p.data = (ArrayList) list.clone(); + this.colData.set(columnIndex, p); + } else { + if (col.type != TSDBConstants.TSDB_DATA_TYPE_INT) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + } + + col.data.addAll(list); + } + } + + @SuppressWarnings("unchecked") + public void setFloat(int columnIndex, ArrayList list) throws SQLException { + ColumnInfo col = (ColumnInfo) this.colData.get(columnIndex); + if (col == null) { + ColumnInfo p = new ColumnInfo(); + p.setType(TSDBConstants.TSDB_DATA_TYPE_INT); + p.data = (ArrayList) list.clone(); + this.colData.set(columnIndex, p); + } else { + if (col.type != TSDBConstants.TSDB_DATA_TYPE_INT) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + } + + col.data.addAll(list); + } + } + + public void addColumnDataBatch() { + // do nothing + } + + public void columnDataExecuteBatch() { + int size = this.colData.size(); + ColumnInfo col = (ColumnInfo) this.colData.get(0); + int rows = col.data.size(); + + // pass the data block to native code + TSDBJNIConnector conn = null; + try { + conn = (TSDBJNIConnector) this.getConnection(); + this.nativeStmtPtr = conn.prepareStmt(rawSql); + conn.setBindTableName(this.nativeStmtPtr, this.tableName); + } catch (SQLException e) { + e.printStackTrace(); + } + + for (int i = 0; i < size; ++i) { + ColumnInfo col1 = this.colData.get(i); + Assert.check(col.isTypeSet()); + ByteBuffer ib = ByteBuffer.allocate(rows); + + switch (col1.type) { + case TSDBConstants.TSDB_DATA_TYPE_INT: { + for (int j = 0; j < rows; ++j) { + Integer val = (Integer) col.data.get(j); + if (val == null) { + ib.putInt(Integer.MIN_VALUE); + } else { + ib.putInt((int) col.data.get(j)); + } + } + + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { + for (int j = 0; j < rows; ++j) { + ib.putLong((long) col.data.get(j)); + } + break; + } + }; + + conn.bindColumnDataArray(this.nativeStmtPtr, ib.array(), col1.type, rows, i); + } + + conn.executeBatch(this.nativeStmtPtr); + } + + public void columnDataClearBatchClear() { + // TODO clear data in this.colData + } + + public void close() { + TSDBJNIConnector conn = null; + try { + conn = (TSDBJNIConnector) this.getConnection(); + this.nativeStmtPtr = conn.prepareStmt(rawSql); + conn.setBindTableName(this.nativeStmtPtr, this.tableName); + } catch (SQLException e) { + e.printStackTrace(); + } + + conn.closeBatch(this.nativeStmtPtr); + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java index ce5290de66..814fd6c18d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java @@ -29,6 +29,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import com.taosdata.jdbc.utils.NullType; + public class TSDBResultSetBlockData { private int numOfRows = 0; private int rowIndex = 0; @@ -164,59 +166,7 @@ public class TSDBResultSetBlockData { } } - private static class NullType { - private static final byte NULL_BOOL_VAL = 0x2; - private static final String NULL_STR = "null"; - - public String toString() { - return NullType.NULL_STR; - } - - public static boolean isBooleanNull(byte val) { - return val == NullType.NULL_BOOL_VAL; - } - - private static boolean isTinyIntNull(byte val) { - return val == Byte.MIN_VALUE; - } - - private static boolean isSmallIntNull(short val) { - return val == Short.MIN_VALUE; - } - - private static boolean isIntNull(int val) { - return val == Integer.MIN_VALUE; - } - - private static boolean isBigIntNull(long val) { - return val == Long.MIN_VALUE; - } - - private static boolean isFloatNull(float val) { - return Float.isNaN(val); - } - - private static boolean isDoubleNull(double val) { - return Double.isNaN(val); - } - - private static boolean isBinaryNull(byte[] val, int length) { - if (length != Byte.BYTES) { - return false; - } - - return val[0] == 0xFF; - } - - private static boolean isNcharNull(byte[] val, int length) { - if (length != Integer.BYTES) { - return false; - } - - return (val[0] & val[1] & val[2] & val[3]) == 0xFF; - } - - } + /** * The original type may not be a string type, but will be converted to by diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java index f846a1162e..f58e3f8cd2 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java @@ -1,26 +1,15 @@ package com.taosdata.jdbc.rs; -import com.google.common.collect.Range; -import com.google.common.collect.RangeSet; -import com.google.common.collect.TreeRangeSet; import com.taosdata.jdbc.TSDBError; import com.taosdata.jdbc.TSDBErrorNumbers; -import com.taosdata.jdbc.utils.SqlSyntaxValidator; import com.taosdata.jdbc.utils.Utils; import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; import java.net.URL; -import java.nio.charset.Charset; import java.sql.*; import java.util.Calendar; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.IntStream; public class RestfulPreparedStatement extends RestfulStatement implements PreparedStatement { diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 00fcaf82f5..4a4897f5c2 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -560,9 +560,9 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { return 0; } -SStrToken tscReplaceStrToken(char **str, SStrToken *token, const char* new) { +SStrToken tscReplaceStrToken(char **str, SStrToken *token, const char* newToken) { char *src = *str; - int32_t nsize = strlen(new); + int32_t nsize = strlen(newToken); int32_t size = strlen(*str) - token->n + nsize + 1; int32_t bsize = (uint64_t)token->z - (uint64_t)src; SStrToken ntoken; @@ -570,7 +570,7 @@ SStrToken tscReplaceStrToken(char **str, SStrToken *token, const char* new) { *str = calloc(1, size); strncpy(*str, src, bsize); - strcat(*str, new); + strcat(*str, newToken); strcat(*str, token->z + token->n); ntoken.n = nsize; diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index 93d48e01cb..550dbba06b 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -182,7 +182,7 @@ static FORCE_INLINE int32_t tGetNumericStringType(const SStrToken* pToken) { void taosCleanupKeywordsTable(); -SStrToken tscReplaceStrToken(char **str, SStrToken *token, const char* new); +SStrToken tscReplaceStrToken(char **str, SStrToken *token, const char* newToken); #ifdef __cplusplus From 22db3987a32450fbad50d1e85c8ba93f1eff7e1f Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sun, 9 May 2021 10:04:42 +0800 Subject: [PATCH 06/27] fix bug --- src/client/src/tscPrepare.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 9af7b11615..79701841df 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1360,6 +1360,7 @@ int taos_stmt_close(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; if (!pStmt->isInsert) { taosHashCleanup(pStmt->mtb.pTableHash); + taosHashCleanup(pStmt->mtb.pTableBlockHashList); SNormalStmt* normal = &pStmt->normal; if (normal->params != NULL) { From 7db07e73beb897092a76e5d317517e941f99ddea Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sun, 9 May 2021 13:30:37 +0800 Subject: [PATCH 07/27] add test file --- tests/script/api/batchprepare.c | 2131 +++++++++++++++++++++++++++++++ tests/script/api/makefile | 17 + 2 files changed, 2148 insertions(+) create mode 100644 tests/script/api/batchprepare.c create mode 100644 tests/script/api/makefile diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c new file mode 100644 index 0000000000..48c668634d --- /dev/null +++ b/tests/script/api/batchprepare.c @@ -0,0 +1,2131 @@ +// TAOS standard API example. The same syntax as MySQL, but only a subet +// to compile: gcc -o prepare prepare.c -ltaos + +#include +#include +#include +#include "taos.h" +#include + + +void taosMsleep(int mseconds); + +unsigned long long getCurrentTime(){ + struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) { + perror("Failed to get current time in ms"); + exit(EXIT_FAILURE); + } + + return (uint64_t)tv.tv_sec * 1000000ULL + (uint64_t)tv.tv_usec; +} + + + +int stmt_func1(TAOS_STMT *stmt) { + struct { + int64_t ts; + int8_t b; + int8_t v1; + int16_t v2; + int32_t v4; + int64_t v8; + float f4; + double f8; + char bin[40]; + char blob[80]; + } v = {0}; + + TAOS_BIND params[10]; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts); + params[0].buffer = &v.ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b); + params[1].buffer = &v.b; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1); + params[2].buffer = &v.v1; + params[2].length = ¶ms[2].buffer_length; + params[2].is_null = NULL; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2); + params[3].buffer = &v.v2; + params[3].length = ¶ms[3].buffer_length; + params[3].is_null = NULL; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4); + params[4].buffer = &v.v4; + params[4].length = ¶ms[4].buffer_length; + params[4].is_null = NULL; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8); + params[5].buffer = &v.v8; + params[5].length = ¶ms[5].buffer_length; + params[5].is_null = NULL; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4); + params[6].buffer = &v.f4; + params[6].length = ¶ms[6].buffer_length; + params[6].is_null = NULL; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8); + params[7].buffer = &v.f8; + params[7].length = ¶ms[7].buffer_length; + params[7].is_null = NULL; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin); + params[8].buffer = v.bin; + params[8].length = ¶ms[8].buffer_length; + params[8].is_null = NULL; + + params[9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[9].buffer_length = sizeof(v.bin); + params[9].buffer = v.bin; + params[9].length = ¶ms[9].buffer_length; + params[9].is_null = NULL; + + int is_null = 1; + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + for (int zz = 0; zz < 10; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + v.ts = 1591060628000 + zz * 10; + for (int i = 0; i < 10; ++i) { + v.ts += 1; + for (int j = 1; j < 10; ++j) { + params[j].is_null = ((i == j) ? &is_null : 0); + } + v.b = (int8_t)(i+zz*10) % 2; + v.v1 = (int8_t)(i+zz*10); + v.v2 = (int16_t)((i+zz*10) * 2); + v.v4 = (int32_t)((i+zz*10) * 4); + v.v8 = (int64_t)((i+zz*10) * 8); + v.f4 = (float)((i+zz*10) * 40); + v.f8 = (double)((i+zz*10) * 80); + for (int j = 0; j < sizeof(v.bin) - 1; ++j) { + v.bin[j] = (char)((i+zz)%10 + '0'); + } + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + } + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + return 0; +} + + +int stmt_func2(TAOS_STMT *stmt) { + struct { + int64_t ts; + int8_t b; + int8_t v1; + int16_t v2; + int32_t v4; + int64_t v8; + float f4; + double f8; + char bin[40]; + char blob[80]; + } v = {0}; + + TAOS_BIND params[10]; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts); + params[0].buffer = &v.ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b); + params[1].buffer = &v.b; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1); + params[2].buffer = &v.v1; + params[2].length = ¶ms[2].buffer_length; + params[2].is_null = NULL; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2); + params[3].buffer = &v.v2; + params[3].length = ¶ms[3].buffer_length; + params[3].is_null = NULL; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4); + params[4].buffer = &v.v4; + params[4].length = ¶ms[4].buffer_length; + params[4].is_null = NULL; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8); + params[5].buffer = &v.v8; + params[5].length = ¶ms[5].buffer_length; + params[5].is_null = NULL; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4); + params[6].buffer = &v.f4; + params[6].length = ¶ms[6].buffer_length; + params[6].is_null = NULL; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8); + params[7].buffer = &v.f8; + params[7].length = ¶ms[7].buffer_length; + params[7].is_null = NULL; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin); + params[8].buffer = v.bin; + params[8].length = ¶ms[8].buffer_length; + params[8].is_null = NULL; + + params[9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[9].buffer_length = sizeof(v.bin); + params[9].buffer = v.bin; + params[9].length = ¶ms[9].buffer_length; + params[9].is_null = NULL; + + int is_null = 1; + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + for (int l = 0; l < 100; l++) { + for (int zz = 0; zz < 10; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + v.ts = 1591060628000 + zz * 100 * l; + for (int i = 0; i < zz; ++i) { + v.ts += 1; + for (int j = 1; j < 10; ++j) { + params[j].is_null = ((i == j) ? &is_null : 0); + } + v.b = (int8_t)(i+zz*10) % 2; + v.v1 = (int8_t)(i+zz*10); + v.v2 = (int16_t)((i+zz*10) * 2); + v.v4 = (int32_t)((i+zz*10) * 4); + v.v8 = (int64_t)((i+zz*10) * 8); + v.f4 = (float)((i+zz*10) * 40); + v.f8 = (double)((i+zz*10) * 80); + for (int j = 0; j < sizeof(v.bin) - 1; ++j) { + v.bin[j] = (char)((i+zz)%10 + '0'); + } + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + } + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + } + + + return 0; +} + + + + +int stmt_func3(TAOS_STMT *stmt) { + struct { + int64_t ts; + int8_t b; + int8_t v1; + int16_t v2; + int32_t v4; + int64_t v8; + float f4; + double f8; + char bin[40]; + char blob[80]; + } v = {0}; + + TAOS_BIND params[10]; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts); + params[0].buffer = &v.ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b); + params[1].buffer = &v.b; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1); + params[2].buffer = &v.v1; + params[2].length = ¶ms[2].buffer_length; + params[2].is_null = NULL; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2); + params[3].buffer = &v.v2; + params[3].length = ¶ms[3].buffer_length; + params[3].is_null = NULL; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4); + params[4].buffer = &v.v4; + params[4].length = ¶ms[4].buffer_length; + params[4].is_null = NULL; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8); + params[5].buffer = &v.v8; + params[5].length = ¶ms[5].buffer_length; + params[5].is_null = NULL; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4); + params[6].buffer = &v.f4; + params[6].length = ¶ms[6].buffer_length; + params[6].is_null = NULL; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8); + params[7].buffer = &v.f8; + params[7].length = ¶ms[7].buffer_length; + params[7].is_null = NULL; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin); + params[8].buffer = v.bin; + params[8].length = ¶ms[8].buffer_length; + params[8].is_null = NULL; + + params[9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[9].buffer_length = sizeof(v.bin); + params[9].buffer = v.bin; + params[9].length = ¶ms[9].buffer_length; + params[9].is_null = NULL; + + int is_null = 1; + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + for (int l = 0; l < 100; l++) { + for (int zz = 0; zz < 10; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + v.ts = 1591060628000 + zz * 100 * l; + for (int i = 0; i < zz; ++i) { + v.ts += 1; + for (int j = 1; j < 10; ++j) { + params[j].is_null = ((i == j) ? &is_null : 0); + } + v.b = (int8_t)(i+zz*10) % 2; + v.v1 = (int8_t)(i+zz*10); + v.v2 = (int16_t)((i+zz*10) * 2); + v.v4 = (int32_t)((i+zz*10) * 4); + v.v8 = (int64_t)((i+zz*10) * 8); + v.f4 = (float)((i+zz*10) * 40); + v.f8 = (double)((i+zz*10) * 80); + for (int j = 0; j < sizeof(v.bin) - 1; ++j) { + v.bin[j] = (char)((i+zz)%10 + '0'); + } + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + } + } + } + + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + + return 0; +} + + +//300 tables 60 records +int stmt_funcb1(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[60]; + int8_t v1[60]; + int16_t v2[60]; + int32_t v4[60]; + int64_t v8[60]; + float f4[60]; + double f8[60]; + char bin[60][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 900000 * 60); + + uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + int* is_null = malloc(sizeof(int) * 60); + int* no_null = malloc(sizeof(int) * 60); + + for (int i = 0; i < 60; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 9000000; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[60*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 60; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 60; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 60; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 60; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 60; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 60; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 60; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 60; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 60; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 60; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 54000000; ++i) { + v.ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int l = 0; l < 3000; l++) { + for (int zz = 0; zz < 300; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + return 0; +} + + +//1table 18000 reocrds +int stmt_funcb2(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[18000]; + int8_t v1[18000]; + int16_t v2[18000]; + int32_t v4[18000]; + int64_t v8[18000]; + float f4[18000]; + double f8[18000]; + char bin[18000][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 900000 * 60); + + uintptr_t *lb = malloc(18000 * sizeof(uintptr_t)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + int* is_null = malloc(sizeof(int) * 18000); + int* no_null = malloc(sizeof(int) * 18000); + + for (int i = 0; i < 18000; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 30000; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[60*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 18000; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 18000; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 18000; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 18000; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 18000; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 18000; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 18000; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 18000; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 18000; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 18000; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 54000000; ++i) { + v.ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int l = 0; l < 10; l++) { + for (int zz = 0; zz < 300; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + ++id; + + } + + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + return 0; +} + + +//disorder +int stmt_funcb3(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[60]; + int8_t v1[60]; + int16_t v2[60]; + int32_t v4[60]; + int64_t v8[60]; + float f4[60]; + double f8[60]; + char bin[60][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 900000 * 60); + + uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + int* is_null = malloc(sizeof(int) * 60); + int* no_null = malloc(sizeof(int) * 60); + + for (int i = 0; i < 60; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 9000000; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[60*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 60; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 60; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 60; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 60; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 60; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 60; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 60; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 60; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 60; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 60; + + } + + int64_t tts = 1591060628000; + int64_t ttt = 0; + for (int i = 0; i < 54000000; ++i) { + v.ts[i] = tts + i; + if (i > 0 && i%60 == 0) { + ttt = v.ts[i-1]; + v.ts[i-1] = v.ts[i-60]; + v.ts[i-60] = ttt; + } + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int l = 0; l < 3000; l++) { + for (int zz = 0; zz < 300; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + return 0; +} + + + + +//samets +int stmt_funcb4(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[60]; + int8_t v1[60]; + int16_t v2[60]; + int32_t v4[60]; + int64_t v8[60]; + float f4[60]; + double f8[60]; + char bin[60][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 900000 * 60); + + uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + int* is_null = malloc(sizeof(int) * 60); + int* no_null = malloc(sizeof(int) * 60); + + for (int i = 0; i < 60; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 9000000; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[60*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 60; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 60; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 60; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 60; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 60; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 60; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 60; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 60; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 60; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 60; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 54000000; ++i) { + v.ts[i] = tts; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int l = 0; l < 3000; l++) { + for (int zz = 0; zz < 300; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + return 0; +} + + +//one table 60 records one time +int stmt_funcb_s1(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[60]; + int8_t v1[60]; + int16_t v2[60]; + int32_t v4[60]; + int64_t v8[60]; + float f4[60]; + double f8[60]; + char bin[60][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 900000 * 60); + + uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + int* is_null = malloc(sizeof(int) * 60); + int* no_null = malloc(sizeof(int) * 60); + + for (int i = 0; i < 60; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 9000000; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[60*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 60; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 60; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 60; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 60; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 60; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 60; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 60; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 60; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 60; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 60; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 54000000; ++i) { + v.ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int l = 0; l < 3000; l++) { + for (int zz = 0; zz < 300; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + } + + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + return 0; +} + + + + + + +//300 tables 60 records single column bind +int stmt_funcb_sc1(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[60]; + int8_t v1[60]; + int16_t v2[60]; + int32_t v4[60]; + int64_t v8[60]; + float f4[60]; + double f8[60]; + char bin[60][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 900000 * 60); + + uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + int* is_null = malloc(sizeof(int) * 60); + int* no_null = malloc(sizeof(int) * 60); + + for (int i = 0; i < 60; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 9000000; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[60*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 60; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 60; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 60; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 60; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 60; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 60; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 60; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 60; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 60; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 60; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 54000000; ++i) { + v.ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int l = 0; l < 3000; l++) { + for (int zz = 0; zz < 300; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + for (int col=0; col < 10; ++col) { + taos_stmt_bind_single_param_batch(stmt, params + id++, col); + } + + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + return 0; +} + + +//1 tables 60 records single column bind +int stmt_funcb_sc2(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[60]; + int8_t v1[60]; + int16_t v2[60]; + int32_t v4[60]; + int64_t v8[60]; + float f4[60]; + double f8[60]; + char bin[60][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 900000 * 60); + + uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + int* is_null = malloc(sizeof(int) * 60); + int* no_null = malloc(sizeof(int) * 60); + + for (int i = 0; i < 60; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 9000000; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[60*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 60; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 60; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 60; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 60; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 60; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 60; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 60; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 60; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 60; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 60; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 54000000; ++i) { + v.ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int l = 0; l < 3000; l++) { + for (int zz = 0; zz < 300; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + for (int col=0; col < 10; ++col) { + taos_stmt_bind_single_param_batch(stmt, params + id++, col); + } + + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + } + + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + return 0; +} + + +//10 tables [1...10] records single column bind +int stmt_funcb_sc3(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[60]; + int8_t v1[60]; + int16_t v2[60]; + int32_t v4[60]; + int64_t v8[60]; + float f4[60]; + double f8[60]; + char bin[60][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 60); + + uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); + int* is_null = malloc(sizeof(int) * 60); + int* no_null = malloc(sizeof(int) * 60); + + for (int i = 0; i < 60; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + int g = 0; + for (int i = 0; i < 600; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = g%10+1; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = g%10+1; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = g%10+1; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = g%10+1; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = g%10+1; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = g%10+1; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = g%10+1; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = g%10+1; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = g%10+1; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = g%10+1; + ++g; + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 60; ++i) { + v.ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int zz = 0; zz < 10; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + for (int col=0; col < 10; ++col) { + taos_stmt_bind_single_param_batch(stmt, params + id++, col); + } + + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + return 0; +} + + +void check_result(TAOS *taos, char *tname, int printr, int expected) { + char sql[255] = "SELECT * FROM "; + TAOS_RES *result; + + strcat(sql, tname); + + result = taos_query(taos, sql); + int code = taos_errno(result); + if (code != 0) { + printf("failed to query table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + + + TAOS_ROW row; + int rows = 0; + int num_fields = taos_num_fields(result); + TAOS_FIELD *fields = taos_fetch_fields(result); + char temp[256]; + + // fetch the records row by row + while ((row = taos_fetch_row(result))) { + rows++; + if (printr) { + memset(temp, 0, sizeof(temp)); + taos_print_row(temp, row, fields, num_fields); + printf("[%s]\n", temp); + } + } + + if (rows == expected) { + printf("%d rows are fetched as expectation\n", rows); + } else { + printf("!!!expect %d rows, but %d rows are fetched\n", expected, rows); + exit(1); + } + + taos_free_result(result); + +} + + + +//120table 60 record each table +int sql_perf1(TAOS *taos) { + char *sql[3000] = {0}; + TAOS_RES *result; + + for (int i = 0; i < 3000; i++) { + sql[i] = calloc(1, 1048576); + } + + int len = 0; + int tss = 0; + for (int l = 0; l < 3000; ++l) { + len = sprintf(sql[l], "insert into "); + for (int t = 0; t < 120; ++t) { + len += sprintf(sql[l] + len, "m%d values ", t); + for (int m = 0; m < 60; ++m) { + len += sprintf(sql[l] + len, "(%d, %d, %d, %d, %d, %d, %f, %f, \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\", \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\") ", tss++, m, m, m, m, m, m+1.0, m+1.0); + } + } + } + + + unsigned long long starttime = getCurrentTime(); + for (int i = 0; i < 3000; ++i) { + result = taos_query(taos, sql[i]); + int code = taos_errno(result); + if (code != 0) { + printf("failed to query table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + + taos_free_result(result); + } + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); + + return 0; +} + + + + + +//one table 60 records one time +int sql_perf_s1(TAOS *taos) { + char **sql = calloc(1, sizeof(char*) * 360000); + TAOS_RES *result; + + for (int i = 0; i < 360000; i++) { + sql[i] = calloc(1, 9000); + } + + int len = 0; + int tss = 0; + int id = 0; + for (int t = 0; t < 120; ++t) { + for (int l = 0; l < 3000; ++l) { + len = sprintf(sql[id], "insert into m%d values ", t); + for (int m = 0; m < 60; ++m) { + len += sprintf(sql[id] + len, "(%d, %d, %d, %d, %d, %d, %f, %f, \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\", \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\") ", tss++, m, m, m, m, m, m+1.0, m+1.0); + } + if (len >= 9000) { + printf("sql:%s,len:%d\n", sql[id], len); + exit(1); + } + ++id; + } + } + + + unsigned long long starttime = getCurrentTime(); + for (int i = 0; i < 360000; ++i) { + result = taos_query(taos, sql[i]); + int code = taos_errno(result); + if (code != 0) { + printf("failed to query table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + + taos_free_result(result); + } + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); + + return 0; +} + + +//small record size +int sql_s_perf1(TAOS *taos) { + char *sql[3000] = {0}; + TAOS_RES *result; + + for (int i = 0; i < 3000; i++) { + sql[i] = calloc(1, 1048576); + } + + int len = 0; + int tss = 0; + for (int l = 0; l < 3000; ++l) { + len = sprintf(sql[l], "insert into "); + for (int t = 0; t < 120; ++t) { + len += sprintf(sql[l] + len, "m%d values ", t); + for (int m = 0; m < 60; ++m) { + len += sprintf(sql[l] + len, "(%d, %d) ", tss++, m%2); + } + } + } + + + unsigned long long starttime = getCurrentTime(); + for (int i = 0; i < 3000; ++i) { + result = taos_query(taos, sql[i]); + int code = taos_errno(result); + if (code != 0) { + printf("failed to query table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + + taos_free_result(result); + } + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); + + return 0; +} + + +void prepare(TAOS *taos) { + TAOS_RES *result; + int code; + + result = taos_query(taos, "drop database demo"); + taos_free_result(result); + + result = taos_query(taos, "create database demo"); + code = taos_errno(result); + if (code != 0) { + printf("failed to create database, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + taos_free_result(result); + + result = taos_query(taos, "use demo"); + taos_free_result(result); + + // create table + for (int i = 0 ; i < 300; i++) { + char buf[1024]; + sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))", i) ; + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + taos_free_result(result); + } + +} + + +void runcase(TAOS *taos) { + TAOS_STMT *stmt; + + +#if 1 + prepare(taos); + + stmt = taos_stmt_init(taos); + + printf("10t+10records start\n"); + stmt_func1(stmt); + printf("10t+10records end\n"); + printf("check result start\n"); + check_result(taos, "m0", 1, 10); + check_result(taos, "m1", 1, 10); + check_result(taos, "m2", 1, 10); + check_result(taos, "m3", 1, 10); + check_result(taos, "m4", 1, 10); + check_result(taos, "m5", 1, 10); + check_result(taos, "m6", 1, 10); + check_result(taos, "m7", 1, 10); + check_result(taos, "m8", 1, 10); + check_result(taos, "m9", 1, 10); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + +#if 1 + prepare(taos); + + stmt = taos_stmt_init(taos); + + printf("10t+[0,1,2...9]records start\n"); + stmt_func2(stmt); + printf("10t+[0,1,2...9]records end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 0); + check_result(taos, "m1", 0, 100); + check_result(taos, "m2", 0, 200); + check_result(taos, "m3", 0, 300); + check_result(taos, "m4", 0, 400); + check_result(taos, "m5", 0, 500); + check_result(taos, "m6", 0, 600); + check_result(taos, "m7", 0, 700); + check_result(taos, "m8", 0, 800); + check_result(taos, "m9", 0, 900); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + +#if 1 + prepare(taos); + + stmt = taos_stmt_init(taos); + + printf("10t+[0,100,200...900]records start\n"); + stmt_func3(stmt); + printf("10t+[0,100,200...900]records end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 0); + check_result(taos, "m1", 0, 100); + check_result(taos, "m2", 0, 200); + check_result(taos, "m3", 0, 300); + check_result(taos, "m4", 0, 400); + check_result(taos, "m5", 0, 500); + check_result(taos, "m6", 0, 600); + check_result(taos, "m7", 0, 700); + check_result(taos, "m8", 0, 800); + check_result(taos, "m9", 0, 900); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + + +#if 1 + prepare(taos); + + stmt = taos_stmt_init(taos); + + printf("300t+60r+bm start\n"); + stmt_funcb1(stmt); + printf("300t+60r+bm end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 180000); + check_result(taos, "m1", 0, 180000); + check_result(taos, "m111", 0, 180000); + check_result(taos, "m223", 0, 180000); + check_result(taos, "m299", 0, 180000); + printf("check result end\n"); + taos_stmt_close(stmt); +#endif + + +#if 1 + prepare(taos); + + stmt = taos_stmt_init(taos); + + printf("1t+9000r+bm start\n"); + stmt_funcb2(stmt); + printf("1t+9000r+bm end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 180000); + check_result(taos, "m1", 0, 180000); + check_result(taos, "m111", 0, 180000); + check_result(taos, "m223", 0, 180000); + check_result(taos, "m299", 0, 180000); + printf("check result end\n"); + taos_stmt_close(stmt); + + //stmt_perf1(stmt); +#endif + +#if 1 + prepare(taos); + + stmt = taos_stmt_init(taos); + + printf("300t+60r+disorder+bm start\n"); + stmt_funcb3(stmt); + printf("300t+60r+disorder+bm end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 180000); + check_result(taos, "m1", 0, 180000); + check_result(taos, "m111", 0, 180000); + check_result(taos, "m223", 0, 180000); + check_result(taos, "m299", 0, 180000); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + + +#if 1 + prepare(taos); + + stmt = taos_stmt_init(taos); + + printf("300t+60r+samets+bm start\n"); + stmt_funcb4(stmt); + printf("300t+60r+samets+bm end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 1); + check_result(taos, "m1", 0, 1); + check_result(taos, "m111", 0, 1); + check_result(taos, "m223", 0, 1); + check_result(taos, "m299", 0, 1); + printf("check result end\n"); + taos_stmt_close(stmt); +#endif + + +#if 1 + prepare(taos); + + stmt = taos_stmt_init(taos); + + printf("300t+60r+bm+sc start\n"); + stmt_funcb_sc1(stmt); + printf("300t+60r+bm+sc end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 180000); + check_result(taos, "m1", 0, 180000); + check_result(taos, "m111", 0, 180000); + check_result(taos, "m223", 0, 180000); + check_result(taos, "m299", 0, 180000); + printf("check result end\n"); + taos_stmt_close(stmt); +#endif + +#if 1 + prepare(taos); + + stmt = taos_stmt_init(taos); + + printf("1t+60r+bm+sc start\n"); + stmt_funcb_sc2(stmt); + printf("1t+60r+bm+sc end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 180000); + check_result(taos, "m1", 0, 180000); + check_result(taos, "m111", 0, 180000); + check_result(taos, "m223", 0, 180000); + check_result(taos, "m299", 0, 180000); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + +#if 1 + prepare(taos); + + stmt = taos_stmt_init(taos); + + printf("10t+[1...10]r+bm+sc start\n"); + stmt_funcb_sc3(stmt); + printf("10t+[1...10]r+bm+sc end\n"); + printf("check result start\n"); + check_result(taos, "m0", 1, 1); + check_result(taos, "m1", 1, 2); + check_result(taos, "m2", 1, 3); + check_result(taos, "m3", 1, 4); + check_result(taos, "m4", 1, 5); + check_result(taos, "m5", 1, 6); + check_result(taos, "m6", 1, 7); + check_result(taos, "m7", 1, 8); + check_result(taos, "m8", 1, 9); + check_result(taos, "m9", 1, 10); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + + +#if 1 + prepare(taos); + + stmt = taos_stmt_init(taos); + + printf("1t+60r+bm start\n"); + stmt_funcb_s1(stmt); + printf("1t+60r+bm end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 180000); + check_result(taos, "m1", 0, 180000); + check_result(taos, "m111", 0, 180000); + check_result(taos, "m223", 0, 180000); + check_result(taos, "m299", 0, 180000); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + + +#if 1 + prepare(taos); + + (void)stmt; + printf("120t+60r+sql start\n"); + sql_perf1(taos); + printf("120t+60r+sql end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 180000); + check_result(taos, "m1", 0, 180000); + check_result(taos, "m34", 0, 180000); + check_result(taos, "m67", 0, 180000); + check_result(taos, "m99", 0, 180000); + printf("check result end\n"); +#endif + +#if 1 + prepare(taos); + + (void)stmt; + printf("1t+60r+sql start\n"); + sql_perf_s1(taos); + printf("1t+60r+sql end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 180000); + check_result(taos, "m1", 0, 180000); + check_result(taos, "m34", 0, 180000); + check_result(taos, "m67", 0, 180000); + check_result(taos, "m99", 0, 180000); + printf("check result end\n"); +#endif + + + +} + +int main(int argc, char *argv[]) +{ + TAOS *taos; + + // connect to server + if (argc < 2) { + printf("please input server ip \n"); + return 0; + } + + taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + exit(1); + } + + prepare(taos); + + runcase(taos); + + + return 0; +} + diff --git a/tests/script/api/makefile b/tests/script/api/makefile new file mode 100644 index 0000000000..c5bbde0f0b --- /dev/null +++ b/tests/script/api/makefile @@ -0,0 +1,17 @@ +# Copyright (c) 2017 by TAOS Technologies, Inc. +# todo: library dependency, header file dependency + +ROOT=./ +TARGET=exe +LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt +CFLAGS = -O0 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ + -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \ + -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 + +all: $(TARGET) + +exe: + gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) + +clean: + rm $(ROOT)batchprepare From b7cec3072beec73d16d1436c3c44feb8896f2217 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sun, 9 May 2021 16:45:02 +0800 Subject: [PATCH 08/27] fix --- src/client/src/tscPrepare.c | 87 ++++--- tests/script/api/batchprepare.c | 413 ++++++++++++++++++++++++++++++-- 2 files changed, 445 insertions(+), 55 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 79701841df..4d5fdc2287 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -148,7 +148,7 @@ static int normalStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { break; default: - tscDebug("param %d: type mismatch or invalid", i); + tscDebug("0x%"PRIx64" param %d: type mismatch or invalid", stmt->pSql->self, i); return TSDB_CODE_TSC_INVALID_VALUE; } } @@ -804,7 +804,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { if (pStmt->multiTbInsert) { if (pCmd->pTableBlockHashList == NULL) { - tscError("Table block hash list is empty"); + tscError("0x%"PRIx64" Table block hash list is empty", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } @@ -850,7 +850,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { int code = doBindParam(pBlock, data, param, &bind[param->idx], 1); if (code != TSDB_CODE_SUCCESS) { - tscDebug("param %d: type mismatch or invalid", param->idx); + tscDebug("0x%"PRIx64" param %d: type mismatch or invalid", pStmt->pSql->self, param->idx); return code; } } @@ -866,18 +866,34 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int c STableDataBlocks* pBlock = NULL; - if (pCmd->pTableBlockHashList == NULL) { - tscError("Table block hash list is empty"); - return TSDB_CODE_TSC_APP_ERROR; - } + if (pStmt->multiTbInsert) { + if (pCmd->pTableBlockHashList == NULL) { + tscError("0x%"PRIx64" Table block hash list is empty", pStmt->pSql->self); + return TSDB_CODE_TSC_APP_ERROR; + } - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); - if (t1 == NULL) { - tscError("no table data block in hash list, uid:%" PRId64 , pStmt->mtb.currentUid); - return TSDB_CODE_TSC_APP_ERROR; - } + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); + if (t1 == NULL) { + tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pStmt->pSql->self, pStmt->mtb.currentUid); + return TSDB_CODE_TSC_APP_ERROR; + } - pBlock = *t1; + pBlock = *t1; + } else { + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); + + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pCmd->pTableBlockHashList == NULL) { + pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + } + + int32_t ret = + tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); + if (ret != 0) { + return ret; + } + } assert(colIdx == -1 || (colIdx >= 0 && colIdx < pBlock->numOfParams)); @@ -898,13 +914,13 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int c for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { SParamInfo* param = &pBlock->params[j]; if (bind[param->idx].num != rowNum) { - tscError("param %d: num[%d:%d] not match", param->idx, rowNum, bind[param->idx].num); + tscError("0x%"PRIx64" param %d: num[%d:%d] not match", pStmt->pSql->self, param->idx, rowNum, bind[param->idx].num); return TSDB_CODE_TSC_INVALID_VALUE; } int code = doBindBatchParam(pBlock, param, &bind[param->idx], pCmd->batchSize); if (code != TSDB_CODE_SUCCESS) { - tscError("param %d: type mismatch or invalid", param->idx); + tscError("0x%"PRIx64" param %d: type mismatch or invalid", pStmt->pSql->self, param->idx); return code; } } @@ -915,7 +931,7 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int c int code = doBindBatchParam(pBlock, param, bind, pCmd->batchSize); if (code != TSDB_CODE_SUCCESS) { - tscError("param %d: type mismatch or invalid", param->idx); + tscError("0x%"PRIx64" param %d: type mismatch or invalid", pStmt->pSql->self, param->idx); return code; } @@ -940,7 +956,7 @@ static int insertStmtUpdateBatch(STscStmt* stmt) { STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&stmt->mtb.currentUid, sizeof(stmt->mtb.currentUid)); if (t1 == NULL) { - tscError("no table data block in hash list, uid:%" PRId64 , stmt->mtb.currentUid); + tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pSql->self, stmt->mtb.currentUid); return TSDB_CODE_TSC_APP_ERROR; } @@ -1110,7 +1126,7 @@ static int insertBatchStmtExecute(STscStmt* pStmt) { int32_t code = 0; if(pStmt->mtb.nameSet == false) { - tscError("no table name set"); + tscError("0x%"PRIx64" no table name set", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } @@ -1276,13 +1292,13 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { if (name == NULL) { terrno = TSDB_CODE_TSC_APP_ERROR; - tscError("name is NULL"); + tscError("0x%"PRIx64" name is NULL", pSql->self); return TSDB_CODE_TSC_APP_ERROR; } if (pStmt->multiTbInsert == false || !tscIsInsertData(pSql->sqlstr)) { terrno = TSDB_CODE_TSC_APP_ERROR; - tscError("not multi table insert"); + tscError("0x%"PRIx64" not multi table insert", pSql->self); return TSDB_CODE_TSC_APP_ERROR; } @@ -1292,7 +1308,7 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pStmt->mtb.pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); if (t1 == NULL) { - tscError("no table data block in hash list, uid:%" PRId64 , pStmt->mtb.currentUid); + tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pSql->self, pStmt->mtb.currentUid); return TSDB_CODE_TSC_APP_ERROR; } @@ -1301,7 +1317,7 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { taosHashPut(pCmd->pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)t1, POINTER_BYTES); - tscDebug("table:%s is already prepared, uid:%" PRIu64, name, pStmt->mtb.currentUid); + tscDebug("0x%"PRIx64" table:%s is already prepared, uid:%" PRIu64, pSql->self, name, pStmt->mtb.currentUid); return TSDB_CODE_SUCCESS; } @@ -1350,7 +1366,7 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); - tscDebug("table:%s is prepared, uid:%" PRIu64, name, pStmt->mtb.currentUid); + tscDebug("0x%"PRIx64" table:%s is prepared, uid:%" PRIu64, pSql->self, name, pStmt->mtb.currentUid); } return code; @@ -1396,12 +1412,17 @@ int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { STscStmt* pStmt = (STscStmt*)stmt; if (bind == NULL || bind->num <= 0) { - tscError("invalid parameter"); + tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } - if (!pStmt->isInsert || !pStmt->multiTbInsert || !pStmt->mtb.nameSet) { - tscError("not or invalid batch insert"); + if (!pStmt->isInsert) { + tscError("0x%"PRIx64" not or invalid batch insert", pStmt->pSql->self); + return TSDB_CODE_TSC_APP_ERROR; + } + + if (pStmt->multiTbInsert && !pStmt->mtb.nameSet) { + tscError("0x%"PRIx64" no table name set", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } @@ -1411,14 +1432,20 @@ int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int colIdx) { STscStmt* pStmt = (STscStmt*)stmt; if (bind == NULL || bind->num <= 0) { - tscError("invalid parameter"); + tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } - if (!pStmt->isInsert || !pStmt->multiTbInsert || !pStmt->mtb.nameSet) { - tscError("not or invalid batch insert"); + if (!pStmt->isInsert) { + tscError("0x%"PRIx64" not or invalid batch insert", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } + + if (pStmt->multiTbInsert && !pStmt->mtb.nameSet) { + tscError("0x%"PRIx64" no table name set", pStmt->pSql->self); + return TSDB_CODE_TSC_APP_ERROR; + } + return insertStmtBindParamBatch(pStmt, bind, colIdx); } @@ -1544,7 +1571,7 @@ int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes) { } if (idx<0 || idx>=pBlock->numOfParams) { - tscError("param %d: out of range", idx); + tscError("0x%"PRIx64" param %d: out of range", pStmt->pSql->self, idx); abort(); } diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 48c668634d..9b204c2f4a 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -6,7 +6,13 @@ #include #include "taos.h" #include +#include +#include +typedef struct { + TAOS *taos; + int idx; +}T_par; void taosMsleep(int mseconds); @@ -585,7 +591,7 @@ int stmt_funcb2(TAOS_STMT *stmt) { for (int i = 0; i < 30000; i+=10) { params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; + params[i+0].buffer = &v.ts[18000*i/10]; params[i+0].length = NULL; params[i+0].is_null = no_null; params[i+0].num = 18000; @@ -1008,6 +1014,232 @@ int stmt_funcb4(TAOS_STMT *stmt) { } + + +//1table 18000 reocrds +int stmt_funcb5(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[18000]; + int8_t v1[18000]; + int16_t v2[18000]; + int32_t v4[18000]; + int64_t v8[18000]; + float f4[18000]; + double f8[18000]; + char bin[18000][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 900000 * 60); + + uintptr_t *lb = malloc(18000 * sizeof(uintptr_t)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + int* is_null = malloc(sizeof(int) * 18000); + int* no_null = malloc(sizeof(int) * 18000); + + for (int i = 0; i < 18000; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 30000; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[18000*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 18000; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 18000; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 18000; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 18000; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 18000; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 18000; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 18000; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 18000; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 18000; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 18000; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 54000000; ++i) { + v.ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into m0 values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int l = 0; l < 10; l++) { + for (int zz = 0; zz < 1; zz++) { + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + ++id; + + } + + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + return 0; +} + + +//1table 200000 reocrds +int stmt_funcb_ssz1(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int b[30000]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 30000 * 3000); + + uintptr_t *lb = malloc(30000 * sizeof(uintptr_t)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + int* no_null = malloc(sizeof(int) * 200000); + + for (int i = 0; i < 30000; ++i) { + lb[i] = 40; + no_null[i] = 0; + v.b[i] = (int8_t)(i % 2); + } + + for (int i = 0; i < 30000; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[30000*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 30000; + + params[i+1].buffer_type = TSDB_DATA_TYPE_INT; + params[i+1].buffer_length = sizeof(int); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = no_null; + params[i+1].num = 30000; + } + + int64_t tts = 0; + for (int64_t i = 0; i < 90000000LL; ++i) { + v.ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? values(?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int l = 0; l < 10; l++) { + for (int zz = 0; zz < 300; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + ++id; + + } + + } + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + return 0; +} + + //one table 60 records one time int stmt_funcb_s1(TAOS_STMT *stmt) { struct { @@ -1796,7 +2028,7 @@ int sql_s_perf1(TAOS *taos) { } -void prepare(TAOS *taos) { +void prepare(TAOS *taos, int bigsize) { TAOS_RES *result; int code; @@ -1818,7 +2050,11 @@ void prepare(TAOS *taos) { // create table for (int i = 0 ; i < 300; i++) { char buf[1024]; - sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))", i) ; + if (bigsize) { + sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))", i) ; + } else { + sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; + } result = taos_query(taos, buf); code = taos_errno(result); if (code != 0) { @@ -1832,12 +2068,68 @@ void prepare(TAOS *taos) { } -void runcase(TAOS *taos) { + +void preparem(TAOS *taos, int bigsize, int idx) { + TAOS_RES *result; + int code; + char dbname[32],sql[255]; + + sprintf(dbname, "demo%d", idx); + sprintf(sql, "drop database %s", dbname); + + + result = taos_query(taos, sql); + taos_free_result(result); + + sprintf(sql, "create database %s", dbname); + result = taos_query(taos, sql); + code = taos_errno(result); + if (code != 0) { + printf("failed to create database, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + taos_free_result(result); + + sprintf(sql, "use %s", dbname); + result = taos_query(taos, sql); + taos_free_result(result); + + // create table + for (int i = 0 ; i < 300; i++) { + char buf[1024]; + if (bigsize) { + sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))", i) ; + } else { + sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; + } + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + taos_free_result(result); + } + +} + + + +//void runcase(TAOS *taos, int idx) { +void* runcase(void *par) { + T_par* tpar = (T_par *)par; + TAOS *taos = tpar->taos; + int idx = tpar->idx; + TAOS_STMT *stmt; + (void)idx; + #if 1 - prepare(taos); + prepare(taos, 1); stmt = taos_stmt_init(taos); @@ -1861,7 +2153,7 @@ void runcase(TAOS *taos) { #endif #if 1 - prepare(taos); + prepare(taos, 1); stmt = taos_stmt_init(taos); @@ -1885,7 +2177,7 @@ void runcase(TAOS *taos) { #endif #if 1 - prepare(taos); + prepare(taos, 1); stmt = taos_stmt_init(taos); @@ -1910,7 +2202,7 @@ void runcase(TAOS *taos) { #if 1 - prepare(taos); + prepare(taos, 1); stmt = taos_stmt_init(taos); @@ -1928,14 +2220,14 @@ void runcase(TAOS *taos) { #endif -#if 1 - prepare(taos); +#if 1 + prepare(taos, 1); stmt = taos_stmt_init(taos); - printf("1t+9000r+bm start\n"); + printf("1t+18000r+bm start\n"); stmt_funcb2(stmt); - printf("1t+9000r+bm end\n"); + printf("1t+18000r+bm end\n"); printf("check result start\n"); check_result(taos, "m0", 0, 180000); check_result(taos, "m1", 0, 180000); @@ -1945,11 +2237,10 @@ void runcase(TAOS *taos) { printf("check result end\n"); taos_stmt_close(stmt); - //stmt_perf1(stmt); #endif #if 1 - prepare(taos); + prepare(taos, 1); stmt = taos_stmt_init(taos); @@ -1969,7 +2260,7 @@ void runcase(TAOS *taos) { #if 1 - prepare(taos); + prepare(taos, 1); stmt = taos_stmt_init(taos); @@ -1986,9 +2277,24 @@ void runcase(TAOS *taos) { taos_stmt_close(stmt); #endif +#if 1 + prepare(taos, 1); + + stmt = taos_stmt_init(taos); + + printf("1t+18000r+nodyntable+bm start\n"); + stmt_funcb5(stmt); + printf("1t+18000r+nodyntable+bm end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 180000); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + #if 1 - prepare(taos); + prepare(taos, 1); stmt = taos_stmt_init(taos); @@ -2006,7 +2312,7 @@ void runcase(TAOS *taos) { #endif #if 1 - prepare(taos); + prepare(taos, 1); stmt = taos_stmt_init(taos); @@ -2025,7 +2331,7 @@ void runcase(TAOS *taos) { #endif #if 1 - prepare(taos); + prepare(taos, 1); stmt = taos_stmt_init(taos); @@ -2050,7 +2356,7 @@ void runcase(TAOS *taos) { #if 1 - prepare(taos); + prepare(taos, 1); stmt = taos_stmt_init(taos); @@ -2070,7 +2376,7 @@ void runcase(TAOS *taos) { #if 1 - prepare(taos); + prepare(taos, 1); (void)stmt; printf("120t+60r+sql start\n"); @@ -2086,7 +2392,7 @@ void runcase(TAOS *taos) { #endif #if 1 - prepare(taos); + prepare(taos, 1); (void)stmt; printf("1t+60r+sql start\n"); @@ -2102,12 +2408,32 @@ void runcase(TAOS *taos) { #endif +#if 1 + preparem(taos, 0, idx); + + stmt = taos_stmt_init(taos); + + printf("1t+30000r+bm start\n"); + stmt_funcb_ssz1(stmt); + printf("1t+30000r+bm end\n"); + printf("check result start\n"); + check_result(taos, "m0", 0, 300000); + check_result(taos, "m1", 0, 300000); + check_result(taos, "m111", 0, 300000); + check_result(taos, "m223", 0, 300000); + check_result(taos, "m299", 0, 300000); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + + return NULL; } int main(int argc, char *argv[]) { - TAOS *taos; + TAOS *taos[4]; // connect to server if (argc < 2) { @@ -2115,17 +2441,54 @@ int main(int argc, char *argv[]) return 0; } - taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); + taos[0] = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) { printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); exit(1); } - prepare(taos); + taos[1] = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + exit(1); + } - runcase(taos); + taos[2] = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + exit(1); + } + taos[3] = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + exit(1); + } + + pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), 4); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + T_par par[4]; + + par[0].taos = taos[0]; + par[0].idx = 0; + par[1].taos = taos[1]; + par[1].idx = 1; + par[2].taos = taos[2]; + par[2].idx = 2; + par[3].taos = taos[3]; + par[3].idx = 3; + pthread_create(&(pThreadList[0]), &thattr, runcase, (void *)&par[0]); + //pthread_create(&(pThreadList[1]), &thattr, runcase, (void *)&par[1]); + //pthread_create(&(pThreadList[2]), &thattr, runcase, (void *)&par[2]); + //pthread_create(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); + + while(1) { + sleep(1); + } return 0; } From 511cdbeae0996869a899df8061bfcbd8b5b530f5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 9 May 2021 16:48:51 +0800 Subject: [PATCH 09/27] [td-4038] --- .../jni/com_taosdata_jdbc_TSDBJNIConnector.h | 4 +- src/client/src/TSDBJNIConnector.c | 50 ++++- src/client/src/tscPrepare.c | 5 +- src/client/src/tscUtil.c | 2 +- .../com/taosdata/jdbc/TSDBConnection.java | 14 +- .../com/taosdata/jdbc/TSDBJNIConnector.java | 44 ++--- .../taosdata/jdbc/TSDBPreparedStatement.java | 173 ++++++++++++------ .../java/com/taosdata/jdbc/TSDBStatement.java | 66 +++---- src/inc/taos.h | 9 +- 9 files changed, 223 insertions(+), 144 deletions(-) diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index bcd0f63818..07fd46f859 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -204,10 +204,10 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameI /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: bindColDataImp - * Signature: (J[BIIIJ)J + * Signature: (J[B[BIIIIJ)J */ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp -(JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jint, jlong); +(JNIEnv *, jobject, jlong, jbyteArray, jbyteArray, jint, jint, jint, jint, jlong); /* * Class: com_taosdata_jdbc_TSDBJNIConnector diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index c28766b860..8ffd021e51 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -746,14 +746,14 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameI return JNI_TDENGINE_ERROR; } - jniDebug("jobj:%p, conn:%p, set stmt bind table name", jobj, tsconn); + jniDebug("jobj:%p, conn:%p, set stmt bind table name:%s", jobj, tsconn, name); (*env)->ReleaseStringUTFChars(env, jname, name); return JNI_SUCCESS; } JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp(JNIEnv *env, jobject jobj, jlong stmt, - jbyteArray data, jint dataType, jint numOfRows, jint colIndex, jlong con) { + jbyteArray data, jbyteArray length, jint dataType, jint dataBytes, jint numOfRows, jint colIndex, jlong con) { TAOS *tscon = (TAOS *)con; if (tscon == NULL) { jniError("jobj:%p, connection already closed", jobj); @@ -766,15 +766,50 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp(J return JNI_SQL_NULL; } -#if 0 - TAOS_BIND* b = malloc(20); - b.num= jrows; - int32_t code = taos_stmt_bind_param_batch(stmt, b, colInex); + // todo refactor + jsize len = (*env)->GetArrayLength(env, data); + char *colBuf = (char *)calloc(1, sizeof(char) * len); + (*env)->GetByteArrayRegion(env, data, 0, len, (jbyte *)colBuf); + if ((*env)->ExceptionCheck(env)) { + // todo handle error + } + + len = (*env)->GetArrayLength(env, length); + char *lengthArray = (char*) calloc(1, sizeof(char) * len); + (*env)->GetByteArrayRegion(env, length, 0, len, (jbyte*) lengthArray); + if ((*env)->ExceptionCheck(env)) { + } + + // bind multi-rows with only one invoke. + TAOS_MULTI_BIND* b = calloc(1, sizeof(TAOS_MULTI_BIND)); + + b->num = numOfRows; + b->buffer_type = dataType; // todo check data type + b->buffer_length = tDataTypes[dataType].bytes; + b->is_null = calloc(numOfRows, sizeof(int32_t)); + b->buffer = colBuf; + b->length = (uintptr_t*)lengthArray; + + // set the length and is_null array + switch(dataType) { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: { + int32_t bytes = tDataTypes[dataType].bytes; + for(int32_t i = 0; i < numOfRows; ++i) { + b->length[i] = bytes; + b->is_null[i] = isNull(colBuf + bytes * i, dataType); + } + } + } + + int32_t code = taos_stmt_bind_single_param_batch(pStmt, b, colIndex); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; } -#endif return JNI_SUCCESS; } @@ -792,6 +827,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(J return JNI_SQL_NULL; } + taos_stmt_add_batch(pStmt); int32_t code = taos_stmt_execute(pStmt); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 9af7b11615..d251867eaa 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1306,10 +1306,9 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { } pStmt->mtb.tbname = tscReplaceStrToken(&pSql->sqlstr, &pStmt->mtb.tbname, name); - pStmt->mtb.nameSet = true; - tscDebug("sqlstr set to %s", pSql->sqlstr); + tscDebug("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); pSql->cmd.parseFinished = 0; pSql->cmd.numOfParams = 0; @@ -1350,7 +1349,7 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); - tscDebug("table:%s is prepared, uid:%" PRIu64, name, pStmt->mtb.currentUid); + tscDebug("0x%"PRIx64" table:%s is prepared, uid:%" PRIx64, pSql->self, name, pStmt->mtb.currentUid); } return code; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7a6842869a..5f14236ded 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1319,7 +1319,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap) { pBlocks->numOfRows = 0; }else { - tscDebug("table %s data block is empty", pOneTableBlock->tableName.tname); + tscDebug("0x%"PRIx64" table %s data block is empty", pSql->self, pOneTableBlock->tableName.tname); } p = taosHashIterate(pCmd->pTableBlockHashList, p); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index c8ab9fb15a..02fee74eb5 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -49,7 +49,7 @@ public class TSDBConnection extends AbstractConnection { this.databaseMetaData.setConnection(this); } - public TSDBJNIConnector getConnection() { + public TSDBJNIConnector getConnector() { return this.connector; } @@ -58,7 +58,7 @@ public class TSDBConnection extends AbstractConnection { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } - return new TSDBStatement(this, this.connector); + return new TSDBStatement(this); } public TSDBSubscribe subscribe(String topic, String sql, boolean restart) throws SQLException { @@ -74,14 +74,18 @@ public class TSDBConnection extends AbstractConnection { } public PreparedStatement prepareStatement(String sql) throws SQLException { - if (isClosed()) + if (isClosed()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); - return new TSDBPreparedStatement(this, this.connector, sql); + } + + return new TSDBPreparedStatement(this, sql); } public void close() throws SQLException { - if (isClosed) + if (isClosed) { return; + } + this.connector.closeConnection(); this.isClosed = true; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 05e28578f1..62cd441b3a 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -30,10 +30,13 @@ public class TSDBJNIConnector { private static volatile Boolean isInitialized = false; private TaosInfo taosInfo = TaosInfo.getInstance(); + // Connection pointer used in C private long taos = TSDBConstants.JNI_NULL_POINTER; + // result set status in current connection private boolean isResultsetClosed = true; + private int affectedRows = -1; static { @@ -163,37 +166,14 @@ public class TSDBJNIConnector { private native long isUpdateQueryImp(long connection, long pSql); /** - * Free resultset operation from C to release resultset pointer by JNI + * Free result set operation from C to release result set pointer by JNI */ public int freeResultSet(long pSql) { - int res = TSDBConstants.JNI_SUCCESS; -// if (result != taosResultSetPointer && taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { -// throw new RuntimeException("Invalid result set pointer"); -// } - -// if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - res = this.freeResultSetImp(this.taos, pSql); -// taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; -// } - + int res = this.freeResultSetImp(this.taos, pSql); isResultsetClosed = true; return res; } - /** - * Close the open result set which is associated to the current connection. If the result set is already - * closed, return 0 for success. - */ -// public int freeResultSet() { -// int resCode = TSDBConstants.JNI_SUCCESS; -// if (!isResultsetClosed) { -// resCode = this.freeResultSetImp(this.taos, this.taosResultSetPointer); -// taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; -// isResultsetClosed = true; -// } -// return resCode; -// } - private native int freeResultSetImp(long connection, long result); /** @@ -240,6 +220,7 @@ public class TSDBJNIConnector { */ public void closeConnection() throws SQLException { int code = this.closeConnectionImp(this.taos); + if (code < 0) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); } else if (code == 0) { @@ -247,6 +228,7 @@ public class TSDBJNIConnector { } else { throw new SQLException("Undefined error code returned by TDengine when closing a connection"); } + // invoke closeConnectionImpl only here taosInfo.connect_close_increment(); } @@ -283,7 +265,7 @@ public class TSDBJNIConnector { private native void unsubscribeImp(long subscription, boolean isKeep); /** - * Validate if a create table sql statement is correct without actually creating that table + * Validate if a create table SQL statement is correct without actually creating that table */ public boolean validateCreateTableSql(String sql) { int res = validateCreateTableSqlImp(taos, sql.getBytes()); @@ -295,7 +277,7 @@ public class TSDBJNIConnector { public long prepareStmt(String sql) throws SQLException { Long stmt = 0L; try { - stmt = prepareStmtImp(sql, this.taos); + stmt = prepareStmtImp(sql.getBytes(), this.taos); } catch (Exception e) { e.printStackTrace(); throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_ENCODING); @@ -316,7 +298,7 @@ public class TSDBJNIConnector { return stmt; } - private native long prepareStmtImp(String sql, long con); + private native long prepareStmtImp(byte[] sql, long con); public int setBindTableName(long stmt, String tableName) { return setBindTableNameImp(stmt, tableName, this.taos); @@ -324,11 +306,11 @@ public class TSDBJNIConnector { private native int setBindTableNameImp(long stmt, String name, long conn); - public int bindColumnDataArray(long stmt, byte[] data, int type, int numOfRows, int columnIndex) { - return bindColDataImp(stmt, data, type, numOfRows, columnIndex, this.taos); + public int bindColumnDataArray(long stmt, ByteBuffer colList, ByteBuffer lengthList, int type, int bytes, int numOfRows,int columnIndex) { + return bindColDataImp(stmt, colList.array(), lengthList.array(), type, bytes, numOfRows, columnIndex, this.taos); } - private native int bindColDataImp(long stmt, byte[] data, int type, int numOfRows, int columnIndex, long conn); + private native int bindColDataImp(long stmt, byte[] data, byte[] length, int type, int bytes, int numOfRows, int columnIndex, long conn); public int executeBatch(long stmt) { return executeBatchImp(stmt, this.taos); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index be6c2361a1..cc781d1613 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -22,8 +22,7 @@ import java.io.Reader; import java.math.BigDecimal; import java.net.URL; import java.nio.ByteBuffer; -import java.nio.DoubleBuffer; -import java.nio.IntBuffer; +import java.nio.ByteOrder; import java.sql.*; import java.util.ArrayList; import java.util.Calendar; @@ -45,12 +44,12 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat private int type; private String tableName; - private long nativeStmtPtr = 0; + private long nativeStmtHandle = 0; private volatile TSDBParameterMetaData parameterMetaData; - TSDBPreparedStatement(TSDBConnection connection, TSDBJNIConnector connector, String sql) { - super(connection, connector); + TSDBPreparedStatement(TSDBConnection connection, String sql) { + super(connection); init(sql); int parameterCnt = 0; @@ -64,8 +63,9 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat this.isPrepared = true; } - this.colData = new ArrayList(parameterCnt); - this.colData.addAll(Collections.nCopies(parameterCnt, null)); + // the table name is also a parameter, so ignore it. + this.colData = new ArrayList(parameterCnt - 1); + this.colData.addAll(Collections.nCopies(parameterCnt - 1, null)); } private void init(String sql) { @@ -543,12 +543,15 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat private int type; private boolean typeIsSet; - public void ClumnInfo() { + public ColumnInfo() { this.typeIsSet = false; } - public void setType(int type) { - Assert.check(!this.typeIsSet); + public void setType(int type) throws SQLException { + if (this.isTypeSet()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "column data type has been set"); + } + this.typeIsSet = true; this.type = type; } @@ -562,106 +565,160 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat this.tableName = name; } - @SuppressWarnings("unchecked") - public void setInt(int columnIndex, ArrayList list) throws SQLException { + public void setValueImpl(int columnIndex, ArrayList list, int type) throws SQLException { ColumnInfo col = (ColumnInfo) this.colData.get(columnIndex); if (col == null) { ColumnInfo p = new ColumnInfo(); - p.setType(TSDBConstants.TSDB_DATA_TYPE_INT); - p.data = (ArrayList) list.clone(); + p.setType(type); + p.data = (ArrayList) list.clone(); this.colData.set(columnIndex, p); } else { - if (col.type != TSDBConstants.TSDB_DATA_TYPE_INT) { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (col.type != type) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "column data type mismatch"); } col.data.addAll(list); } } - @SuppressWarnings("unchecked") - public void setFloat(int columnIndex, ArrayList list) throws SQLException { - ColumnInfo col = (ColumnInfo) this.colData.get(columnIndex); - if (col == null) { - ColumnInfo p = new ColumnInfo(); - p.setType(TSDBConstants.TSDB_DATA_TYPE_INT); - p.data = (ArrayList) list.clone(); - this.colData.set(columnIndex, p); - } else { - if (col.type != TSDBConstants.TSDB_DATA_TYPE_INT) { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - } - - col.data.addAll(list); - } + public void setInt(int columnIndex, ArrayList list) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_INT); } - public void addColumnDataBatch() { + public void setFloat(int columnIndex, ArrayList list) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_FLOAT); + } + + public void setTimestamp(int columnIndex, ArrayList list) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP); + } + + public void setLong(int columnIndex, ArrayList list) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_BIGINT); + } + + public void setDouble(int columnIndex, ArrayList list) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_DOUBLE); + } + + public void setBoolean(int columnIndex, ArrayList list) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_BOOL); + } + + public void setByte(int columnIndex, ArrayList list) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_TINYINT); + } + + public void setShort(int columnIndex, ArrayList list) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_SMALLINT); + } + + public void setString(int columnIndex, ArrayList list) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_BINARY); + } + + public void setNString(int columnIndex, ArrayList list) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_NCHAR); + } + + public void columnDataAddBatch() { // do nothing } - public void columnDataExecuteBatch() { - int size = this.colData.size(); - ColumnInfo col = (ColumnInfo) this.colData.get(0); - int rows = col.data.size(); + public void columnDataExecuteBatch() throws SQLException { + int numOfCols = this.colData.size(); + int rows = ((ColumnInfo) this.colData.get(0)).data.size(); // pass the data block to native code - TSDBJNIConnector conn = null; + TSDBJNIConnector connector = null; try { - conn = (TSDBJNIConnector) this.getConnection(); - this.nativeStmtPtr = conn.prepareStmt(rawSql); - conn.setBindTableName(this.nativeStmtPtr, this.tableName); + connector = ((TSDBConnection) this.getConnection()).getConnector(); + this.nativeStmtHandle = connector.prepareStmt(rawSql); + + // table name is not set yet, abort + if (this.tableName == null) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "table name not set yet"); + } + connector.setBindTableName(this.nativeStmtHandle, this.tableName); } catch (SQLException e) { e.printStackTrace(); } - for (int i = 0; i < size; ++i) { + int bytes = 0; + + for (int i = 0; i < numOfCols; ++i) { ColumnInfo col1 = this.colData.get(i); - Assert.check(col.isTypeSet()); - ByteBuffer ib = ByteBuffer.allocate(rows); - + if (!col1.isTypeSet()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "column data not bind"); + } + + if (rows != col1.data.size()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "the rows in column data not identical"); + } + + ByteBuffer bbuf = null; + + ByteBuffer lengthBuf = ByteBuffer.allocate(rows * Integer.BYTES); + lengthBuf.order(ByteOrder.LITTLE_ENDIAN); + switch (col1.type) { case TSDBConstants.TSDB_DATA_TYPE_INT: { + bbuf = ByteBuffer.allocate(rows * Integer.BYTES); + bbuf.order(ByteOrder.LITTLE_ENDIAN); + for (int j = 0; j < rows; ++j) { - Integer val = (Integer) col.data.get(j); + Integer val = (Integer) col1.data.get(j); if (val == null) { - ib.putInt(Integer.MIN_VALUE); + bbuf.putInt(j * Integer.BYTES, Integer.MIN_VALUE); } else { - ib.putInt((int) col.data.get(j)); + bbuf.putInt(j * Integer.BYTES, val); } + + lengthBuf.putInt(j * Integer.BYTES, Integer.BYTES); } + bytes = Integer.BYTES; break; } case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { + bbuf = ByteBuffer.allocate(rows * Long.BYTES); + bbuf.order(ByteOrder.LITTLE_ENDIAN); + for (int j = 0; j < rows; ++j) { - ib.putLong((long) col.data.get(j)); + Long val = (Long) col1.data.get(j); + if (val == null) { + bbuf.putLong(j * Long.BYTES, Long.MIN_VALUE); + } else { + bbuf.putLong(j * Long.BYTES, val); + } + lengthBuf.putInt(j * Integer.BYTES, Long.BYTES); } + + bytes = Long.BYTES; break; } }; - conn.bindColumnDataArray(this.nativeStmtPtr, ib.array(), col1.type, rows, i); + connector.bindColumnDataArray(this.nativeStmtHandle, bbuf, lengthBuf, col1.type, bytes, rows, i); } - conn.executeBatch(this.nativeStmtPtr); + connector.executeBatch(this.nativeStmtHandle); } - public void columnDataClearBatchClear() { + public void columnDataClearBatch() { // TODO clear data in this.colData } - public void close() { - TSDBJNIConnector conn = null; + public void columnDataCloseBatch() { + TSDBJNIConnector connector = null; try { - conn = (TSDBJNIConnector) this.getConnection(); - this.nativeStmtPtr = conn.prepareStmt(rawSql); - conn.setBindTableName(this.nativeStmtPtr, this.tableName); + connector = ((TSDBConnection) this.getConnection()).getConnector(); + connector.closeBatch(this.nativeStmtHandle); + this.nativeStmtHandle = 0L; + this.tableName = null; } catch (SQLException e) { e.printStackTrace(); } - - conn.closeBatch(this.nativeStmtPtr); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index fb20a621b0..d8ba67576d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -19,8 +19,6 @@ import java.sql.ResultSet; import java.sql.SQLException; public class TSDBStatement extends AbstractStatement { - - private TSDBJNIConnector connector; /** * Status of current statement */ @@ -29,29 +27,26 @@ public class TSDBStatement extends AbstractStatement { private TSDBConnection connection; private TSDBResultSet resultSet; - public void setConnection(TSDBConnection connection) { + TSDBStatement(TSDBConnection connection) { this.connection = connection; } - TSDBStatement(TSDBConnection connection, TSDBJNIConnector connector) { - this.connection = connection; - this.connector = connector; - } - public ResultSet executeQuery(String sql) throws SQLException { // check if closed - if (isClosed()) + if (isClosed()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + } + //TODO: 如果在executeQuery方法中执行insert语句，那么先执行了SQL，再通过pSql来检查是否为一个insert语句，但这个insert SQL已经执行成功了 // execute query - long pSql = this.connector.executeQuery(sql); + long pSql = this.connection.getConnector().executeQuery(sql); // if pSql is create/insert/update/delete/alter SQL - if (this.connector.isUpdateQuery(pSql)) { - this.connector.freeResultSet(pSql); + if (this.connection.getConnector().isUpdateQuery(pSql)) { + this.connection.getConnector().freeResultSet(pSql); throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEQUERY); } - TSDBResultSet res = new TSDBResultSet(this, this.connector, pSql); + TSDBResultSet res = new TSDBResultSet(this, this.connection.getConnector(), pSql); res.setBatchFetch(this.connection.getBatchFetch()); return res; } @@ -60,14 +55,14 @@ public class TSDBStatement extends AbstractStatement { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - long pSql = this.connector.executeQuery(sql); + long pSql = this.connection.getConnector().executeQuery(sql); // if pSql is create/insert/update/delete/alter SQL - if (!this.connector.isUpdateQuery(pSql)) { - this.connector.freeResultSet(pSql); + if (!this.connection.getConnector().isUpdateQuery(pSql)) { + this.connection.getConnector().freeResultSet(pSql); throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEUPDATE); } - int affectedRows = this.connector.getAffectedRows(pSql); - this.connector.freeResultSet(pSql); + int affectedRows = this.connection.getConnector().getAffectedRows(pSql); + this.connection.getConnector().freeResultSet(pSql); return affectedRows; } @@ -81,30 +76,29 @@ public class TSDBStatement extends AbstractStatement { public boolean execute(String sql) throws SQLException { // check if closed - if (isClosed()) + if (isClosed()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + } + // execute query - long pSql = this.connector.executeQuery(sql); + long pSql = this.connection.getConnector().executeQuery(sql); // if pSql is create/insert/update/delete/alter SQL - if (this.connector.isUpdateQuery(pSql)) { - this.affectedRows = this.connector.getAffectedRows(pSql); - this.connector.freeResultSet(pSql); + if (this.connection.getConnector().isUpdateQuery(pSql)) { + this.affectedRows = this.connection.getConnector().getAffectedRows(pSql); + this.connection.getConnector().freeResultSet(pSql); return false; } - this.resultSet = new TSDBResultSet(this, this.connector, pSql); + this.resultSet = new TSDBResultSet(this, this.connection.getConnector(), pSql); this.resultSet.setBatchFetch(this.connection.getBatchFetch()); return true; } public ResultSet getResultSet() throws SQLException { - if (isClosed()) + if (isClosed()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); -// long resultSetPointer = connector.getResultSet(); -// TSDBResultSet resSet = null; -// if (resultSetPointer != TSDBConstants.JNI_NULL_POINTER) { -// resSet = new TSDBResultSet(connector, resultSetPointer); -// } + } + return this.resultSet; } @@ -115,12 +109,20 @@ public class TSDBStatement extends AbstractStatement { } public Connection getConnection() throws SQLException { - if (isClosed()) + if (isClosed()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - if (this.connector == null) + } + + if (this.connection.getConnector() == null) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + } + return this.connection; } + + public void setConnection(TSDBConnection connection) { + this.connection = connection; + } public boolean isClosed() throws SQLException { return isClosed; diff --git a/src/inc/taos.h b/src/inc/taos.h index 788502b45a..bb3cfa2744 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -102,11 +102,10 @@ typedef struct TAOS_BIND { typedef struct TAOS_MULTI_BIND { int buffer_type; - void * buffer; - uintptr_t buffer_length; // unused - uintptr_t *length; - int * is_null; - + void *buffer; + uintptr_t buffer_length; + uintptr_t *length; + int *is_null; int num; } TAOS_MULTI_BIND; From 6aba3f5eb1b6cd49a77ec84d204bee3be6f16785 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 9 May 2021 09:00:35 +0000 Subject: [PATCH 10/27] fix bug --- .../taosdata/jdbc/TSDBPreparedStatement.java | 1 - .../com/taosdata/jdbc/utils/NullType.java | 55 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100755 src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index cc781d1613..e8a75cb10b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -14,7 +14,6 @@ *****************************************************************************/ package com.taosdata.jdbc; -import com.sun.tools.javac.util.Assert; import com.taosdata.jdbc.utils.Utils; import java.io.InputStream; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java new file mode 100755 index 0000000000..3aceee79c3 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java @@ -0,0 +1,55 @@ +package com.taosdata.jdbc.utils; + +public class NullType { + private static final byte NULL_BOOL_VAL = 0x2; + private static final String NULL_STR = "null"; + + public String toString() { + return NullType.NULL_STR; + } + + public static boolean isBooleanNull(byte val) { + return val == NullType.NULL_BOOL_VAL; + } + + public static boolean isTinyIntNull(byte val) { + return val == Byte.MIN_VALUE; + } + + public static boolean isSmallIntNull(short val) { + return val == Short.MIN_VALUE; + } + + public static boolean isIntNull(int val) { + return val == Integer.MIN_VALUE; + } + + public static boolean isBigIntNull(long val) { + return val == Long.MIN_VALUE; + } + + public static boolean isFloatNull(float val) { + return Float.isNaN(val); + } + + public static boolean isDoubleNull(double val) { + return Double.isNaN(val); + } + + public static boolean isBinaryNull(byte[] val, int length) { + if (length != Byte.BYTES) { + return false; + } + + return val[0] == 0xFF; + } + + public static boolean isNcharNull(byte[] val, int length) { + if (length != Integer.BYTES) { + return false; + } + + return (val[0] & val[1] & val[2] & val[3]) == 0xFF; + } + +} From 0b91c2cfc6c5741338db5a2144b2d838d02d564f Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sun, 9 May 2021 17:23:10 +0800 Subject: [PATCH 11/27] fix bug --- src/client/src/tscPrepare.c | 6 +-- tests/script/api/batchprepare.c | 71 +++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index d33bdbb588..77ab9e84c3 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1374,9 +1374,6 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { int taos_stmt_close(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; if (!pStmt->isInsert) { - taosHashCleanup(pStmt->mtb.pTableHash); - taosHashCleanup(pStmt->mtb.pTableBlockHashList); - SNormalStmt* normal = &pStmt->normal; if (normal->params != NULL) { for (uint16_t i = 0; i < normal->numParams; i++) { @@ -1386,6 +1383,9 @@ int taos_stmt_close(TAOS_STMT* stmt) { } free(normal->parts); free(normal->sql); + } else { + taosHashCleanup(pStmt->mtb.pTableHash); + taosHashCleanup(pStmt->mtb.pTableBlockHashList); } taos_free_result(pStmt->pSql); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 9b204c2f4a..5303b53329 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -548,6 +548,12 @@ int stmt_funcb1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + return 0; } @@ -700,6 +706,12 @@ int stmt_funcb2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + return 0; } @@ -857,6 +869,12 @@ int stmt_funcb3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + return 0; } @@ -1010,6 +1028,12 @@ int stmt_funcb4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + return 0; } @@ -1157,6 +1181,12 @@ int stmt_funcb5(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + return 0; } @@ -1236,6 +1266,11 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + free(v.ts); + free(lb); + free(params); + free(no_null); + return 0; } @@ -1388,6 +1423,12 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + return 0; } @@ -1544,6 +1585,12 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + return 0; } @@ -1698,6 +1745,12 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + return 0; } @@ -1849,6 +1902,12 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + return 0; } @@ -1934,6 +1993,10 @@ int sql_perf1(TAOS *taos) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); + for (int i = 0; i < 3000; i++) { + free(sql[i]); + } + return 0; } @@ -1983,6 +2046,10 @@ int sql_perf_s1(TAOS *taos) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); + for (int i = 0; i < 360000; i++) { + free(sql[i]); + } + return 0; } @@ -2024,6 +2091,10 @@ int sql_s_perf1(TAOS *taos) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); + for (int i = 0; i < 3000; i++) { + free(sql[i]); + } + return 0; } From 7e2345e371b7e1e2d428001c539d42f8a4f2c185 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sun, 9 May 2021 17:31:18 +0800 Subject: [PATCH 12/27] fix --- src/client/src/tscPrepare.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 77ab9e84c3..bc333625ef 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -810,7 +810,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); if (t1 == NULL) { - tscError("no table data block in hash list, uid:%" PRId64 , pStmt->mtb.currentUid); + tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pStmt->pSql->self, pStmt->mtb.currentUid); return TSDB_CODE_TSC_APP_ERROR; } From fa3f970748bf3794781477cbb4ff12ea021320b7 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 10 May 2021 09:22:10 +0800 Subject: [PATCH 13/27] fix mem leak --- src/client/src/tscPrepare.c | 7 +++++-- tests/script/api/batchprepare.c | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index bc333625ef..c8a3b963f0 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1384,8 +1384,11 @@ int taos_stmt_close(TAOS_STMT* stmt) { free(normal->parts); free(normal->sql); } else { - taosHashCleanup(pStmt->mtb.pTableHash); - taosHashCleanup(pStmt->mtb.pTableBlockHashList); + if (pStmt->multiTbInsert) { + taosHashCleanup(pStmt->mtb.pTableHash); + pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, true); + taosHashCleanup(pStmt->pSql->cmd.pTableBlockHashList); + } } taos_free_result(pStmt->pSql); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 5303b53329..f3ea95abfa 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -2050,6 +2050,8 @@ int sql_perf_s1(TAOS *taos) { free(sql[i]); } + free(sql); + return 0; } From 2e08de2b45bdf99c320176c64f29a11452ab1647 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 10 May 2021 09:26:19 +0800 Subject: [PATCH 14/27] fix crash issue --- src/client/src/tscPrepare.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index c8a3b963f0..6307643c95 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1388,6 +1388,7 @@ int taos_stmt_close(TAOS_STMT* stmt) { taosHashCleanup(pStmt->mtb.pTableHash); pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, true); taosHashCleanup(pStmt->pSql->cmd.pTableBlockHashList); + pStmt->pSql->cmd.pTableBlockHashList = NULL; } } From 29307ce31ebcd99e6e40b00b46393db1b70eb697 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 May 2021 11:07:29 +0800 Subject: [PATCH 15/27] [td-4038]: support column data batch bind. --- .../jni/com_taosdata_jdbc_TSDBJNIConnector.h | 4 +- src/client/src/TSDBJNIConnector.c | 33 ++-- src/client/src/tscPrepare.c | 13 +- .../com/taosdata/jdbc/TSDBJNIConnector.java | 6 +- .../taosdata/jdbc/TSDBPreparedStatement.java | 166 +++++++++++++----- .../taosdata/jdbc/TSDBResultSetBlockData.java | 4 +- .../com/taosdata/jdbc/utils/NullType.java | 50 +++++- src/inc/taos.h | 4 +- 8 files changed, 199 insertions(+), 81 deletions(-) diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index 07fd46f859..04bccc1a4a 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -204,10 +204,10 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameI /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: bindColDataImp - * Signature: (J[B[BIIIIJ)J + * Signature: (J[B[B[BIIIIJ)J */ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp -(JNIEnv *, jobject, jlong, jbyteArray, jbyteArray, jint, jint, jint, jint, jlong); +(JNIEnv *, jobject, jlong, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jlong); /* * Class: com_taosdata_jdbc_TSDBJNIConnector diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 8ffd021e51..4842bb2ed1 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -753,7 +753,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameI } JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp(JNIEnv *env, jobject jobj, jlong stmt, - jbyteArray data, jbyteArray length, jint dataType, jint dataBytes, jint numOfRows, jint colIndex, jlong con) { + jbyteArray colDataList, jbyteArray lengthList, jbyteArray nullList, jint dataType, jint dataBytes, jint numOfRows, jint colIndex, jlong con) { TAOS *tscon = (TAOS *)con; if (tscon == NULL) { jniError("jobj:%p, connection already closed", jobj); @@ -767,16 +767,22 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp(J } // todo refactor - jsize len = (*env)->GetArrayLength(env, data); - char *colBuf = (char *)calloc(1, sizeof(char) * len); - (*env)->GetByteArrayRegion(env, data, 0, len, (jbyte *)colBuf); + jsize len = (*env)->GetArrayLength(env, colDataList); + char *colBuf = (char *)calloc(1, len); + (*env)->GetByteArrayRegion(env, colDataList, 0, len, (jbyte *)colBuf); if ((*env)->ExceptionCheck(env)) { // todo handle error } - len = (*env)->GetArrayLength(env, length); - char *lengthArray = (char*) calloc(1, sizeof(char) * len); - (*env)->GetByteArrayRegion(env, length, 0, len, (jbyte*) lengthArray); + len = (*env)->GetArrayLength(env, lengthList); + char *lengthArray = (char*) calloc(1, len); + (*env)->GetByteArrayRegion(env, lengthList, 0, len, (jbyte*) lengthArray); + if ((*env)->ExceptionCheck(env)) { + } + + len = (*env)->GetArrayLength(env, nullList); + char *nullArray = (char*) calloc(1, len); + (*env)->GetByteArrayRegion(env, nullList, 0, len, (jbyte*) nullArray); if ((*env)->ExceptionCheck(env)) { } @@ -785,10 +791,10 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp(J b->num = numOfRows; b->buffer_type = dataType; // todo check data type - b->buffer_length = tDataTypes[dataType].bytes; - b->is_null = calloc(numOfRows, sizeof(int32_t)); + b->buffer_length = IS_VAR_DATA_TYPE(dataType)? dataBytes:tDataTypes[dataType].bytes; + b->is_null = nullArray; b->buffer = colBuf; - b->length = (uintptr_t*)lengthArray; + b->length = (int32_t*)lengthArray; // set the length and is_null array switch(dataType) { @@ -800,8 +806,13 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp(J int32_t bytes = tDataTypes[dataType].bytes; for(int32_t i = 0; i < numOfRows; ++i) { b->length[i] = bytes; - b->is_null[i] = isNull(colBuf + bytes * i, dataType); } + break; + } + + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_BINARY: { + // do nothing } } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 6307643c95..90023f9ded 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -148,7 +148,7 @@ static int normalStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { break; default: - tscDebug("0x%"PRIx64" param %d: type mismatch or invalid", stmt->pSql->self, i); + tscDebug("0x%"PRIx64" bind column%d: type mismatch or invalid", stmt->pSql->self, i); return TSDB_CODE_TSC_INVALID_VALUE; } } @@ -776,7 +776,7 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU } } else if (param->type == TSDB_DATA_TYPE_BINARY) { if (bind->length[i] > (uintptr_t)param->bytes) { - tscError("invalid binary length"); + tscError("binary length too long, ignore it, expect:%d, actual:%d", param->bytes, (int32_t)bind->length[i]); return TSDB_CODE_TSC_INVALID_VALUE; } int16_t bsize = (short)bind->length[i]; @@ -784,9 +784,10 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU } else if (param->type == TSDB_DATA_TYPE_NCHAR) { int32_t output = 0; if (!taosMbsToUcs4(bind->buffer + bind->buffer_length * i, bind->length[i], varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { - tscError("convert failed"); + tscError("convert nchar string to UCS4_LE failed:%s", (char*)(bind->buffer + bind->buffer_length * i)); return TSDB_CODE_TSC_INVALID_VALUE; } + varDataSetLen(data + param->offset, output); } } @@ -850,7 +851,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { int code = doBindParam(pBlock, data, param, &bind[param->idx], 1); if (code != TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" param %d: type mismatch or invalid", pStmt->pSql->self, param->idx); + tscDebug("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); return code; } } @@ -920,7 +921,7 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int c int code = doBindBatchParam(pBlock, param, &bind[param->idx], pCmd->batchSize); if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" param %d: type mismatch or invalid", pStmt->pSql->self, param->idx); + tscError("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); return code; } } @@ -931,7 +932,7 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int c int code = doBindBatchParam(pBlock, param, bind, pCmd->batchSize); if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" param %d: type mismatch or invalid", pStmt->pSql->self, param->idx); + tscError("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); return code; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 62cd441b3a..016a898e3f 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -306,11 +306,11 @@ public class TSDBJNIConnector { private native int setBindTableNameImp(long stmt, String name, long conn); - public int bindColumnDataArray(long stmt, ByteBuffer colList, ByteBuffer lengthList, int type, int bytes, int numOfRows,int columnIndex) { - return bindColDataImp(stmt, colList.array(), lengthList.array(), type, bytes, numOfRows, columnIndex, this.taos); + public int bindColumnDataArray(long stmt, ByteBuffer colDataList, ByteBuffer lengthList, ByteBuffer isNullList, int type, int bytes, int numOfRows,int columnIndex) { + return bindColDataImp(stmt, colDataList.array(), lengthList.array(), isNullList.array(), type, bytes, numOfRows, columnIndex, this.taos); } - private native int bindColDataImp(long stmt, byte[] data, byte[] length, int type, int bytes, int numOfRows, int columnIndex, long conn); + private native int bindColDataImp(long stmt, byte[] colDataList, byte[] lengthList, byte[] isNullList, int type, int bytes, int numOfRows, int columnIndex, long conn); public int executeBatch(long stmt) { return executeBatchImp(stmt, this.taos); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index e8a75cb10b..357f225dd4 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -18,6 +18,7 @@ import com.taosdata.jdbc.utils.Utils; import java.io.InputStream; import java.io.Reader; +import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.net.URL; import java.nio.ByteBuffer; @@ -40,8 +41,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat private boolean isPrepared; private ArrayList colData; - private int type; - private String tableName; private long nativeStmtHandle = 0; @@ -540,6 +539,7 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @SuppressWarnings("rawtypes") private ArrayList data; private int type; + private int bytes; private boolean typeIsSet; public ColumnInfo() { @@ -564,60 +564,61 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat this.tableName = name; } - public void setValueImpl(int columnIndex, ArrayList list, int type) throws SQLException { + public void setValueImpl(int columnIndex, ArrayList list, int type, int bytes) throws SQLException { ColumnInfo col = (ColumnInfo) this.colData.get(columnIndex); if (col == null) { ColumnInfo p = new ColumnInfo(); p.setType(type); + p.bytes = bytes; p.data = (ArrayList) list.clone(); this.colData.set(columnIndex, p); } else { if (col.type != type) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "column data type mismatch"); } - col.data.addAll(list); } } public void setInt(int columnIndex, ArrayList list) throws SQLException { - setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_INT); + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_INT, Integer.BYTES); } public void setFloat(int columnIndex, ArrayList list) throws SQLException { - setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_FLOAT); + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_FLOAT, Float.BYTES); } public void setTimestamp(int columnIndex, ArrayList list) throws SQLException { - setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP); + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP, Long.BYTES); } public void setLong(int columnIndex, ArrayList list) throws SQLException { - setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_BIGINT); + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_BIGINT, Long.BYTES); } public void setDouble(int columnIndex, ArrayList list) throws SQLException { - setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_DOUBLE); + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_DOUBLE, Double.BYTES); } public void setBoolean(int columnIndex, ArrayList list) throws SQLException { - setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_BOOL); + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_BOOL, Byte.BYTES); } public void setByte(int columnIndex, ArrayList list) throws SQLException { - setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_TINYINT); + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_TINYINT, Byte.BYTES); } public void setShort(int columnIndex, ArrayList list) throws SQLException { - setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_SMALLINT); + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_SMALLINT, Short.BYTES); } - public void setString(int columnIndex, ArrayList list) throws SQLException { - setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_BINARY); + public void setString(int columnIndex, ArrayList list, int size) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_BINARY, size); } - public void setNString(int columnIndex, ArrayList list) throws SQLException { - setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_NCHAR); + // note: expand the required space for each NChar character + public void setNString(int columnIndex, ArrayList list, int size) throws SQLException { + setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_NCHAR, size * Integer.BYTES); } public void columnDataAddBatch() { @@ -643,11 +644,9 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat e.printStackTrace(); } - int bytes = 0; - for (int i = 0; i < numOfCols; ++i) { ColumnInfo col1 = this.colData.get(i); - if (!col1.isTypeSet()) { + if (col1 == null || !col1.isTypeSet()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "column data not bind"); } @@ -655,51 +654,122 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "the rows in column data not identical"); } - ByteBuffer bbuf = null; + ByteBuffer colDataList = ByteBuffer.allocate(rows * col1.bytes); + colDataList.order(ByteOrder.LITTLE_ENDIAN); - ByteBuffer lengthBuf = ByteBuffer.allocate(rows * Integer.BYTES); - lengthBuf.order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer lengthList = ByteBuffer.allocate(rows * Integer.BYTES); + lengthList.order(ByteOrder.LITTLE_ENDIAN); + + ByteBuffer isNullList = ByteBuffer.allocate(rows * Byte.BYTES); + isNullList.order(ByteOrder.LITTLE_ENDIAN); switch (col1.type) { case TSDBConstants.TSDB_DATA_TYPE_INT: { - bbuf = ByteBuffer.allocate(rows * Integer.BYTES); - bbuf.order(ByteOrder.LITTLE_ENDIAN); - for (int j = 0; j < rows; ++j) { Integer val = (Integer) col1.data.get(j); - if (val == null) { - bbuf.putInt(j * Integer.BYTES, Integer.MIN_VALUE); - } else { - bbuf.putInt(j * Integer.BYTES, val); - } - - lengthBuf.putInt(j * Integer.BYTES, Integer.BYTES); + colDataList.putInt(val == null? Integer.MIN_VALUE:val); + isNullList.put((byte) (val == null? 1:0)); + } + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: { + for (int j = 0; j < rows; ++j) { + Byte val = (Byte) col1.data.get(j); + colDataList.put(val == null? 0:val); + isNullList.put((byte) (val == null? 1:0)); + } + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_BOOL: { + for (int j = 0; j < rows; ++j) { + Byte val = (Byte) col1.data.get(j); + colDataList.put(val == null? 0:val); + isNullList.put((byte) (val == null? 1:0)); + } + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: { + for (int j = 0; j < rows; ++j) { + Short val = (Short) col1.data.get(j); + colDataList.putShort(val == null? 0:val); + isNullList.put((byte) (val == null? 1:0)); + } + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: { + for (int j = 0; j < rows; ++j) { + Long val = (Long) col1.data.get(j); + colDataList.putLong(val == null? 0:val); + isNullList.put((byte) (val == null? 1:0)); + } + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: { + for (int j = 0; j < rows; ++j) { + Float val = (Float) col1.data.get(j); + colDataList.putFloat(val == null? 0:val); + isNullList.put((byte) (val == null? 1:0)); } - - bytes = Integer.BYTES; break; } - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { - bbuf = ByteBuffer.allocate(rows * Long.BYTES); - bbuf.order(ByteOrder.LITTLE_ENDIAN); - + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { for (int j = 0; j < rows; ++j) { - Long val = (Long) col1.data.get(j); - if (val == null) { - bbuf.putLong(j * Long.BYTES, Long.MIN_VALUE); - } else { - bbuf.putLong(j * Long.BYTES, val); - } - lengthBuf.putInt(j * Integer.BYTES, Long.BYTES); + Double val = (Double) col1.data.get(j); + colDataList.putDouble(val == null? 0:val); + isNullList.put((byte) (val == null? 1:0)); } - - bytes = Long.BYTES; break; } + + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + case TSDBConstants.TSDB_DATA_TYPE_BINARY: { + String charset = TaosGlobalConfig.getCharset(); + for (int j = 0; j < rows; ++j) { + String val = (String) col1.data.get(j); + if (val != null && val.length() > col1.bytes) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "string data too long"); + } + + colDataList.position(j * col1.bytes); // seek to the correct position + if (val != null) { + byte[] b = null; + try { + if (col1.type == TSDBConstants.TSDB_DATA_TYPE_BINARY) { + b = val.getBytes(); + } else { + b = val.getBytes(charset); + } + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + + colDataList.put(b); + lengthList.putInt(b.length); + isNullList.put((byte) 0); + } else { + lengthList.putInt(0); + isNullList.put((byte) 1); + } + } + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: + case TSDBConstants.TSDB_DATA_TYPE_USMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_UINT: + case TSDBConstants.TSDB_DATA_TYPE_UBIGINT: { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "not support data types"); + } }; - connector.bindColumnDataArray(this.nativeStmtHandle, bbuf, lengthBuf, col1.type, bytes, rows, i); + connector.bindColumnDataArray(this.nativeStmtHandle, colDataList, lengthList, isNullList, col1.type, col1.bytes, rows, i); } connector.executeBatch(this.nativeStmtHandle); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java index 814fd6c18d..7b3be5d263 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java @@ -438,8 +438,8 @@ public class TSDBResultSetBlockData { } try { - String ss = TaosGlobalConfig.getCharset(); - return new String(dest, ss); + String charset = TaosGlobalConfig.getCharset(); + return new String(dest, charset); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java index 3aceee79c3..0e05aeeee7 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java @@ -11,11 +11,11 @@ public class NullType { public static boolean isBooleanNull(byte val) { return val == NullType.NULL_BOOL_VAL; } - + public static boolean isTinyIntNull(byte val) { return val == Byte.MIN_VALUE; } - + public static boolean isSmallIntNull(short val) { return val == Short.MIN_VALUE; } @@ -23,19 +23,19 @@ public class NullType { public static boolean isIntNull(int val) { return val == Integer.MIN_VALUE; } - + public static boolean isBigIntNull(long val) { return val == Long.MIN_VALUE; } - + public static boolean isFloatNull(float val) { return Float.isNaN(val); } - + public static boolean isDoubleNull(double val) { return Double.isNaN(val); } - + public static boolean isBinaryNull(byte[] val, int length) { if (length != Byte.BYTES) { return false; @@ -43,7 +43,7 @@ public class NullType { return val[0] == 0xFF; } - + public static boolean isNcharNull(byte[] val, int length) { if (length != Integer.BYTES) { return false; @@ -51,5 +51,41 @@ public class NullType { return (val[0] & val[1] & val[2] & val[3]) == 0xFF; } + + public static byte getBooleanNull() { + return NullType.NULL_BOOL_VAL; + } + + public static byte getTinyintNull() { + return Byte.MIN_VALUE; + } + + public static int getIntNull() { + return Integer.MIN_VALUE; + } + + public static short getSmallIntNull() { + return Short.MIN_VALUE; + } + + public static long getBigIntNull() { + return Long.MIN_VALUE; + } + + public static int getFloatNull() { + return 0x7FF00000; + } + + public static long getDoubleNull() { + return 0x7FFFFF0000000000L; + } + + public static byte getBinaryNull() { + return (byte) 0xFF; + } + + public static byte[] getNcharNull() { + return new byte[] {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF}; + } } diff --git a/src/inc/taos.h b/src/inc/taos.h index bb3cfa2744..6dd695b320 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -104,8 +104,8 @@ typedef struct TAOS_MULTI_BIND { int buffer_type; void *buffer; uintptr_t buffer_length; - uintptr_t *length; - int *is_null; + int32_t *length; + char *is_null; int num; } TAOS_MULTI_BIND; From 8223aa416f3af774cf1f93a6bc231680d1fc5178 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 May 2021 11:15:52 +0800 Subject: [PATCH 16/27] [td-4038]: support column data batch bind. --- src/client/inc/tsclient.h | 3 +-- src/client/src/tscPrepare.c | 52 +++++++++---------------------------- 2 files changed, 13 insertions(+), 42 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 73b0172e85..ce44d4ab83 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -154,13 +154,12 @@ typedef struct STagCond { typedef struct SParamInfo { int32_t idx; - char type; + uint8_t type; uint8_t timePrec; int16_t bytes; uint32_t offset; } SParamInfo; - typedef struct SBoundColumn { bool hasVal; // denote if current column has bound or not int32_t offset; // all column offset value diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 90023f9ded..23dcff276d 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -717,45 +717,14 @@ static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MULTI_BIND* bind, int32_t rowNum) { - if (bind->buffer_type != param->type) { + if (bind->buffer_type != param->type || !isValidDataType(param->type)) { return TSDB_CODE_TSC_INVALID_VALUE; } - short size = 0; - switch(param->type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - size = 1; - break; - - case TSDB_DATA_TYPE_SMALLINT: - size = 2; - break; - - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_FLOAT: - size = 4; - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - size = 8; - break; - - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - if (bind->length == NULL) { - tscError("BINARY/NCHAR no length"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - break; - - default: - assert(false); - return TSDB_CODE_TSC_INVALID_VALUE; + if (IS_VAR_DATA_TYPE(param->type) && bind->length == NULL) { + tscError("BINARY/NCHAR no length"); + return TSDB_CODE_TSC_INVALID_VALUE; } - for (int i = 0; i < bind->num; ++i) { char* data = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * (rowNum + i); @@ -765,8 +734,8 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU continue; } - if (size > 0) { - memcpy(data + param->offset, bind->buffer + bind->buffer_length * i, size); + if (!IS_VAR_DATA_TYPE(param->type)) { + memcpy(data + param->offset, bind->buffer + bind->buffer_length * i, tDataTypes[param->type].bytes); if (param->offset == 0) { if (tsCheckTimestamp(pBlock, data + param->offset) != TSDB_CODE_SUCCESS) { @@ -776,12 +745,17 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU } } else if (param->type == TSDB_DATA_TYPE_BINARY) { if (bind->length[i] > (uintptr_t)param->bytes) { - tscError("binary length too long, ignore it, expect:%d, actual:%d", param->bytes, (int32_t)bind->length[i]); + tscError("binary length too long, ignore it, max:%d, actual:%d", param->bytes, (int32_t)bind->length[i]); return TSDB_CODE_TSC_INVALID_VALUE; } int16_t bsize = (short)bind->length[i]; STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer + bind->buffer_length * i, bsize); } else if (param->type == TSDB_DATA_TYPE_NCHAR) { + if (bind->length[i] > (uintptr_t)param->bytes) { + tscError("nchar string length too long, ignore it, max:%d, actual:%d", param->bytes, (int32_t)bind->length[i]); + return TSDB_CODE_TSC_INVALID_VALUE; + } + int32_t output = 0; if (!taosMbsToUcs4(bind->buffer + bind->buffer_length * i, bind->length[i], varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { tscError("convert nchar string to UCS4_LE failed:%s", (char*)(bind->buffer + bind->buffer_length * i)); @@ -795,8 +769,6 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU return TSDB_CODE_SUCCESS; } - - static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { SSqlCmd* pCmd = &stmt->pSql->cmd; STscStmt* pStmt = (STscStmt*)stmt; From ecb735deb75e1554e8bb3eab4614af9802e0a1f5 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 10 May 2021 11:21:57 +0800 Subject: [PATCH 17/27] add api check --- src/client/src/tscPrepare.c | 112 ++++++++++++++++++++++++++++++++---- 1 file changed, 102 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 6307643c95..f4e1aba009 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -53,10 +53,20 @@ typedef struct SMultiTbStmt { SHashObj *pTableBlockHashList; // data block for each table } SMultiTbStmt; +typedef enum { + STMT_INIT = 1, + STMT_PREPARE, + STMT_SETTBNAME, + STMT_BIND, + STMT_BIND_COL, + STMT_ADD_BATCH, + STMT_EXECUTE +} STMT_ST; + typedef struct STscStmt { bool isInsert; bool multiTbInsert; - int64_t prevTs; + int16_t last; STscObj* taos; SSqlObj* pSql; SMultiTbStmt mtb; @@ -1185,6 +1195,7 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) { pSql->maxRetry = TSDB_MAX_REPLICA; pSql->isBind = true; pStmt->pSql = pSql; + pStmt->last = STMT_INIT; return pStmt; } @@ -1197,6 +1208,13 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { return TSDB_CODE_TSC_DISCONNECTED; } + if (pStmt->last != STMT_INIT) { + tscError("prepare status error, last:%d", pStmt->last); + return TSDB_CODE_TSC_APP_ERROR; + } + + pStmt->last = STMT_PREPARE; + SSqlObj* pSql = pStmt->pSql; size_t sqlLen = strlen(sql); @@ -1302,6 +1320,13 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { return TSDB_CODE_TSC_APP_ERROR; } + if (pStmt->last == STMT_INIT && pStmt->last == STMT_BIND && pStmt->last == STMT_BIND_COL) { + tscError("0x%"PRIx64" settbname status error, last:%d", pSql->self, pStmt->last); + return TSDB_CODE_TSC_APP_ERROR; + } + + pStmt->last = STMT_SETTBNAME; + uint64_t* uid = (uint64_t*)taosHashGet(pStmt->mtb.pTableHash, name, strlen(name)); if (uid != NULL) { pStmt->mtb.currentUid = *uid; @@ -1399,11 +1424,25 @@ int taos_stmt_close(TAOS_STMT* stmt) { int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { STscStmt* pStmt = (STscStmt*)stmt; + if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + if (pStmt->isInsert) { - if (pStmt->multiTbInsert && pStmt->mtb.nameSet == false) { - tscError("no table name set"); - return TSDB_CODE_TSC_APP_ERROR; + if (pStmt->multiTbInsert) { + if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH) { + tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); + return TSDB_CODE_TSC_APP_ERROR; + } + } else { + if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH) { + tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); + return TSDB_CODE_TSC_APP_ERROR; + } } + + pStmt->last = STMT_BIND; return insertStmtBindParam(pStmt, bind); } else { @@ -1414,6 +1453,12 @@ int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { STscStmt* pStmt = (STscStmt*)stmt; + + if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + if (bind == NULL || bind->num <= 0) { tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; @@ -1424,16 +1469,30 @@ int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { return TSDB_CODE_TSC_APP_ERROR; } - if (pStmt->multiTbInsert && !pStmt->mtb.nameSet) { - tscError("0x%"PRIx64" no table name set", pStmt->pSql->self); - return TSDB_CODE_TSC_APP_ERROR; + if (pStmt->multiTbInsert) { + if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH) { + tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); + return TSDB_CODE_TSC_APP_ERROR; + } + } else { + if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH) { + tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); + return TSDB_CODE_TSC_APP_ERROR; + } } + + pStmt->last = STMT_BIND; return insertStmtBindParamBatch(pStmt, bind, -1); } int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int colIdx) { STscStmt* pStmt = (STscStmt*)stmt; + if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + if (bind == NULL || bind->num <= 0) { tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; @@ -1444,11 +1503,19 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, in return TSDB_CODE_TSC_APP_ERROR; } - if (pStmt->multiTbInsert && !pStmt->mtb.nameSet) { - tscError("0x%"PRIx64" no table name set", pStmt->pSql->self); - return TSDB_CODE_TSC_APP_ERROR; + if (pStmt->multiTbInsert) { + if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_BIND_COL) { + tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); + return TSDB_CODE_TSC_APP_ERROR; + } + } else { + if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_BIND_COL) { + tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); + return TSDB_CODE_TSC_APP_ERROR; + } } + pStmt->last = STMT_BIND_COL; return insertStmtBindParamBatch(pStmt, bind, colIdx); } @@ -1457,9 +1524,22 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, in int taos_stmt_add_batch(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; + if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + if (pStmt->isInsert) { + if (pStmt->last != STMT_BIND && pStmt->last != STMT_BIND_COL) { + tscError("0x%"PRIx64" add batch status error, last:%d", pStmt->pSql->self, pStmt->last); + return TSDB_CODE_TSC_APP_ERROR; + } + + pStmt->last = STMT_ADD_BATCH; + return insertStmtAddBatch(pStmt); } + return TSDB_CODE_COM_OPS_NOT_SUPPORT; } @@ -1474,7 +1554,19 @@ int taos_stmt_reset(TAOS_STMT* stmt) { int taos_stmt_execute(TAOS_STMT* stmt) { int ret = 0; STscStmt* pStmt = (STscStmt*)stmt; + if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + if (pStmt->isInsert) { + if (pStmt->last != STMT_ADD_BATCH) { + tscError("0x%"PRIx64" exec status error, last:%d", pStmt->pSql->self, pStmt->last); + return TSDB_CODE_TSC_APP_ERROR; + } + + pStmt->last = STMT_EXECUTE; + if (pStmt->multiTbInsert) { ret = insertBatchStmtExecute(pStmt); } else { From b3b0782ce2581fe1184337560d0e4de88e2bc3c6 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 10 May 2021 11:43:16 +0800 Subject: [PATCH 18/27] fix issue --- tests/script/api/batchprepare.c | 58 ++++++++++++++++----------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index f3ea95abfa..8f1337486e 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -417,11 +417,11 @@ int stmt_funcb1(TAOS_STMT *stmt) { v.ts = malloc(sizeof(int64_t) * 900000 * 60); - uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + int *lb = malloc(60 * sizeof(int)); TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - int* is_null = malloc(sizeof(int) * 60); - int* no_null = malloc(sizeof(int) * 60); + char* is_null = malloc(sizeof(char) * 60); + char* no_null = malloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -574,11 +574,11 @@ int stmt_funcb2(TAOS_STMT *stmt) { v.ts = malloc(sizeof(int64_t) * 900000 * 60); - uintptr_t *lb = malloc(18000 * sizeof(uintptr_t)); + int *lb = malloc(18000 * sizeof(int)); TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - int* is_null = malloc(sizeof(int) * 18000); - int* no_null = malloc(sizeof(int) * 18000); + char* is_null = malloc(sizeof(char) * 18000); + char* no_null = malloc(sizeof(char) * 18000); for (int i = 0; i < 18000; ++i) { lb[i] = 40; @@ -732,11 +732,11 @@ int stmt_funcb3(TAOS_STMT *stmt) { v.ts = malloc(sizeof(int64_t) * 900000 * 60); - uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + int *lb = malloc(60 * sizeof(int)); TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - int* is_null = malloc(sizeof(int) * 60); - int* no_null = malloc(sizeof(int) * 60); + char* is_null = malloc(sizeof(char) * 60); + char* no_null = malloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -897,11 +897,11 @@ int stmt_funcb4(TAOS_STMT *stmt) { v.ts = malloc(sizeof(int64_t) * 900000 * 60); - uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + int *lb = malloc(60 * sizeof(int)); TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - int* is_null = malloc(sizeof(int) * 60); - int* no_null = malloc(sizeof(int) * 60); + char* is_null = malloc(sizeof(char) * 60); + char* no_null = malloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -1056,11 +1056,11 @@ int stmt_funcb5(TAOS_STMT *stmt) { v.ts = malloc(sizeof(int64_t) * 900000 * 60); - uintptr_t *lb = malloc(18000 * sizeof(uintptr_t)); + int *lb = malloc(18000 * sizeof(int)); TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - int* is_null = malloc(sizeof(int) * 18000); - int* no_null = malloc(sizeof(int) * 18000); + char* is_null = malloc(sizeof(char) * 18000); + char* no_null = malloc(sizeof(char) * 18000); for (int i = 0; i < 18000; ++i) { lb[i] = 40; @@ -1200,10 +1200,10 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { v.ts = malloc(sizeof(int64_t) * 30000 * 3000); - uintptr_t *lb = malloc(30000 * sizeof(uintptr_t)); + int *lb = malloc(30000 * sizeof(int)); TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - int* no_null = malloc(sizeof(int) * 200000); + char* no_null = malloc(sizeof(int) * 200000); for (int i = 0; i < 30000; ++i) { lb[i] = 40; @@ -1291,11 +1291,11 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { v.ts = malloc(sizeof(int64_t) * 900000 * 60); - uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + int *lb = malloc(60 * sizeof(int)); TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - int* is_null = malloc(sizeof(int) * 60); - int* no_null = malloc(sizeof(int) * 60); + char* is_null = malloc(sizeof(char) * 60); + char* no_null = malloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -1453,11 +1453,11 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { v.ts = malloc(sizeof(int64_t) * 900000 * 60); - uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + int *lb = malloc(60 * sizeof(int)); TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - int* is_null = malloc(sizeof(int) * 60); - int* no_null = malloc(sizeof(int) * 60); + char* is_null = malloc(sizeof(char) * 60); + char* no_null = malloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -1611,11 +1611,11 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { v.ts = malloc(sizeof(int64_t) * 900000 * 60); - uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + int *lb = malloc(60 * sizeof(int)); TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - int* is_null = malloc(sizeof(int) * 60); - int* no_null = malloc(sizeof(int) * 60); + char* is_null = malloc(sizeof(char) * 60); + char* no_null = malloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -1771,11 +1771,11 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { v.ts = malloc(sizeof(int64_t) * 60); - uintptr_t *lb = malloc(60 * sizeof(uintptr_t)); + int *lb = malloc(60 * sizeof(int)); TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); - int* is_null = malloc(sizeof(int) * 60); - int* no_null = malloc(sizeof(int) * 60); + char* is_null = malloc(sizeof(char) * 60); + char* no_null = malloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; From 7f3d0dd5ad41170625ddf969234706c21dcd28e5 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 10 May 2021 13:19:53 +0800 Subject: [PATCH 19/27] fix bug --- src/client/src/tscPrepare.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index e19d692094..10bf3f43e3 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1670,3 +1670,4 @@ const char *taos_data_type(int type) { } } + \ No newline at end of file From f02fd80a318ba4b150f680966d0fb00366618447 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 10 May 2021 13:24:28 +0800 Subject: [PATCH 20/27] fix bug --- src/client/src/tscPrepare.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 10bf3f43e3..ca887427e0 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1409,7 +1409,7 @@ int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { return TSDB_CODE_TSC_APP_ERROR; } } else { - if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH) { + if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_EXECUTE) { tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); return TSDB_CODE_TSC_APP_ERROR; } @@ -1448,7 +1448,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { return TSDB_CODE_TSC_APP_ERROR; } } else { - if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH) { + if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_EXECUTE) { tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); return TSDB_CODE_TSC_APP_ERROR; } @@ -1482,7 +1482,7 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, in return TSDB_CODE_TSC_APP_ERROR; } } else { - if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_BIND_COL) { + if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_BIND_COL && pStmt->last != STMT_EXECUTE) { tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); return TSDB_CODE_TSC_APP_ERROR; } @@ -1670,4 +1670,3 @@ const char *taos_data_type(int type) { } } - \ No newline at end of file From 420a83437f70391a3a003834e9a67ba423bae67f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 May 2021 19:04:43 +0800 Subject: [PATCH 21/27] [td-4038]refactor --- src/client/src/TSDBJNIConnector.c | 7 +++++ .../com/taosdata/jdbc/TSDBJNIConnector.java | 28 +++++++++++++------ .../taosdata/jdbc/TSDBPreparedStatement.java | 7 +++-- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 4842bb2ed1..da7da17aa3 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -742,6 +742,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameI int32_t code = taos_stmt_set_tbname((void*)stmt, name); if (code != TSDB_CODE_SUCCESS) { + (*env)->ReleaseStringUTFChars(env, jname, name); + jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); return JNI_TDENGINE_ERROR; } @@ -817,6 +819,11 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp(J } int32_t code = taos_stmt_bind_single_param_batch(pStmt, b, colIndex); + tfree(b->length); + tfree(b->buffer); + tfree(b->is_null); + tfree(b); + if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 016a898e3f..2111ab2743 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -300,26 +300,38 @@ public class TSDBJNIConnector { private native long prepareStmtImp(byte[] sql, long con); - public int setBindTableName(long stmt, String tableName) { - return setBindTableNameImp(stmt, tableName, this.taos); + public void setBindTableName(long stmt, String tableName) throws SQLException { + int code = setBindTableNameImp(stmt, tableName, this.taos); + if (code != TSDBConstants.JNI_SUCCESS) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "failed to set table name"); + } } private native int setBindTableNameImp(long stmt, String name, long conn); - public int bindColumnDataArray(long stmt, ByteBuffer colDataList, ByteBuffer lengthList, ByteBuffer isNullList, int type, int bytes, int numOfRows,int columnIndex) { - return bindColDataImp(stmt, colDataList.array(), lengthList.array(), isNullList.array(), type, bytes, numOfRows, columnIndex, this.taos); + public void bindColumnDataArray(long stmt, ByteBuffer colDataList, ByteBuffer lengthList, ByteBuffer isNullList, int type, int bytes, int numOfRows,int columnIndex) throws SQLException { + int code = bindColDataImp(stmt, colDataList.array(), lengthList.array(), isNullList.array(), type, bytes, numOfRows, columnIndex, this.taos); + if (code != TSDBConstants.JNI_SUCCESS) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "failed to bind column data"); + } } private native int bindColDataImp(long stmt, byte[] colDataList, byte[] lengthList, byte[] isNullList, int type, int bytes, int numOfRows, int columnIndex, long conn); - public int executeBatch(long stmt) { - return executeBatchImp(stmt, this.taos); + public void executeBatch(long stmt) throws SQLException { + int code = executeBatchImp(stmt, this.taos); + if (code != TSDBConstants.JNI_SUCCESS) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "failed to execute batch bind"); + } } private native int executeBatchImp(long stmt, long con); - public int closeBatch(long stmt) { - return closeStmt(stmt, this.taos); + public void closeBatch(long stmt) throws SQLException { + int code = closeStmt(stmt, this.taos); + if (code != TSDBConstants.JNI_SUCCESS) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "failed to close batch bind"); + } } private native int closeStmt(long stmt, long con); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index 357f225dd4..9bdebe36b6 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -733,9 +733,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat String charset = TaosGlobalConfig.getCharset(); for (int j = 0; j < rows; ++j) { String val = (String) col1.data.get(j); - if (val != null && val.length() > col1.bytes) { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "string data too long"); - } colDataList.position(j * col1.bytes); // seek to the correct position if (val != null) { @@ -750,6 +747,10 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat e.printStackTrace(); } + if (val.length() > col1.bytes) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "string data too long"); + } + colDataList.put(b); lengthList.putInt(b.length); isNullList.put((byte) 0); From 1d6f4b2123bd22f3064d34092fd52ac4cfdec80a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 May 2021 13:48:23 +0800 Subject: [PATCH 22/27] [td-225]refactor. --- .../taosdata/jdbc/TSDBPreparedStatement.java | 84 +++++++++++-------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index 9bdebe36b6..b7e2a9abd3 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -621,29 +621,32 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat setValueImpl(columnIndex, list, TSDBConstants.TSDB_DATA_TYPE_NCHAR, size * Integer.BYTES); } - public void columnDataAddBatch() { - // do nothing - } - - public void columnDataExecuteBatch() throws SQLException { - int numOfCols = this.colData.size(); - int rows = ((ColumnInfo) this.colData.get(0)).data.size(); - - // pass the data block to native code - TSDBJNIConnector connector = null; - try { - connector = ((TSDBConnection) this.getConnection()).getConnector(); - this.nativeStmtHandle = connector.prepareStmt(rawSql); - - // table name is not set yet, abort - if (this.tableName == null) { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "table name not set yet"); - } - connector.setBindTableName(this.nativeStmtHandle, this.tableName); - } catch (SQLException e) { - e.printStackTrace(); + public void columnDataAddBatch() throws SQLException { + // pass the data block to native code + if (rawSql == null) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "sql statement not set yet"); } - + + // table name is not set yet, abort + if (this.tableName == null) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "table name not set yet"); + } + + int numOfCols = this.colData.size(); + if (numOfCols == 0) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "column data not bind"); + } + + TSDBJNIConnector connector = ((TSDBConnection) this.getConnection()).getConnector(); + this.nativeStmtHandle = connector.prepareStmt(rawSql); + connector.setBindTableName(this.nativeStmtHandle, this.tableName); + + ColumnInfo colInfo = (ColumnInfo) this.colData.get(0); + if (colInfo == null) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "column data not bind"); + } + + int rows = colInfo.data.size(); for (int i = 0; i < numOfCols; ++i) { ColumnInfo col1 = this.colData.get(i); if (col1 == null || !col1.isTypeSet()) { @@ -684,8 +687,13 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat case TSDBConstants.TSDB_DATA_TYPE_BOOL: { for (int j = 0; j < rows; ++j) { - Byte val = (Byte) col1.data.get(j); - colDataList.put(val == null? 0:val); + Boolean val = (Boolean) col1.data.get(j); + if (val == null) { + colDataList.put((byte) 0); + } else { + colDataList.put((byte) (val? 1:0)); + } + isNullList.put((byte) (val == null? 1:0)); } break; @@ -772,23 +780,27 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat connector.bindColumnDataArray(this.nativeStmtHandle, colDataList, lengthList, isNullList, col1.type, col1.bytes, rows, i); } - + } + + public void columnDataExecuteBatch() throws SQLException { + TSDBJNIConnector connector = ((TSDBConnection) this.getConnection()).getConnector(); connector.executeBatch(this.nativeStmtHandle); + this.columnDataClearBatch(); } public void columnDataClearBatch() { - // TODO clear data in this.colData + int size = this.colData.size(); + this.colData.clear(); + + this.colData.addAll(Collections.nCopies(size, null)); + this.tableName = null; // clear the table name } - public void columnDataCloseBatch() { - TSDBJNIConnector connector = null; - try { - connector = ((TSDBConnection) this.getConnection()).getConnector(); - connector.closeBatch(this.nativeStmtHandle); - this.nativeStmtHandle = 0L; - this.tableName = null; - } catch (SQLException e) { - e.printStackTrace(); - } + public void columnDataCloseBatch() throws SQLException { + TSDBJNIConnector connector = ((TSDBConnection) this.getConnection()).getConnector(); + connector.closeBatch(this.nativeStmtHandle); + + this.nativeStmtHandle = 0L; + this.tableName = null; } } From 98feeefd4e9f874daf1baa7791fdb82635b98e1f Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Tue, 11 May 2021 14:10:01 +0800 Subject: [PATCH 23/27] fix compile error --- src/client/src/tscPrepare.c | 10 +++++----- src/query/src/qTokenizer.c | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index ca887427e0..952a29eab4 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -745,7 +745,7 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU } if (!IS_VAR_DATA_TYPE(param->type)) { - memcpy(data + param->offset, bind->buffer + bind->buffer_length * i, tDataTypes[param->type].bytes); + memcpy(data + param->offset, (char *)bind->buffer + bind->buffer_length * i, tDataTypes[param->type].bytes); if (param->offset == 0) { if (tsCheckTimestamp(pBlock, data + param->offset) != TSDB_CODE_SUCCESS) { @@ -759,7 +759,7 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU return TSDB_CODE_TSC_INVALID_VALUE; } int16_t bsize = (short)bind->length[i]; - STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer + bind->buffer_length * i, bsize); + STR_WITH_SIZE_TO_VARSTR(data + param->offset, (char *)bind->buffer + bind->buffer_length * i, bsize); } else if (param->type == TSDB_DATA_TYPE_NCHAR) { if (bind->length[i] > (uintptr_t)param->bytes) { tscError("nchar string length too long, ignore it, max:%d, actual:%d", param->bytes, (int32_t)bind->length[i]); @@ -767,8 +767,8 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU } int32_t output = 0; - if (!taosMbsToUcs4(bind->buffer + bind->buffer_length * i, bind->length[i], varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { - tscError("convert nchar string to UCS4_LE failed:%s", (char*)(bind->buffer + bind->buffer_length * i)); + if (!taosMbsToUcs4((char *)bind->buffer + bind->buffer_length * i, bind->length[i], varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { + tscError("convert nchar string to UCS4_LE failed:%s", (char*)((char *)bind->buffer + bind->buffer_length * i)); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -1293,7 +1293,7 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { return TSDB_CODE_TSC_APP_ERROR; } - if (pStmt->last == STMT_INIT && pStmt->last == STMT_BIND && pStmt->last == STMT_BIND_COL) { + if (pStmt->last == STMT_INIT || pStmt->last == STMT_BIND || pStmt->last == STMT_BIND_COL) { tscError("0x%"PRIx64" settbname status error, last:%d", pSql->self, pStmt->last); return TSDB_CODE_TSC_APP_ERROR; } diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 4a4897f5c2..a16bcd4fc9 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -562,9 +562,9 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { SStrToken tscReplaceStrToken(char **str, SStrToken *token, const char* newToken) { char *src = *str; - int32_t nsize = strlen(newToken); - int32_t size = strlen(*str) - token->n + nsize + 1; - int32_t bsize = (uint64_t)token->z - (uint64_t)src; + size_t nsize = strlen(newToken); + int32_t size = (int32_t)strlen(*str) - token->n + (int32_t)nsize + 1; + int32_t bsize = (int32_t)((uint64_t)token->z - (uint64_t)src); SStrToken ntoken; *str = calloc(1, size); @@ -573,7 +573,7 @@ SStrToken tscReplaceStrToken(char **str, SStrToken *token, const char* newToken) strcat(*str, newToken); strcat(*str, token->z + token->n); - ntoken.n = nsize; + ntoken.n = (uint32_t)nsize; ntoken.z = *str + bsize; tfree(src); From b2d68568d886707c114aa8c13e5ecf2be8a86f72 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 May 2021 14:20:08 +0800 Subject: [PATCH 24/27] [td-4038]refactor. --- .../main/java/com/taosdata/jdbc/AbstractResultSet.java | 7 +++++-- .../java/com/taosdata/jdbc/TSDBResultSetRowData.java | 5 +++-- .../test/java/com/taosdata/jdbc/TSDBResultSetTest.java | 10 +++++----- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java index 4b5b88d93b..f8ea9af423 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java @@ -84,10 +84,12 @@ public abstract class AbstractResultSet extends WrapperImpl implements ResultSet } @Override + @Deprecated public InputStream getUnicodeStream(int columnIndex) throws SQLException { - if (isClosed()) + if (isClosed()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @@ -171,6 +173,7 @@ public abstract class AbstractResultSet extends WrapperImpl implements ResultSet } @Override + @Deprecated public InputStream getUnicodeStream(String columnLabel) throws SQLException { return getUnicodeStream(findColumn(columnLabel)); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java index 34470fbc4e..618e896a6d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java @@ -84,7 +84,8 @@ public class TSDBResultSetRowData { data.set(col, value); } - public int getInt(int col, int srcType) throws SQLException { + @SuppressWarnings("deprecation") + public int getInt(int col, int srcType) throws SQLException { Object obj = data.get(col); switch (srcType) { @@ -128,7 +129,7 @@ public class TSDBResultSetRowData { long value = (long) obj; if (value < 0) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); - return new Long(value).intValue(); + return Long.valueOf(value).intValue(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java index c5c6f7bca5..f304fd6874 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java @@ -3,7 +3,6 @@ package com.taosdata.jdbc; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.primitives.Shorts; -import com.taosdata.jdbc.rs.RestfulResultSet; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -177,7 +176,8 @@ public class TSDBResultSetTest { rs.getAsciiStream("f1"); } - @Test(expected = SQLFeatureNotSupportedException.class) + @SuppressWarnings("deprecation") + @Test(expected = SQLFeatureNotSupportedException.class) public void getUnicodeStream() throws SQLException { rs.getUnicodeStream("f1"); } @@ -326,7 +326,7 @@ public class TSDBResultSetTest { @Test(expected = SQLFeatureNotSupportedException.class) public void getRow() throws SQLException { - int row = rs.getRow(); + rs.getRow(); } @Test(expected = SQLFeatureNotSupportedException.class) @@ -405,12 +405,12 @@ public class TSDBResultSetTest { @Test(expected = SQLFeatureNotSupportedException.class) public void updateByte() throws SQLException { - rs.updateByte(1, new Byte("0")); + rs.updateByte(1, (byte) 0); } @Test(expected = SQLFeatureNotSupportedException.class) public void updateShort() throws SQLException { - rs.updateShort(1, new Short("0")); + rs.updateShort(1, (short) 0); } @Test(expected = SQLFeatureNotSupportedException.class) From fd500aba599ba2fe478116325c955fff196ce322 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Tue, 11 May 2021 15:32:28 +0800 Subject: [PATCH 25/27] add records num check --- src/client/src/tscPrepare.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 952a29eab4..611cb604c4 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -932,6 +932,11 @@ static int insertStmtUpdateBatch(STscStmt* stmt) { SSqlCmd* pCmd = &pSql->cmd; STableDataBlocks* pBlock = NULL; + if (pCmd->batchSize > INT16_MAX) { + tscError("too many record:%d", pCmd->batchSize); + return TSDB_CODE_TSC_APP_ERROR; + } + assert(pCmd->numOfClause == 1); if (taosHashGetSize(pCmd->pTableBlockHashList) == 0) { return TSDB_CODE_SUCCESS; @@ -1432,7 +1437,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { return TSDB_CODE_TSC_DISCONNECTED; } - if (bind == NULL || bind->num <= 0) { + if (bind == NULL || bind->num <= 0 || bind->num > INT16_MAX) { tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } @@ -1466,7 +1471,7 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, in return TSDB_CODE_TSC_DISCONNECTED; } - if (bind == NULL || bind->num <= 0) { + if (bind == NULL || bind->num <= 0 || bind->num > INT16_MAX) { tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } From eab8e5e8f2f1c008a942dc9672ff618cf0d4580f Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Tue, 11 May 2021 17:07:35 +0800 Subject: [PATCH 26/27] Update makefile --- tests/examples/c/makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/examples/c/makefile b/tests/examples/c/makefile index 09682d35ef..304623c27a 100644 --- a/tests/examples/c/makefile +++ b/tests/examples/c/makefile @@ -14,7 +14,6 @@ exe: gcc $(CFLAGS) ./asyncdemo.c -o $(ROOT)asyncdemo $(LFLAGS) gcc $(CFLAGS) ./demo.c -o $(ROOT)demo $(LFLAGS) gcc $(CFLAGS) ./prepare.c -o $(ROOT)prepare $(LFLAGS) - gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) gcc $(CFLAGS) ./stream.c -o $(ROOT)stream $(LFLAGS) gcc $(CFLAGS) ./subscribe.c -o $(ROOT)subscribe $(LFLAGS) gcc $(CFLAGS) ./apitest.c -o $(ROOT)apitest $(LFLAGS) From ace482e293c8c0ddccd52fcc878cca6dd8209071 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Tue, 11 May 2021 18:59:51 +0800 Subject: [PATCH 27/27] fix bug --- .../java/com/taosdata/jdbc/TSDBPreparedStatement.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index b7e2a9abd3..71e07252a3 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -60,10 +60,12 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat parameters = new Object[parameterCnt]; this.isPrepared = true; } - - // the table name is also a parameter, so ignore it. - this.colData = new ArrayList(parameterCnt - 1); - this.colData.addAll(Collections.nCopies(parameterCnt - 1, null)); + + if (parameterCnt > 1) { + // the table name is also a parameter, so ignore it. + this.colData = new ArrayList(parameterCnt - 1); + this.colData.addAll(Collections.nCopies(parameterCnt - 1, null)); + } } private void init(String sql) {