diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index e4857dd18d..b3e05cb0ca 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -132,10 +132,7 @@ bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd); void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t isTag); -//void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex); -void addRequiredTagColumn(STableMetaInfo* pTableMetaInfo, SColumnIndex* index); - -int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertData(char* sqlstr); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 23db9c3eea..8e72850822 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -294,7 +294,6 @@ typedef struct STscObj { char sversion[TSDB_VERSION_LEN]; char writeAuth : 1; char superAuth : 1; - struct SSqlObj * pSql; struct SSqlObj * pHb; struct SSqlObj * sqlList; struct SSqlStream *streamList; @@ -411,7 +410,7 @@ void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql); void tscKillSTableQuery(SSqlObj *pSql); void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen); -bool tscIsUpdateQuery(STscObj *pObj); +bool tscIsUpdateQuery(SSqlObj* pSql); bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); char *tscGetErrorMsgPayload(SSqlCmd *pCmd); diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index 8dbe63d75a..3782d1693d 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -62,7 +62,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp * Method: executeQueryImp * Signature: ([BJ)I */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp (JNIEnv *, jobject, jbyteArray, jlong); /* @@ -71,7 +71,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp * Signature: (J)I */ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrCodeImp - (JNIEnv *, jobject, jlong); + (JNIEnv *, jobject, jlong, jlong); /* * Class: com_taosdata_jdbc_TSDBJNIConnector @@ -87,7 +87,7 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp * Signature: (J)J */ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp - (JNIEnv *, jobject, jlong); + (JNIEnv *env, jobject jobj, jlong con, jlong tres); /* * Class: com_taosdata_jdbc_TSDBJNIConnector @@ -103,7 +103,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp * Signature: (J)I */ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp - (JNIEnv *, jobject, jlong); + (JNIEnv *env, jobject jobj, jlong con, jlong res); /* * Class: com_taosdata_jdbc_TSDBJNIConnector diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 6ab1b73d1e..2a5f7f441e 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -13,19 +13,35 @@ * along with this program. If not, see . */ -#include "com_taosdata_jdbc_TSDBJNIConnector.h" #include "os.h" #include "taos.h" -#include "tscSubquery.h" +#include "tlog.h" #include "tscUtil.h" #include "tsclient.h" -#include "tlog.h" #include "ttime.h" -#define jniError(...) { if (jniDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR JNI ", jniDebugFlag, __VA_ARGS__); }} -#define jniWarn(...) { if (jniDebugFlag & DEBUG_WARN) { taosPrintLog("WARN JNI ", jniDebugFlag, __VA_ARGS__); }} -#define jniTrace(...) { if (jniDebugFlag & DEBUG_TRACE) { taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); }} -#define jniPrint(...) { taosPrintLog("JNI ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); } +#include "com_taosdata_jdbc_TSDBJNIConnector.h" + +#define jniError(...) \ + { \ + if (jniDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("ERROR JNI ", jniDebugFlag, __VA_ARGS__); \ + } \ + } +#define jniWarn(...) \ + { \ + if (jniDebugFlag & DEBUG_WARN) { \ + taosPrintLog("WARN JNI ", jniDebugFlag, __VA_ARGS__); \ + } \ + } +#define jniTrace(...) \ + { \ + if (jniDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \ + } \ + } +#define jniPrint(...) \ + { taosPrintLog("JNI ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); } int __init = 0; @@ -117,7 +133,8 @@ void jniGetGlobalMethod(JNIEnv *env) { jniTrace("native method register finished"); } -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp(JNIEnv *env, jobject jobj, jint jMode, jstring jPath, jboolean jAutoDump) { +JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp(JNIEnv *env, jobject jobj, jint jMode, + jstring jPath, jboolean jAutoDump) { if (jPath != NULL) { const char *path = (*env)->GetStringUTFChars(env, jPath, NULL); taosSetAllocMode(jMode, path, !!jAutoDump); @@ -172,14 +189,14 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setOptions(JNIEnv } (*env)->ReleaseStringUTFChars(env, optionValue, charset); } else if (optionIndex == TSDB_OPTION_TIMEZONE) { - const char *timezone = (*env)->GetStringUTFChars(env, optionValue, NULL); - if (timezone && strlen(timezone) != 0) { - res = taos_options(TSDB_OPTION_TIMEZONE, timezone); + const char *tz1 = (*env)->GetStringUTFChars(env, optionValue, NULL); + if (tz1 && strlen(tz1) != 0) { + res = taos_options(TSDB_OPTION_TIMEZONE, tz1); jniTrace("set timezone to %s, result:%d", timezone, res); } else { jniTrace("input timezone is empty"); } - (*env)->ReleaseStringUTFChars(env, optionValue, timezone); + (*env)->ReleaseStringUTFChars(env, optionValue, tz1); } else { jniError("option index:%d is not found", optionIndex); } @@ -219,9 +236,9 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEn } /* - * set numOfThreadsPerCore = 0 - * means only one thread for client side scheduler - */ + * set numOfThreadsPerCore = 0 + * means only one thread for client side scheduler + */ tsNumOfThreadsPerCore = 0.0; ret = (jlong)taos_connect((char *)host, (char *)user, (char *)pass, (char *)dbname, jport); @@ -241,8 +258,8 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEn return ret; } -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(JNIEnv *env, jobject jobj, - jbyteArray jsql, jlong con) { +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(JNIEnv *env, jobject jobj, + jbyteArray jsql, jlong con) { TAOS *tscon = (TAOS *)con; if (tscon == NULL) { jniError("jobj:%p, connection is already closed", jobj); @@ -264,65 +281,76 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(J (*env)->GetByteArrayRegion(env, jsql, 0, len, (jbyte *)dst); if ((*env)->ExceptionCheck(env)) { - //todo handle error + // todo handle error } jniTrace("jobj:%p, conn:%p, sql:%s", jobj, tscon, dst); - int code = taos_query(tscon, dst); - if (code != 0) { - jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, tscon, tstrerror(code), taos_errstr(tscon)); - free(dst); - return JNI_TDENGINE_ERROR; - } else { - int32_t affectRows = 0; - SSqlObj *pSql = ((STscObj *)tscon)->pSql; + SSqlObj *pSql = taos_query(tscon, dst); + int32_t code = taos_errno(pSql); + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, tscon, tstrerror(code), taos_errstr(pSql)); + } else { + int32_t affectRows = 0; if (pSql->cmd.command == TSDB_SQL_INSERT) { - affectRows = taos_affected_rows(tscon); + affectRows = taos_affected_rows(pSql); jniTrace("jobj:%p, conn:%p, code:%s, affect rows:%d", jobj, tscon, tstrerror(code), affectRows); } else { jniTrace("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); } - - free(dst); - return affectRows; } + + free(dst); + return (jlong)pSql; } -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrCodeImp(JNIEnv *env, jobject jobj, jlong con) { +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrCodeImp(JNIEnv *env, jobject jobj, jlong con, jlong tres) { TAOS *tscon = (TAOS *)con; if (tscon == NULL) { jniError("jobj:%p, connection is closed", jobj); - return (jint)-TSDB_CODE_INVALID_CONNECTION; + return (jint)TSDB_CODE_INVALID_CONNECTION; } - return (jint)-taos_errno(tscon); + if ((void *)tres == NULL) { + jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); + return JNI_RESULT_SET_NULL; + } + + TAOS_RES *pSql = (TAOS_RES *)tres; + + return (jint)taos_errno(pSql); } -JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp(JNIEnv *env, jobject jobj, jlong con) { - TAOS *tscon = (TAOS *)con; - return (*env)->NewStringUTF(env, (const char *)taos_errstr(tscon)); +JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp(JNIEnv *env, jobject jobj, jlong tres) { + TAOS_RES *pSql = (TAOS_RES *)tres; + return (*env)->NewStringUTF(env, (const char *)taos_errstr(pSql)); } -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp(JNIEnv *env, jobject jobj, jlong con) { +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp(JNIEnv *env, jobject jobj, jlong con, + jlong tres) { TAOS *tscon = (TAOS *)con; if (tscon == NULL) { jniError("jobj:%p, connection is closed", jobj); return JNI_CONNECTION_NULL; } - jlong ret = 0; - - if (tscIsUpdateQuery(tscon)) { - ret = 0; // for update query, no result pointer - jniTrace("jobj:%p, conn:%p, no resultset", jobj, tscon); - } else { - ret = (jlong) taos_use_result(tscon); - jniTrace("jobj:%p, conn:%p, get resultset:%p", jobj, tscon, (void *) ret); + if ((void *)tres == NULL) { + jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); + return JNI_RESULT_SET_NULL; } - return ret; + SSqlObj *pSql = (TAOS_RES *)tres; + STscObj *pObj = pSql->pTscObj; + + if (tscIsUpdateQuery(pSql)) { + taos_free_result(pSql); // free result here + jniTrace("jobj:%p, conn:%p, no resultset, %p", jobj, pObj, (void *)tres); + return 0; + } else { + jniTrace("jobj:%p, conn:%p, get resultset, %p", jobj, pObj, (void *)tres); + return tres; + } } JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp(JNIEnv *env, jobject jobj, jlong con, @@ -343,17 +371,21 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp( return JNI_SUCCESS; } -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp(JNIEnv *env, jobject jobj, - jlong con) { +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp(JNIEnv *env, jobject jobj, jlong con, + jlong res) { TAOS *tscon = (TAOS *)con; if (tscon == NULL) { jniError("jobj:%p, connection is closed", jobj); return JNI_CONNECTION_NULL; } - jint ret = taos_affected_rows(tscon); + if ((void *)res == NULL) { + jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); + return JNI_RESULT_SET_NULL; + } - jniTrace("jobj:%p, conn:%p, affect rows:%d", jobj, tscon, (void *)con, ret); + jint ret = taos_affected_rows((SSqlObj *)res); + jniTrace("jobj:%p, conn:%p, sql:%p, affect rows:%d", jobj, tscon, (void *)con, res, ret); return ret; } @@ -405,7 +437,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaData * @return */ jstring jniFromNCharToByteArray(JNIEnv *env, char *nchar, int32_t maxBytes) { - int len = (int)strlen(nchar); + int len = (int)strlen(nchar); if (len > maxBytes) { // no terminated symbol exists '\0' len = maxBytes; } @@ -439,7 +471,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn TAOS_ROW row = taos_fetch_row(result); if (row == NULL) { - int tserrno = taos_errno(tscon); + int tserrno = taos_errno(result); if (tserrno == 0) { jniTrace("jobj:%p, conn:%p, resultset:%p, fields size is %d, fetch row to the end", jobj, tscon, res, num_fields); return JNI_FETCH_END; @@ -476,24 +508,22 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn float fv = 0; fv = GET_FLOAT_VAL(row[i]); (*env)->CallVoidMethod(env, rowobj, g_rowdataSetFloatFp, i, (jfloat)fv); - } - break; + } break; case TSDB_DATA_TYPE_DOUBLE: { double dv = 0; dv = GET_DOUBLE_VAL(row[i]); (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv); - } - break; + } break; case TSDB_DATA_TYPE_BINARY: { - strncpy(tmp, row[i], (size_t) fields[i].bytes); // handle the case that terminated does not exist + strncpy(tmp, row[i], (size_t)fields[i].bytes); // handle the case that terminated does not exist (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp)); - memset(tmp, 0, (size_t) fields[i].bytes); + memset(tmp, 0, (size_t)fields[i].bytes); break; } case TSDB_DATA_TYPE_NCHAR: { (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteArrayFp, i, - jniFromNCharToByteArray(env, (char*)row[i], fields[i].bytes)); + jniFromNCharToByteArray(env, (char *)row[i], fields[i].bytes)); break; } case TSDB_DATA_TYPE_TIMESTAMP: @@ -521,7 +551,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeConnectionIm } JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNIEnv *env, jobject jobj, jlong con, - jboolean restart, jstring jtopic, jstring jsql, jint jinterval) { + jboolean restart, jstring jtopic, + jstring jsql, jint jinterval) { jlong sub = 0; TAOS *taos = (TAOS *)con; char *topic = NULL; @@ -552,7 +583,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNI return sub; } -static jobject convert_one_row(JNIEnv *env, TAOS_ROW row, TAOS_FIELD* fields, int num_fields) { +static jobject convert_one_row(JNIEnv *env, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { jobject rowobj = (*env)->NewObject(env, g_rowdataClass, g_rowdataConstructor, num_fields); jniTrace("created a rowdata object, rowobj:%p", rowobj); @@ -581,25 +612,23 @@ static jobject convert_one_row(JNIEnv *env, TAOS_ROW row, TAOS_FIELD* fields, in float fv = 0; fv = GET_FLOAT_VAL(row[i]); (*env)->CallVoidMethod(env, rowobj, g_rowdataSetFloatFp, i, (jfloat)fv); - } - break; - case TSDB_DATA_TYPE_DOUBLE:{ + } break; + case TSDB_DATA_TYPE_DOUBLE: { double dv = 0; dv = GET_DOUBLE_VAL(row[i]); (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv); - } - break; + } break; case TSDB_DATA_TYPE_BINARY: { char tmp[TSDB_MAX_BYTES_PER_ROW] = {0}; - strncpy(tmp, row[i], (size_t) fields[i].bytes); // handle the case that terminated does not exist + strncpy(tmp, row[i], (size_t)fields[i].bytes); // handle the case that terminated does not exist (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp)); - memset(tmp, 0, (size_t) fields[i].bytes); + memset(tmp, 0, (size_t)fields[i].bytes); break; } case TSDB_DATA_TYPE_NCHAR: (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteArrayFp, i, - jniFromNCharToByteArray(env, (char*)row[i], fields[i].bytes)); + jniFromNCharToByteArray(env, (char *)row[i], fields[i].bytes)); break; case TSDB_DATA_TYPE_TIMESTAMP: (*env)->CallVoidMethod(env, rowobj, g_rowdataSetTimestampFp, i, (jlong) * ((int64_t *)row[i])); @@ -611,7 +640,8 @@ static jobject convert_one_row(JNIEnv *env, TAOS_ROW row, TAOS_FIELD* fields, in return rowobj; } -JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub, jint timeout) { +JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub, + jint timeout) { jniTrace("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%ld", jobj, sub); jniGetGlobalMethod(env); @@ -619,10 +649,10 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNI jobject rows = (*env)->NewObject(env, g_arrayListClass, g_arrayListConstructFp); int64_t start = taosGetTimestampMs(); - int count = 0; + int count = 0; while (true) { - TAOS_RES * res = taos_consume(tsub); + TAOS_RES *res = taos_consume(tsub); if (res == NULL) { jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub); return NULL; @@ -655,7 +685,8 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNI return rows; } -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, jboolean keepProgress) { +JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, + jboolean keepProgress) { TAOS_SUB *tsub = (TAOS_SUB *)sub; taos_unsubscribe(tsub, keepProgress); } @@ -678,7 +709,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTab char *dst = (char *)calloc(1, sizeof(char) * (len + 1)); (*env)->GetByteArrayRegion(env, jsql, 0, len, (jbyte *)dst); if ((*env)->ExceptionCheck(env)) { - //todo handle error + // todo handle error } int code = taos_validate_sql(tscon, dst); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 18f9872160..1e23094ca8 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -50,14 +50,15 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pSql->fp = fp; + sem_init(&pSql->rspSem, 0, 0); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { tscError("failed to malloc payload"); tscQueueAsyncError(fp, param, TSDB_CODE_CLI_OUT_OF_MEMORY); return; } - pSql->sqlstr = realloc(pSql->sqlstr, sqlLen + 1); - + // todo check for OOM problem + pSql->sqlstr = calloc(1, sqlLen + 1); if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); tscQueueAsyncError(fp, param, TSDB_CODE_CLI_OUT_OF_MEMORY); @@ -95,7 +96,7 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa int32_t sqlLen = strlen(sqlstr); if (sqlLen > tsMaxSQLStringLen) { - tscError("sql string too long"); + tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); terrno = TSDB_CODE_INVALID_SQL; tscQueueAsyncError(fp, param, TSDB_CODE_INVALID_SQL); return; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index d914f392fd..8f4bcedfd6 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -793,7 +793,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - tscSetTableId(pSTableMeterMetaInfo, &sToken, pSql); + tscSetTableFullName(pSTableMeterMetaInfo, &sToken, pSql); strncpy(pTag->name, pSTableMeterMetaInfo->name, TSDB_TABLE_ID_LEN); code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); @@ -834,9 +834,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sql += index; if (TK_STRING == sToken.type) { - sToken.n = strdequote(sToken.z); - strtrim(sToken.z); - sToken.n = (uint32_t)strlen(sToken.z); + strdequote(sToken.z); + sToken.n = strtrim(sToken.z); } if (sToken.type == TK_RP) { @@ -925,7 +924,11 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { for (int32_t i = 0; i < spd.numOfCols; ++i) { if (!spd.hasVal[i]) { // current tag column do not have any value to insert, set it to null - setNull(ptr, pTagSchema[i].type, pTagSchema[i].bytes); + if (pTagSchema[i].type == TSDB_DATA_TYPE_BINARY || pTagSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + setVardataNull(ptr, pTagSchema[i].type); + } else { + setNull(ptr, pTagSchema[i].type, pTagSchema[i].bytes); + } } ptr += pTagSchema[i].bytes; @@ -944,7 +947,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr); } - int32_t ret = tscSetTableId(pTableMetaInfo, &tableToken, pSql); + int32_t ret = tscSetTableFullName(pTableMetaInfo, &tableToken, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -1087,7 +1090,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _error_clean; } - if ((code = tscSetTableId(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscSetTableFullName(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { goto _error_clean; } @@ -1205,9 +1208,8 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { str += index; if (TK_STRING == sToken.type) { - sToken.n = strdequote(sToken.z); - strtrim(sToken.z); - sToken.n = (uint32_t)strlen(sToken.z); + strdequote(sToken.z); + sToken.n = strtrim(sToken.z); } if (sToken.type == TK_RP) { diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 74294d38e0..e6d9aad6e2 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -88,13 +88,23 @@ void tscSaveSlowQueryFp(void *handle, void *tmrId) { } void tscSaveSlowQuery(SSqlObj *pSql) { - const static int64_t SLOW_QUERY_INTERVAL = 3000000L; - if (pSql->res.useconds < SLOW_QUERY_INTERVAL) return; + const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable + size_t size = 200; // other part of sql string, expect the main sql str + + if (pSql->res.useconds < SLOW_QUERY_INTERVAL) { + return; + } tscTrace("%p query time:%" PRId64 " sql:%s", pSql, pSql->res.useconds, pSql->sqlstr); - - char *sql = malloc(200); - int len = snprintf(sql, 200, "insert into %s.slowquery values(now, '%s', %" PRId64 ", %" PRId64 ", '", tsMonitorDbName, + int32_t sqlSize = TSDB_SHOW_SQL_LEN + size; + + char *sql = malloc(sqlSize); + if (sql == NULL) { + tscError("%p failed to allocate memory to sent slow query to dnode", pSql); + return; + } + + int len = snprintf(sql, size, "insert into %s.slowquery values(now, '%s', %" PRId64 ", %" PRId64 ", '", tsMonitorDbName, pSql->pTscObj->user, pSql->stime, pSql->res.useconds); int sqlLen = snprintf(sql + len, TSDB_SHOW_SQL_LEN, "%s", pSql->sqlstr); if (sqlLen > TSDB_SHOW_SQL_LEN - 1) { @@ -102,8 +112,8 @@ void tscSaveSlowQuery(SSqlObj *pSql) { } else { sqlLen += len; } + strcpy(sql + sqlLen, "')"); - taosTmrStart(tscSaveSlowQueryFp, 200, sql, tscTmr); } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4e05e71a0b..14ab14603f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -163,8 +163,7 @@ static int32_t handlePassword(SSqlCmd* pCmd, SSQLToken* pPwd) { } strdequote(pPwd->z); - strtrim(pPwd->z); // trim space before and after passwords - pPwd->n = strlen(pPwd->z); + pPwd->n = strtrim(pPwd->z); // trim space before and after passwords if (pPwd->n <= 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -226,7 +225,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { assert(pInfo->pDCLInfo->nTokens == 1); - if (tscSetTableId(pTableMetaInfo, pzName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableFullName(pTableMetaInfo, pzName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } else if (pInfo->type == TSDB_SQL_DROP_DNODE) { @@ -353,7 +352,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (tscSetTableId(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableFullName(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -686,7 +685,7 @@ int32_t parseSlidingClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { return TSDB_CODE_SUCCESS; } -int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { const char* msg = "name too long"; SSqlCmd* pCmd = &pSql->cmd; @@ -1465,7 +1464,6 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) { STableMetaInfo* pTableMetaInfo = NULL; - int32_t optr = pItem->pNode->nSQLOptr; const char* msg1 = "not support column types"; @@ -4294,7 +4292,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (tscSetTableId(pTableMetaInfo, &(pAlterSQL->name), pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableFullName(pTableMetaInfo, &(pAlterSQL->name), pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -4671,8 +4669,7 @@ int32_t validateColumnName(char* name) { if (token.type == TK_STRING) { strdequote(token.z); - strtrim(token.z); - token.n = (uint32_t)strlen(token.z); + token.n = strtrim(token.z); int32_t k = tSQLGetToken(token.z, &token.type); if (k != token.n) { @@ -5516,7 +5513,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (tscSetTableId(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableFullName(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -5571,7 +5568,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (tscSetTableId(pStableMeterMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableFullName(pStableMeterMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5617,7 +5614,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } STableMetaInfo* pTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); - ret = tscSetTableId(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql); + ret = tscSetTableFullName(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -5660,7 +5657,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (tscSetTableId(pTableMetaInfo, &srcToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableFullName(pTableMetaInfo, &srcToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -5691,7 +5688,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } // set the created table[stream] name - if (tscSetTableId(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableFullName(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -5800,7 +5797,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { STableMetaInfo* pMeterInfo1 = tscGetMetaInfo(pQueryInfo, i); SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; - if (tscSetTableId(pMeterInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableFullName(pMeterInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 86f9eac11d..defb49c573 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -220,9 +220,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { if (pObj->signature != pObj) { tscTrace("%p sql is already released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed, pObj, pObj->signature); - if (pSql != pObj->pSql) { - tscFreeSqlObj(pSql); - } + tscFreeSqlObj(pSql); rpcFreeCont(rpcMsg->pCont); return; } @@ -257,6 +255,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { rpcMsg->code = TSDB_CODE_NOT_READY; rpcFreeCont(rpcMsg->pCont); return; + } else if (pCmd->command == TSDB_SQL_META) { +// rpcFreeCont(rpcMsg->pCont); +// return; } else { tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); @@ -331,21 +332,11 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) { - void *taosres = tscKeepConn[pCmd->command] ? pSql : NULL; rpcMsg->code = pRes->code ? pRes->code : pRes->numOfRows; tscTrace("%p SQL result:%s res:%p", pSql, tstrerror(pRes->code), pSql); - - /* - * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj - * may be freed in UDF, and reused by other threads before tscShouldBeFreed called, in which case - * tscShouldBeFreed checks an object which is actually allocated by other threads. - * - * If this block of memory is re-allocated for an insert thread, in which tscKeepConn[command] equals to 0, - * the tscShouldBeFreed will success and tscFreeSqlObj free it immediately. - */ bool shouldFree = tscShouldBeFreed(pSql); - (*pSql->fp)(pSql->param, taosres, rpcMsg->code); + (*pSql->fp)(pSql->param, pSql, rpcMsg->code); if (shouldFree) { tscTrace("%p sqlObj is automatically freed", pSql); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index a8e4a077ea..2421dc48db 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -52,7 +52,7 @@ static bool validPassword(const char* passwd) { return validImpl(passwd, TSDB_PASSWORD_LEN); } -STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *db, uint16_t port, +SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { taos_init(); @@ -120,10 +120,8 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con pSql->pTscObj = pObj; pSql->signature = pSql; pSql->maxRetry = TSDB_MAX_REPLICA_NUM; - tsem_init(&pSql->rspSem, 0, 0); - pObj->pSql = pSql; pObj->pDnodeConn = pDnodeConn; pSql->fp = fp; @@ -143,42 +141,38 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con // tsRpcHeaderSize will be updated during RPC initialization, so only after it initialization, this value is valid tsInsertHeadSize = tsRpcHeadSize + sizeof(SMsgDesc) + sizeof(SSubmitMsg); - return pObj; + return pSql; } static void syncConnCallback(void *param, TAOS_RES *tres, int code) { - STscObj *pObj = (STscObj *)param; - assert(pObj != NULL && pObj->pSql != NULL); + SSqlObj *pSql = (SSqlObj *) tres; + assert(pSql != NULL); - if (code < 0) { - pObj->pSql->res.code = code; - } - - sem_post(&pObj->pSql->rspSem); + sem_post(&pSql->rspSem); } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { tscTrace("try to create a connection to %s:%u, user:%s db:%s", ip, port, user, db); - STscObj *pObj = taosConnectImpl(ip, user, pass, db, port, NULL, NULL, NULL); - if (pObj != NULL) { - SSqlObj* pSql = pObj->pSql; - assert(pSql != NULL); - + STscObj* pObj = NULL; + SSqlObj *pSql = taosConnectImpl(ip, user, pass, db, port, syncConnCallback, NULL, (void**) &pObj); + if (pSql != NULL) { pSql->fp = syncConnCallback; - pSql->param = pObj; + pSql->param = pSql; tscProcessSql(pSql); sem_wait(&pSql->rspSem); if (pSql->res.code != TSDB_CODE_SUCCESS) { terrno = pSql->res.code; + taos_free_result(pSql); taos_close(pObj); return NULL; } tscTrace("%p DB connection is opening, dnodeConn:%p", pObj, pObj->pDnodeConn); - + taos_free_result(pSql); + // version compare only requires the first 3 segments of the version string int code = taosCheckVersion(version, taos_get_server_info(pObj), 3); if (code != 0) { @@ -195,17 +189,14 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { - STscObj* pObj = taosConnectImpl(ip, user, pass, db, port, fp, param, taos); - if (pObj == NULL) { + SSqlObj* pSql = taosConnectImpl(ip, user, pass, db, port, fp, param, taos); + if (pSql == NULL) { return NULL; } - SSqlObj* pSql = pObj->pSql; - pSql->res.code = tscProcessSql(pSql); - tscTrace("%p DB async connection is opening", pObj); - - return pObj; + tscTrace("%p DB async connection is opening", taos); + return taos; } void taos_close(TAOS *taos) { @@ -273,33 +264,38 @@ void waitForQueryRsp(void *param, TAOS_RES *tres, int code) { pSql->res.code = code; } + SSqlObj *pSql = (SSqlObj *) tres; sem_post(&pSql->rspSem); } -int taos_query(TAOS *taos, const char *sqlstr) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_DISCONNECTED; - return TSDB_CODE_DISCONNECTED; - } - - SSqlObj* pSql = pObj->pSql; - size_t sqlLen = strlen(sqlstr); - doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); - - // wait for the callback function to post the semaphore - tsem_wait(&pSql->rspSem); - return pSql->res.code; -} - -TAOS_RES *taos_use_result(TAOS *taos) { +TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { terrno = TSDB_CODE_DISCONNECTED; return NULL; } + + int32_t sqlLen = strlen(sqlstr); + if (sqlLen > tsMaxSQLStringLen) { + tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); + terrno = TSDB_CODE_INVALID_SQL; + return NULL; + } + + taosNotePrintTsc(sqlstr); + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { + tscError("failed to malloc sqlObj"); + terrno = TSDB_CODE_CLI_OUT_OF_MEMORY; + return NULL; + } + + doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); - return pObj->pSql; + // wait for the callback function to post the semaphore + tsem_wait(&pSql->rspSem); + return pSql; } int taos_result_precision(TAOS_RES *res) { @@ -332,18 +328,18 @@ int taos_num_fields(TAOS_RES *res) { return num; } -int taos_field_count(TAOS *taos) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) return 0; +int taos_field_count(TAOS_RES *tres) { + SSqlObj* pSql = (SSqlObj*) tres; + if (pSql == NULL || pSql->signature != pSql) return 0; - return taos_num_fields(pObj->pSql); + return taos_num_fields(pSql); } -int taos_affected_rows(TAOS *taos) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) return 0; +int taos_affected_rows(TAOS_RES *tres) { + SSqlObj* pSql = (SSqlObj*) tres; + if (pSql == NULL || pSql->signature != pSql) return 0; - return (pObj->pSql->res.numOfRows); + return (pSql->res.numOfRows); } TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { @@ -385,9 +381,8 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { SSqlObj *pSql = (SSqlObj *)res; SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - STscObj *pObj = pSql->pTscObj; - if (pRes->qhandle == 0 || pObj->pSql != pSql) { + if (pRes->qhandle == 0 || pSql->signature != pSql) { *rows = NULL; return 0; } @@ -521,95 +516,76 @@ int taos_select_db(TAOS *taos, const char *db) { } snprintf(sql, tListLen(sql), "use %s", db); - return taos_query(taos, sql); + SSqlObj* pSql = taos_query(taos, sql); + int32_t code = pSql->res.code; + taos_free_result(pSql); + + return code; } void taos_free_result(TAOS_RES *res) { - if (res == NULL) return; - SSqlObj *pSql = (SSqlObj *)res; + tscTrace("%p start to free result", res); + + if (pSql == NULL || pSql->signature != pSql) { + tscTrace("%p result has been freed", pSql); + return; + } + SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; - tscTrace("%p start to free result", pSql); - - if (pSql->signature != pSql) return; - - STscObj* pObj = pSql->pTscObj; + // The semaphore can not be changed while freeing async sub query objects. if (pRes == NULL || pRes->qhandle == 0) { - /* Query rsp is not received from vnode, so the qhandle is NULL */ - tscTrace("%p qhandle is null, abort free, fp:%p", pSql, pSql->fp); - - // The semaphore can not be changed while freeing async sub query objects. - if (pObj->pSql != pSql) { - tscTrace("%p SqlObj is freed by app", pSql); - tscFreeSqlObj(pSql); - } else { - tscPartiallyFreeSqlObj(pSql); - } - + tscTrace("%p SqlObj is freed by app, qhandle is null", pSql); + tscFreeSqlObj(pSql); return; } // set freeFlag to 1 in retrieve message if there are un-retrieved results data in node SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); if (pQueryInfo == NULL) { - tscPartiallyFreeSqlObj(pSql); + tscFreeSqlObj(pSql); return; } pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; - STscObj* pTscObj = pSql->pTscObj; - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); /* - * case 1. Partial data have been retrieved from vnodes, but not all data has been retrieved yet. - * We need to recycle the connection by noticing the vnode return 0 results. - * case 2. When the query response is received from vnodes and the numOfRows is set to 0, the user calls - * taos_free_result before the taos_fetch_row is called in non-stream computing, - * we need to recycle the connection. - * case 3. If the query process is cancelled by user in stable query, tscProcessSql should not be called - * for each subquery. Because the failure of execution tsProcessSql may trigger the callback function - * be executed, and the retry efforts may result in double free the resources, e.g.,SRetrieveSupport + * If the query process is cancelled by user in stable query, tscProcessSql should not be called + * for each subquery. Because the failure of execution tsProcessSql may trigger the callback function + * be executed, and the retry efforts may result in double free the resources, e.g.,SRetrieveSupport */ - if ((pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_SHOW || - pCmd->command == TSDB_SQL_RETRIEVE || - pCmd->command == TSDB_SQL_FETCH) && pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && + if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && + (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_SHOW || + pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_FETCH) && (pCmd->command == TSDB_SQL_SELECT && pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - tscTrace("%p send msg to free qhandle in vnode, code:%d, numOfRows:%d, command:%s", pSql, pRes->code, pRes->numOfRows, - sqlCmd[pCmd->command]); - + tscTrace("%p start to send msg to free qhandle in dnode, command:%s", pSql, sqlCmd[pCmd->command]); pSql->freed = 1; tscProcessSql(pSql); - - // waits for response and then goes on - if (pTscObj->pSql == pSql) { + + // in case of sync model query, waits for response and then goes on + if (pSql->fp == waitForQueryRsp || pSql->fp == waitForRetrieveRsp) { sem_wait(&pSql->rspSem); } - } else { // if no free resource msg is sent to vnode, we free this object immediately. - if (pTscObj->pSql != pSql) { - tscFreeSqlObj(pSql); - tscTrace("%p sql result is freed by app", pSql); - } else { - tscPartiallyFreeSqlObj(pSql); - tscTrace("%p sql result is freed by app", pSql); - } } + + tscFreeSqlObj(pSql); + tscTrace("%p sql result is freed by app", pSql); } // todo should not be used in async query -int taos_errno(TAOS *taos) { - STscObj *pObj = (STscObj *)taos; +int taos_errno(TAOS_RES *tres) { + SSqlObj *pSql = (SSqlObj *) tres; - if (pObj == NULL || pObj->signature != pObj) { + if (pSql == NULL || pSql->signature != pSql) { return terrno; } - return pObj->pSql->res.code; + return pSql->res.code; } /* @@ -632,14 +608,13 @@ static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd *pCmd) { } // todo should not be used in async model -char *taos_errstr(TAOS *taos) { - STscObj *pObj = (STscObj *)taos; +char *taos_errstr(TAOS_RES *tres) { + SSqlObj *pSql = (SSqlObj *) tres; - if (pObj == NULL || pObj->signature != pObj) - return (char*)tstrerror(terrno); + if (pSql == NULL || pSql->signature != pSql) { + return (char*) tstrerror(terrno); + } - SSqlObj* pSql = pObj->pSql; - if (hasAdditionalErrorInfo(pSql->res.code, &pSql->cmd)) { return pSql->cmd.payload; } else { @@ -769,7 +744,8 @@ int taos_validate_sql(TAOS *taos, const char *sql) { return TSDB_CODE_DISCONNECTED; } - SSqlObj *pSql = pObj->pSql; + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -847,9 +823,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t tblName[len] = '\0'; str = nextStr + 1; - - strtrim(tblName); - len = (uint32_t)strlen(tblName); + len = strtrim(tblName); SSQLToken sToken = {.n = len, .type = TK_ID, .z = tblName}; tSQLGetToken(tblName, &sToken.type); @@ -861,7 +835,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t return code; } - if ((code = tscSetTableId(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscSetTableFullName(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { return code; } @@ -902,7 +876,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { return TSDB_CODE_DISCONNECTED; } - SSqlObj *pSql = pObj->pSql; + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); SSqlRes *pRes = &pSql->res; pRes->numOfTotal = 0; // the number of getting table meta from server diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 1b1aaa54c9..9e1628bb9b 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -459,14 +459,14 @@ static int64_t tscGetLaunchTimestamp(const SSqlStream *pStream) { return (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? timer / 1000L : timer; } -static void setErrorInfo(STscObj* pObj, int32_t code, char* info) { - if (pObj == NULL) { +static void setErrorInfo(SSqlObj* pSql, int32_t code, char* info) { + if (pSql == NULL) { return; } - SSqlCmd* pCmd = &pObj->pSql->cmd; + SSqlCmd* pCmd = &pSql->cmd; - pObj->pSql->res.code = code; + pSql->res.code = code; if (info != NULL) { strncpy(pCmd->payload, info, pCmd->payloadLen); @@ -480,7 +480,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); if (pSql == NULL) { - setErrorInfo(pObj, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL); + setErrorInfo(pSql, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL); return NULL; } @@ -490,14 +490,14 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p SSqlRes *pRes = &pSql->res; int ret = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); if (TSDB_CODE_SUCCESS != ret) { - setErrorInfo(pObj, ret, NULL); + setErrorInfo(pSql, ret, NULL); free(pSql); return NULL; } pSql->sqlstr = strdup(sqlstr); if (pSql->sqlstr == NULL) { - setErrorInfo(pObj, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL); + setErrorInfo(pSql, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL); tfree(pSql); return NULL; @@ -511,7 +511,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p tscResetSqlCmdObj(&pSql->cmd); ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); if (TSDB_CODE_SUCCESS != ret) { - setErrorInfo(pObj, ret, NULL); + setErrorInfo(pSql, ret, NULL); tscError("%p open stream failed, sql:%s, code:%d", pSql, sqlstr, TSDB_CODE_CLI_OUT_OF_MEMORY); tscFreeSqlObj(pSql); return NULL; @@ -521,7 +521,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p SQLInfoDestroy(&SQLInfo); if (pRes->code != TSDB_CODE_SUCCESS) { - setErrorInfo(pObj, pRes->code, pCmd->payload); + setErrorInfo(pSql, pRes->code, pCmd->payload); tscError("%p open stream failed, sql:%s, reason:%s, code:%d", pSql, sqlstr, pCmd->payload, pRes->code); tscFreeSqlObj(pSql); @@ -530,7 +530,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p SSqlStream *pStream = (SSqlStream *)calloc(1, sizeof(SSqlStream)); if (pStream == NULL) { - setErrorInfo(pObj, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL); + setErrorInfo(pSql, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL); tscError("%p open stream failed, sql:%s, reason:%s, code:%d", pSql, sqlstr, pCmd->payload, pRes->code); tscFreeSqlObj(pSql); diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 5831ddad4a..f6cbe4a42e 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -181,21 +181,23 @@ static SArray* getTableList( SSqlObj* pSql ) { const char* p = strstr( pSql->sqlstr, " from " ); char* sql = alloca(strlen(p) + 32); sprintf(sql, "select tbid(tbname)%s", p); - int code = taos_query( pSql->pTscObj, sql ); - if (code != TSDB_CODE_SUCCESS) { - tscError("failed to retrieve table id: %s", tstrerror(code)); + + SSqlObj* pSql1 = taos_query(pSql->pTscObj, sql); + if (terrno != TSDB_CODE_SUCCESS) { + tscError("failed to retrieve table id: %s", tstrerror(terrno)); return NULL; } - TAOS_RES* res = taos_use_result( pSql->pTscObj ); TAOS_ROW row; SArray* result = taosArrayInit( 128, sizeof(STidTags) ); - while ((row = taos_fetch_row(res))) { + while ((row = taos_fetch_row(pSql1))) { STidTags tags; memcpy(&tags, row[0], sizeof(tags)); taosArrayPush(result, &tags); } + taos_free_result(pSql1); + return result; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 99163ba327..1fdbd43408 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1715,6 +1715,7 @@ static void multiVnodeInsertMerge(void* param, TAOS_RES* tres, int numOfRows) { pParentObj->res.numOfRows += numOfRows; } + taos_free_result(tres); int32_t completed = atomic_add_fetch_32(&pState->numOfCompleted, 1); if (completed < total) { return; @@ -1732,7 +1733,7 @@ static void multiVnodeInsertMerge(void* param, TAOS_RES* tres, int numOfRows) { pParentObj->fp = pParentObj->fetchFp; // all data has been sent to vnode, call user function - (*pParentObj->fp)(pParentObj->param, tres, numOfRows); + (*pParentObj->fp)(pParentObj->param, pParentObj, numOfRows); } int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 79c580e1a2..4c158ac77a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -409,8 +409,10 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { } void tscFreeSqlObj(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) return; - + if (pSql == NULL || pSql->signature != pSql) { + return; + } + tscTrace("%p start to free sql object", pSql); tscPartiallyFreeSqlObj(pSql); @@ -424,6 +426,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { pCmd->allocSize = 0; tfree(pSql->sqlstr); + sem_destroy(&pSql->rspSem); free(pSql); } @@ -750,16 +753,7 @@ void tscCloseTscObj(STscObj* pObj) { assert(pObj != NULL); pObj->signature = NULL; - SSqlObj* pSql = pObj->pSql; - - if (pSql) { - terrno = pSql->res.code; - sem_destroy(&pSql->rspSem); - } - taosTmrStopA(&(pObj->pTimer)); - tscFreeSqlObj(pSql); - pthread_mutex_destroy(&pObj->mutex); if (pObj->pDnodeConn != NULL) { @@ -1238,9 +1232,8 @@ void tscColumnListDestroy(SArray* pColumnList) { * */ static int32_t validateQuoteToken(SSQLToken* pToken) { - pToken->n = strdequote(pToken->z); - strtrim(pToken->z); - pToken->n = (uint32_t)strlen(pToken->z); + strdequote(pToken->z); + pToken->n = strtrim(pToken->z); int32_t k = tSQLGetToken(pToken->z, &pToken->type); @@ -1262,9 +1255,8 @@ int32_t tscValidateName(SSQLToken* pToken) { char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); if (sep == NULL) { // single part if (pToken->type == TK_STRING) { - pToken->n = strdequote(pToken->z); - strtrim(pToken->z); - pToken->n = (uint32_t)strlen(pToken->z); + strdequote(pToken->z); + pToken->n = strtrim(pToken->z); int len = tSQLGetToken(pToken->z, &pToken->type); @@ -1289,8 +1281,7 @@ int32_t tscValidateName(SSQLToken* pToken) { char* pStr = pToken->z; if (pToken->type == TK_SPACE) { - strtrim(pToken->z); - pToken->n = (uint32_t)strlen(pToken->z); + pToken->n = strtrim(pToken->z); } pToken->n = tSQLGetToken(pToken->z, &pToken->type); @@ -1471,22 +1462,24 @@ bool tscShouldFreeHeatBeat(SSqlObj* pHb) { * If connection need to be recycled, the SqlObj also should be freed. */ bool tscShouldBeFreed(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql || pSql->fp == NULL) { + if (pSql == NULL || pSql->signature != pSql) { return false; } - + + assert(pSql->fp != NULL); + STscObj* pTscObj = pSql->pTscObj; - if (pSql->pStream != NULL || pTscObj->pHb == pSql || pTscObj->pSql == pSql || pSql->pSubscription != NULL) { + if (pSql->pStream != NULL || pTscObj->pHb == pSql || pSql->pSubscription != NULL) { return false; } + // only the table meta and super table vgroup query will free resource automatically int32_t command = pSql->cmd.command; - if (command == TSDB_SQL_CONNECT || command == TSDB_SQL_INSERT) { + if (command == TSDB_SQL_META || command == TSDB_SQL_STABLEVGROUP) { return true; - } else { - return tscKeepConn[command] == 0 || - (pSql->res.code != TSDB_CODE_ACTION_IN_PROGRESS && pSql->res.code != TSDB_CODE_SUCCESS); } + + return false; } /** @@ -1949,15 +1942,14 @@ int16_t tscGetJoinTagColIndexByUid(STagCond* pTagCond, uint64_t uid) { } } -bool tscIsUpdateQuery(STscObj* pObj) { - if (pObj == NULL || pObj->signature != pObj) { +bool tscIsUpdateQuery(SSqlObj* pSql) { + if (pSql == NULL || pSql->signature != pSql) { terrno = TSDB_CODE_DISCONNECTED; return TSDB_CODE_DISCONNECTED; } - SSqlCmd* pCmd = &pObj->pSql->cmd; - return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || - TSDB_SQL_USE_DB == pCmd->command); + SSqlCmd* pCmd = &pSql->cmd; + return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || TSDB_SQL_USE_DB == pCmd->command); } int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java index 027d2197a3..cbd3523b34 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java @@ -18,8 +18,8 @@ import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; import java.net.URL; -import java.sql.*; import java.sql.Date; +import java.sql.*; import java.util.*; /* @@ -102,41 +102,49 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public byte getByte(int columnIndex) throws SQLException { + columnIndex--; return (byte) rowCursor.getInt(columnIndex, columnMetaDataList.get(columnIndex).getColType()); } @Override public short getShort(int columnIndex) throws SQLException { + columnIndex--; return (short) rowCursor.getInt(columnIndex, columnMetaDataList.get(columnIndex).getColType()); } @Override public int getInt(int columnIndex) throws SQLException { + columnIndex--; return rowCursor.getInt(columnIndex, columnMetaDataList.get(columnIndex).getColType()); } @Override public long getLong(int columnIndex) throws SQLException { + columnIndex--; return rowCursor.getLong(columnIndex, columnMetaDataList.get(columnIndex).getColType()); } @Override public float getFloat(int columnIndex) throws SQLException { + columnIndex--; return rowCursor.getFloat(columnIndex, columnMetaDataList.get(columnIndex).getColType()); } @Override public double getDouble(int columnIndex) throws SQLException { + columnIndex--; return rowCursor.getDouble(columnIndex, columnMetaDataList.get(columnIndex).getColType()); } @Override public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { + columnIndex--; return new BigDecimal(rowCursor.getDouble(columnIndex, columnMetaDataList.get(columnIndex).getColType())); } @Override public byte[] getBytes(int columnIndex) throws SQLException { + columnIndex--; return (rowCursor.getString(columnIndex, columnMetaDataList.get(columnIndex).getColType())).getBytes(); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 3adb601822..6fa9e7bc48 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -99,7 +99,7 @@ public class TSDBJNIConnector { this.taos = this.connectImp(host, port, dbName, user, password); if (this.taos == TSDBConstants.JNI_NULL_POINTER) { - throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg()), "", this.getErrCode()); + throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(null)), "", this.getErrCode(null)); } return true; @@ -117,52 +117,57 @@ public class TSDBJNIConnector { freeResultSet(taosResultSetPointer); } - int code; + long pSql = 0l; try { - code = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos); + pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos); } catch (Exception e) { e.printStackTrace(); + this.freeResultSet(pSql); throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding")); } + int code = this.getErrCode(pSql); + affectedRows = code; if (code < 0) { affectedRows = -1; if (code == TSDBConstants.JNI_TDENGINE_ERROR) { - throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg()), "", this.getErrCode()); + this.freeResultSet(pSql); + throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(pSql)), "", this.getErrCode(pSql)); } else { - throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode()); + this.freeResultSet(pSql); + throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode(pSql)); } } // Try retrieving result set for the executed SQL using the current connection pointer. If the executed // SQL is a DML/DDL which doesn't return a result set, then taosResultSetPointer should be 0L. Otherwise, // taosResultSetPointer should be a non-zero value. - taosResultSetPointer = this.getResultSetImp(this.taos); + taosResultSetPointer = this.getResultSetImp(this.taos, pSql); if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { isResultsetClosed = false; } return code; } - private native int executeQueryImp(byte[] sqlBytes, long connection); + private native long executeQueryImp(byte[] sqlBytes, long connection); /** * Get recent error code by connection */ - public int getErrCode() { - return Math.abs(this.getErrCodeImp(this.taos)); + public int getErrCode(Long pSql) { + return Math.abs(this.getErrCodeImp(this.taos, pSql)); } - private native int getErrCodeImp(long connection); + private native int getErrCodeImp(long connection, Long pSql); /** * Get recent error message by connection */ - public String getErrMsg() { - return this.getErrMsgImp(this.taos); + public String getErrMsg(Long pSql) { + return this.getErrMsgImp(this.taos, pSql); } - private native String getErrMsgImp(long connection); + private native String getErrMsgImp(long connection, Long pSql); /** * Get resultset pointer @@ -172,7 +177,7 @@ public class TSDBJNIConnector { return taosResultSetPointer; } - private native long getResultSetImp(long connection); + private native long getResultSetImp(long connection, long pSql); /** * Free resultset operation from C to release resultset pointer by JNI @@ -212,15 +217,15 @@ public class TSDBJNIConnector { /** * Get affected rows count */ - public int getAffectedRows() { + public int getAffectedRows(Long pSql) { int affectedRows = this.affectedRows; if (affectedRows < 0) { - affectedRows = this.getAffectedRowsImp(this.taos); + affectedRows = this.getAffectedRowsImp(this.taos, pSql); } return affectedRows; } - private native int getAffectedRowsImp(long connection); + private native int getAffectedRowsImp(long connection, Long pSql); /** * Get schema metadata @@ -248,7 +253,7 @@ public class TSDBJNIConnector { public void closeConnection() throws SQLException { int code = this.closeConnectionImp(this.taos); if (code < 0) { - throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode()); + throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode(null)); } else if (code == 0) { this.taos = TSDBConstants.JNI_NULL_POINTER; } else { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index 455b73bb92..dbe1e41e6b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -27,6 +27,8 @@ public class TSDBStatement implements Statement { /** Timeout for a query */ protected int queryTimeout = 0; + private Long pSql = 0l; + /** * Status of current statement */ @@ -66,21 +68,23 @@ public class TSDBStatement implements Statement { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } - int res = this.connecter.executeQuery(sql); + long res = this.connecter.executeQuery(sql); long resultSetPointer = this.connecter.getResultSet(); if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { + this.connecter.freeResultSet(res); throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); } else if (resultSetPointer != TSDBConstants.JNI_NULL_POINTER) { this.connecter.freeResultSet(); throw new SQLException("The executed SQL is not a DML or a DDL"); } else { - return res; + int num = this.connecter.getAffectedRows(res); + return num; } } - public String getErrorMsg() { - return this.connecter.getErrMsg(); + public String getErrorMsg(long pSql) { + return this.connecter.getErrMsg(pSql); } public void close() throws SQLException { @@ -170,7 +174,7 @@ public class TSDBStatement implements Statement { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } - return this.connecter.getAffectedRows(); + return this.connecter.getAffectedRows(this.pSql); } public boolean getMoreResults() throws SQLException { diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 76c0a4be9d..1b17bf72b0 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -142,12 +142,14 @@ class CTaosInterface(object): libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) libtaos.taos_init.restype = None libtaos.taos_connect.restype = ctypes.c_void_p - libtaos.taos_use_result.restype = ctypes.c_void_p + #libtaos.taos_use_result.restype = ctypes.c_void_p libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) libtaos.taos_errstr.restype = ctypes.c_char_p libtaos.taos_subscribe.restype = ctypes.c_void_p libtaos.taos_consume.restype = ctypes.c_void_p libtaos.taos_fetch_lengths.restype = ctypes.c_void_p + libtaos.taos_free_result.restype = None + libtaos.taos_errno.restype = ctypes.c_int def __init__(self, config=None): ''' @@ -249,12 +251,12 @@ class CTaosInterface(object): raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod - def affectedRows(connection): + def affectedRows(result): """The affected rows after runing query """ - return CTaosInterface.libtaos.taos_affected_rows(connection) + return CTaosInterface.libtaos.taos_affected_rows(result) @staticmethod def subscribe(connection, restart, topic, sql, interval): @@ -292,18 +294,17 @@ class CTaosInterface(object): CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0) @staticmethod - def useResult(connection): + def useResult(result): '''Use result after calling self.query ''' - result = ctypes.c_void_p(CTaosInterface.libtaos.taos_use_result(connection)) fields = [] pfields = CTaosInterface.fetchFields(result) - for i in range(CTaosInterface.fieldsCount(connection)): + for i in range(CTaosInterface.fieldsCount(result)): fields.append({'name': pfields[i].name.decode('utf-8'), 'bytes': pfields[i].bytes, 'type': ord(pfields[i].type)}) - return result, fields + return fields @staticmethod def fetchBlock(result, fields): @@ -337,8 +338,8 @@ class CTaosInterface(object): result.value = None @staticmethod - def fieldsCount(connection): - return CTaosInterface.libtaos.taos_field_count(connection) + def fieldsCount(result): + return CTaosInterface.libtaos.taos_field_count(result) @staticmethod def fetchFields(result): @@ -386,29 +387,30 @@ class CTaosInterface(object): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') @staticmethod - def errno(connection): + def errno(result): """Return the error number. """ - return CTaosInterface.libtaos.taos_errno(connection) + return CTaosInterface.libtaos.taos_errno(result) @staticmethod - def errStr(connection): + def errStr(result): """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(connection) + return CTaosInterface.libtaos.taos_errstr(result) if __name__ == '__main__': cinter = CTaosInterface() conn = cinter.connect() + result = cinter.query(conn, 'show databases') - print('Query return value: {}'.format(cinter.query(conn, 'show databases'))) - print('Affected rows: {}'.format(cinter.affectedRows(conn))) + print('Query Affected rows: {}'.format(cinter.affectedRows(result))) - result, des = CTaosInterface.useResult(conn) + fields = CTaosInterface.useResult(result) - data, num_of_rows = CTaosInterface.fetchBlock(result, des) + data, num_of_rows = CTaosInterface.fetchBlock(result, fields) print(data) + cinter.freeresult(result) cinter.close(conn) \ No newline at end of file diff --git a/src/connector/python/linux/python2/taos/connection.py b/src/connector/python/linux/python2/taos/connection.py index 04fbbdec04..552250f116 100644 --- a/src/connector/python/linux/python2/taos/connection.py +++ b/src/connector/python/linux/python2/taos/connection.py @@ -78,9 +78,7 @@ class TDengineConnection(object): def clear_result_set(self): """Clear unused result set on this connection. """ - result = self._chandle.useResult(self._conn)[0] - if result: - self._chandle.freeResult(result) + pass if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') diff --git a/src/connector/python/linux/python2/taos/constants.py b/src/connector/python/linux/python2/taos/constants.py index a994bceaf6..feb7050a40 100644 --- a/src/connector/python/linux/python2/taos/constants.py +++ b/src/connector/python/linux/python2/taos/constants.py @@ -28,6 +28,6 @@ class FieldType(object): C_FLOAT_NULL = float('nan') C_DOUBLE_NULL = float('nan') C_BINARY_NULL = bytearray([int('0xff', 16)]) - # Time precision definition + # Timestamp precision definition C_TIMESTAMP_MILLI = 0 C_TIMESTAMP_MICRO = 1 diff --git a/src/connector/python/linux/python2/taos/cursor.py b/src/connector/python/linux/python2/taos/cursor.py index 7ea52aa5ad..75aa7f69d8 100644 --- a/src/connector/python/linux/python2/taos/cursor.py +++ b/src/connector/python/linux/python2/taos/cursor.py @@ -116,25 +116,30 @@ class TDengineCursor(object): if params is not None: pass - res = CTaosInterface.query(self._connection._conn, stmt) - + # global querySeqNum + # querySeqNum += 1 + # localSeqNum = querySeqNum # avoid raice condition + # print(" >> Exec Query ({}): {}".format(localSeqNum, str(stmt))) + self._result = CTaosInterface.query(self._connection._conn, stmt) + # print(" << Query ({}) Exec Done".format(localSeqNum)) if (self._logfile): with open(self._logfile, "a") as logfile: logfile.write("%s;\n" % operation) - if res == 0: - if CTaosInterface.fieldsCount(self._connection._conn) == 0: + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno == 0: + if CTaosInterface.fieldsCount(self._result) == 0: self._affected_rows += CTaosInterface.affectedRows( - self._connection._conn) - return CTaosInterface.affectedRows(self._connection._conn) + self._result ) + return CTaosInterface.affectedRows(self._result ) else: - self._result, self._fields = CTaosInterface.useResult( - self._connection._conn) + self._fields = CTaosInterface.useResult( + self._result) return self._handle_result() else: raise ProgrammingError( CTaosInterface.errStr( - self._connection._conn)) + self._result )) def executemany(self, operation, seq_of_parameters): """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters. diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index d516aad5a7..87540bf326 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -142,12 +142,14 @@ class CTaosInterface(object): libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) libtaos.taos_init.restype = None libtaos.taos_connect.restype = ctypes.c_void_p - libtaos.taos_use_result.restype = ctypes.c_void_p + #libtaos.taos_use_result.restype = ctypes.c_void_p libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) libtaos.taos_errstr.restype = ctypes.c_char_p libtaos.taos_subscribe.restype = ctypes.c_void_p libtaos.taos_consume.restype = ctypes.c_void_p libtaos.taos_fetch_lengths.restype = ctypes.c_void_p + libtaos.taos_free_result.restype = None + libtaos.taos_errno.restype = ctypes.c_int def __init__(self, config=None): ''' @@ -249,12 +251,12 @@ class CTaosInterface(object): raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod - def affectedRows(connection): + def affectedRows(result): """The affected rows after runing query """ - return CTaosInterface.libtaos.taos_affected_rows(connection) + return CTaosInterface.libtaos.taos_affected_rows(result) @staticmethod def subscribe(connection, restart, topic, sql, interval): @@ -292,18 +294,17 @@ class CTaosInterface(object): CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0) @staticmethod - def useResult(connection): + def useResult(result): '''Use result after calling self.query ''' - result = ctypes.c_void_p(CTaosInterface.libtaos.taos_use_result(connection)) fields = [] pfields = CTaosInterface.fetchFields(result) - for i in range(CTaosInterface.fieldsCount(connection)): + for i in range(CTaosInterface.fieldsCount(result)): fields.append({'name': pfields[i].name.decode('utf-8'), 'bytes': pfields[i].bytes, 'type': ord(pfields[i].type)}) - return result, fields + return fields @staticmethod def fetchBlock(result, fields): @@ -337,8 +338,8 @@ class CTaosInterface(object): result.value = None @staticmethod - def fieldsCount(connection): - return CTaosInterface.libtaos.taos_field_count(connection) + def fieldsCount(result): + return CTaosInterface.libtaos.taos_field_count(result) @staticmethod def fetchFields(result): @@ -386,29 +387,30 @@ class CTaosInterface(object): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') @staticmethod - def errno(connection): + def errno(result): """Return the error number. """ - return CTaosInterface.libtaos.taos_errno(connection) + return CTaosInterface.libtaos.taos_errno(result) @staticmethod - def errStr(connection): + def errStr(result): """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(connection).decode('utf-8') + return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') if __name__ == '__main__': cinter = CTaosInterface() conn = cinter.connect() + result = cinter.query(conn, 'show databases') - print('Query return value: {}'.format(cinter.query(conn, 'show databases'))) - print('Affected rows: {}'.format(cinter.affectedRows(conn))) + print('Query Affected rows: {}'.format(cinter.affectedRows(result))) - result, des = CTaosInterface.useResult(conn) + fields = CTaosInterface.useResult(result) - data, num_of_rows = CTaosInterface.fetchBlock(result, des) + data, num_of_rows = CTaosInterface.fetchBlock(result, fields) print(data) + cinter.freeresult(result) cinter.close(conn) \ No newline at end of file diff --git a/src/connector/python/linux/python3/taos/connection.py b/src/connector/python/linux/python3/taos/connection.py index 04fbbdec04..552250f116 100644 --- a/src/connector/python/linux/python3/taos/connection.py +++ b/src/connector/python/linux/python3/taos/connection.py @@ -78,9 +78,7 @@ class TDengineConnection(object): def clear_result_set(self): """Clear unused result set on this connection. """ - result = self._chandle.useResult(self._conn)[0] - if result: - self._chandle.freeResult(result) + pass if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') diff --git a/src/connector/python/linux/python3/taos/cursor.py b/src/connector/python/linux/python3/taos/cursor.py index dc038a72ba..3baca89462 100644 --- a/src/connector/python/linux/python3/taos/cursor.py +++ b/src/connector/python/linux/python3/taos/cursor.py @@ -122,26 +122,26 @@ class TDengineCursor(object): # querySeqNum += 1 # localSeqNum = querySeqNum # avoid raice condition # print(" >> Exec Query ({}): {}".format(localSeqNum, str(stmt))) - res = CTaosInterface.query(self._connection._conn, stmt) + self._result = CTaosInterface.query(self._connection._conn, stmt) # print(" << Query ({}) Exec Done".format(localSeqNum)) - if (self._logfile): with open(self._logfile, "a") as logfile: logfile.write("%s;\n" % operation) - if res == 0: - if CTaosInterface.fieldsCount(self._connection._conn) == 0: + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno == 0: + if CTaosInterface.fieldsCount(self._result) == 0: self._affected_rows += CTaosInterface.affectedRows( - self._connection._conn) - return CTaosInterface.affectedRows(self._connection._conn) + self._result ) + return CTaosInterface.affectedRows(self._result ) else: - self._result, self._fields = CTaosInterface.useResult( - self._connection._conn) + self._fields = CTaosInterface.useResult( + self._result) return self._handle_result() else: raise ProgrammingError( CTaosInterface.errStr( - self._connection._conn)) + self._result )) def executemany(self, operation, seq_of_parameters): """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters. diff --git a/src/connector/python/linux/python3/test.py b/src/connector/python/linux/python3/test.py new file mode 100644 index 0000000000..b11b2c76a0 --- /dev/null +++ b/src/connector/python/linux/python3/test.py @@ -0,0 +1,21 @@ +from taos.cinterface import CTaosInterface +from taos.error import * +from taos.subscription import TDengineSubscription +from taos.connection import TDengineConnection + + +if __name__ == '__main__': + conn = TDengineConnection( + host="127.0.0.1", user="root", password="taosdata", database="test") + + # Generate a cursor object to run SQL commands + sub = conn.subscribe(False, "test", "select * from log0601;", 1000) + + for i in range(100): + print(i) + data = sub.consume() + for d in data: + print(d) + + sub.close() + conn.close() diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py index c4b8fa8328..9fb7d4ec09 100644 --- a/src/connector/python/windows/python2/taos/cinterface.py +++ b/src/connector/python/windows/python2/taos/cinterface.py @@ -142,12 +142,14 @@ class CTaosInterface(object): libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) libtaos.taos_init.restype = None libtaos.taos_connect.restype = ctypes.c_void_p - libtaos.taos_use_result.restype = ctypes.c_void_p + #libtaos.taos_use_result.restype = ctypes.c_void_p libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) libtaos.taos_errstr.restype = ctypes.c_char_p libtaos.taos_subscribe.restype = ctypes.c_void_p libtaos.taos_consume.restype = ctypes.c_void_p libtaos.taos_fetch_lengths.restype = ctypes.c_void_p + libtaos.taos_free_result.restype = None + libtaos.taos_errno.restype = ctypes.c_int def __init__(self, config=None): ''' @@ -251,10 +253,10 @@ class CTaosInterface(object): # CTaosInterface.libtaos.close(connection) @staticmethod - def affectedRows(connection): + def affectedRows(result): """The affected rows after runing query """ - return CTaosInterface.libtaos.taos_affected_rows(connection) + return CTaosInterface.libtaos.taos_affected_rows(result) @staticmethod def subscribe(connection, restart, topic, sql, interval): @@ -292,18 +294,17 @@ class CTaosInterface(object): CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0) @staticmethod - def useResult(connection): + def useResult(result): '''Use result after calling self.query ''' - result = ctypes.c_void_p(CTaosInterface.libtaos.taos_use_result(connection)) fields = [] pfields = CTaosInterface.fetchFields(result) - for i in range(CTaosInterface.fieldsCount(connection)): + for i in range(CTaosInterface.fieldsCount(result)): fields.append({'name': pfields[i].name.decode('utf-8'), 'bytes': pfields[i].bytes, 'type': ord(pfields[i].type)}) - return result, fields + return fields @staticmethod def fetchBlock(result, fields): @@ -337,8 +338,8 @@ class CTaosInterface(object): result.value = None @staticmethod - def fieldsCount(connection): - return CTaosInterface.libtaos.taos_field_count(connection) + def fieldsCount(result): + return CTaosInterface.libtaos.taos_field_count(result) @staticmethod def fetchFields(result): @@ -386,29 +387,30 @@ class CTaosInterface(object): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') @staticmethod - def errno(connection): + def errno(result): """Return the error number. """ - return CTaosInterface.libtaos.taos_errno(connection) + return CTaosInterface.libtaos.taos_errno(result) @staticmethod - def errStr(connection): + def errStr(result): """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(connection) + return CTaosInterface.libtaos.taos_errstr(result) if __name__ == '__main__': cinter = CTaosInterface() conn = cinter.connect() + result = cinter.query(conn, 'show databases') - print('Query return value: {}'.format(cinter.query(conn, 'show databases'))) - print('Affected rows: {}'.format(cinter.affectedRows(conn))) + print('Query Affected rows: {}'.format(cinter.affectedRows(result))) - result, des = CTaosInterface.useResult(conn) + fields = CTaosInterface.useResult(result) - data, num_of_rows = CTaosInterface.fetchBlock(result, des) + data, num_of_rows = CTaosInterface.fetchBlock(result, fields) print(data) + cinter.freeresult(result) cinter.close(conn) \ No newline at end of file diff --git a/src/connector/python/windows/python2/taos/connection.py b/src/connector/python/windows/python2/taos/connection.py index e2783975d9..d9576a553b 100644 --- a/src/connector/python/windows/python2/taos/connection.py +++ b/src/connector/python/windows/python2/taos/connection.py @@ -79,9 +79,7 @@ class TDengineConnection(object): def clear_result_set(self): """Clear unused result set on this connection. """ - result = self._chandle.useResult(self._conn)[0] - if result: - self._chandle.freeResult(result) + pass if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') diff --git a/src/connector/python/windows/python2/taos/cursor.py b/src/connector/python/windows/python2/taos/cursor.py index 976ec06707..6af82e72de 100644 --- a/src/connector/python/windows/python2/taos/cursor.py +++ b/src/connector/python/windows/python2/taos/cursor.py @@ -109,16 +109,17 @@ class TDengineCursor(object): if params is not None: pass - res = CTaosInterface.query(self._connection._conn, stmt) - if res == 0: - if CTaosInterface.fieldsCount(self._connection._conn) == 0: - self._affected_rows += CTaosInterface.affectedRows(self._connection._conn) - return CTaosInterface.affectedRows(self._connection._conn) + self._result = CTaosInterface.query(self._connection._conn, stmt) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno == 0: + if CTaosInterface.fieldsCount(self._result) == 0: + self._affected_rows += CTaosInterface.affectedRows(self._result) + return CTaosInterface.affectedRows(self._result ) else: - self._result, self._fields = CTaosInterface.useResult(self._connection._conn) + self._fields = CTaosInterface.useResult(self._result) return self._handle_result() else: - raise ProgrammingError(CTaosInterface.errStr(self._connection._conn)) + raise ProgrammingError(CTaosInterface.errStr(self._result)) def executemany(self, operation, seq_of_parameters): """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters. diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py index d8c45f1171..0dbb2ce3a1 100644 --- a/src/connector/python/windows/python3/taos/cinterface.py +++ b/src/connector/python/windows/python3/taos/cinterface.py @@ -142,12 +142,14 @@ class CTaosInterface(object): libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) libtaos.taos_init.restype = None libtaos.taos_connect.restype = ctypes.c_void_p - libtaos.taos_use_result.restype = ctypes.c_void_p + #libtaos.taos_use_result.restype = ctypes.c_void_p libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) libtaos.taos_errstr.restype = ctypes.c_char_p libtaos.taos_subscribe.restype = ctypes.c_void_p libtaos.taos_consume.restype = ctypes.c_void_p libtaos.taos_fetch_lengths.restype = ctypes.c_void_p + libtaos.taos_free_result.restype = None + libtaos.taos_errno.restype = ctypes.c_int def __init__(self, config=None): ''' @@ -251,10 +253,10 @@ class CTaosInterface(object): # CTaosInterface.libtaos.close(connection) @staticmethod - def affectedRows(connection): + def affectedRows(result): """The affected rows after runing query """ - return CTaosInterface.libtaos.taos_affected_rows(connection) + return CTaosInterface.libtaos.taos_affected_rows(result) @staticmethod def subscribe(connection, restart, topic, sql, interval): @@ -292,18 +294,17 @@ class CTaosInterface(object): CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0) @staticmethod - def useResult(connection): + def useResult(result): '''Use result after calling self.query ''' - result = ctypes.c_void_p(CTaosInterface.libtaos.taos_use_result(connection)) fields = [] pfields = CTaosInterface.fetchFields(result) - for i in range(CTaosInterface.fieldsCount(connection)): + for i in range(CTaosInterface.fieldsCount(result)): fields.append({'name': pfields[i].name.decode('utf-8'), 'bytes': pfields[i].bytes, 'type': ord(pfields[i].type)}) - return result, fields + return fields @staticmethod def fetchBlock(result, fields): @@ -337,8 +338,8 @@ class CTaosInterface(object): result.value = None @staticmethod - def fieldsCount(connection): - return CTaosInterface.libtaos.taos_field_count(connection) + def fieldsCount(result): + return CTaosInterface.libtaos.taos_field_count(result) @staticmethod def fetchFields(result): @@ -386,29 +387,30 @@ class CTaosInterface(object): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') @staticmethod - def errno(connection): + def errno(result): """Return the error number. """ - return CTaosInterface.libtaos.taos_errno(connection) + return CTaosInterface.libtaos.taos_errno(result) @staticmethod - def errStr(connection): + def errStr(result): """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(connection).decode('utf-8') + return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') if __name__ == '__main__': cinter = CTaosInterface() conn = cinter.connect() + result = cinter.query(conn, 'show databases') - print('Query return value: {}'.format(cinter.query(conn, 'show databases'))) - print('Affected rows: {}'.format(cinter.affectedRows(conn))) + print('Query Affected rows: {}'.format(cinter.affectedRows(result))) - result, des = CTaosInterface.useResult(conn) + fields = CTaosInterface.useResult(result) - data, num_of_rows = CTaosInterface.fetchBlock(result, des) + data, num_of_rows = CTaosInterface.fetchBlock(result, fields) print(data) + cinter.freeresult(result) cinter.close(conn) \ No newline at end of file diff --git a/src/connector/python/windows/python3/taos/connection.py b/src/connector/python/windows/python3/taos/connection.py index e2783975d9..d9576a553b 100644 --- a/src/connector/python/windows/python3/taos/connection.py +++ b/src/connector/python/windows/python3/taos/connection.py @@ -79,9 +79,7 @@ class TDengineConnection(object): def clear_result_set(self): """Clear unused result set on this connection. """ - result = self._chandle.useResult(self._conn)[0] - if result: - self._chandle.freeResult(result) + pass if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') diff --git a/src/connector/python/windows/python3/taos/cursor.py b/src/connector/python/windows/python3/taos/cursor.py index c7f3400642..0ee8af4ed0 100644 --- a/src/connector/python/windows/python3/taos/cursor.py +++ b/src/connector/python/windows/python3/taos/cursor.py @@ -109,16 +109,17 @@ class TDengineCursor(object): if params is not None: pass - res = CTaosInterface.query(self._connection._conn, stmt) - if res == 0: - if CTaosInterface.fieldsCount(self._connection._conn) == 0: - self._affected_rows += CTaosInterface.affectedRows(self._connection._conn) - return CTaosInterface.affectedRows(self._connection._conn) + self._result = CTaosInterface.query(self._connection._conn, stmt) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno == 0: + if CTaosInterface.fieldsCount(self._result) == 0: + self._affected_rows += CTaosInterface.affectedRows(self._result ) + return CTaosInterface.affectedRows(self._result ) else: - self._result, self._fields = CTaosInterface.useResult(self._connection._conn) + self._fields = CTaosInterface.useResult(self._result ) return self._handle_result() else: - raise ProgrammingError(CTaosInterface.errStr(self._connection._conn)) + raise ProgrammingError(CTaosInterface.errStr(self._result )) def executemany(self, operation, seq_of_parameters): """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters. diff --git a/src/inc/taos.h b/src/inc/taos.h index 2f23b10a61..947cbe6759 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -88,14 +88,13 @@ int taos_stmt_execute(TAOS_STMT *stmt); TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); int taos_stmt_close(TAOS_STMT *stmt); -DLL_EXPORT int taos_query(TAOS *taos, const char *sql); -DLL_EXPORT TAOS_RES *taos_use_result(TAOS *taos); +DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result DLL_EXPORT void taos_free_result(TAOS_RES *res); -DLL_EXPORT int taos_field_count(TAOS *taos); +DLL_EXPORT int taos_field_count(TAOS_RES *tres); DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS *taos); +DLL_EXPORT int taos_affected_rows(TAOS_RES *taos); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); @@ -112,9 +111,9 @@ int* taos_fetch_lengths(TAOS_RES *res); // TODO: the return value should be `const` DLL_EXPORT char *taos_get_server_info(TAOS *taos); DLL_EXPORT char *taos_get_client_info(); -DLL_EXPORT char *taos_errstr(TAOS *taos); +DLL_EXPORT char *taos_errstr(TAOS_RES *tres); -DLL_EXPORT int taos_errno(TAOS *taos); +DLL_EXPORT int taos_errno(TAOS_RES *tres); DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param); DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 7ec9aef295..7d1b3046d9 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -322,7 +322,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_QUERY_TYPE_SUBQUERY 0x02u #define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table -#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side #define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table #define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query #define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index a23db65656..f11a2cf750 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -92,16 +92,15 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NO_USER_FROM_CONN, 0, 0x0185, "can not get // table TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ALREADY_EXIST, 0, 0x0200, "table already exist") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_ID, 0, 0x0201, "invalid table id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_TYPE, 0, 0x0202, "invalid table typee") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_TYPE, 0, 0x0202, "invalid table type") TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPER_TABLE, 0, 0x0203, "no super table") // operation only available for super table TAOS_DEFINE_ERROR(TSDB_CODE_TAG_ALREAY_EXIST, 0, 0x0204, "tag already exist") TAOS_DEFINE_ERROR(TSDB_CODE_TAG_NOT_EXIST, 0, 0x0205, "tag not exist") TAOS_DEFINE_ERROR(TSDB_CODE_FIELD_ALREAY_EXIST, 0, 0x0206, "field already exist") TAOS_DEFINE_ERROR(TSDB_CODE_FIELD_NOT_EXIST, 0, 0x0207, "field not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_COL_NAME_TOO_LONG, 0, 0x0209, "column name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_COL_NAME_TOO_LONG, 0, 0x0208, "column name too long") TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TAGS, 0, 0x0209, "too many tags") - // dnode & mnode TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 0x0280, "no enough dnodes") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ALREADY_EXIST, 0, 0x0281, "dnode already exist") diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 5b4da875de..a01decc6c6 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -275,22 +275,28 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { st = taosGetTimestampUs(); - if (taos_query(con, command)) { - taos_error(con); + TAOS_RES* pSql = taos_query(con, command); + if (taos_errno(pSql)) { + taos_error(pSql); return; } if (regex_match(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); + + taos_free_result(pSql); return; } - int num_fields = taos_field_count(con); + int num_fields = taos_field_count(pSql); if (num_fields != 0) { // select and show kinds of commands int error_no = 0; - int numOfRows = shellDumpResult(con, fname, &error_no, printMode); - if (numOfRows < 0) return; + int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); + if (numOfRows < 0) { + taos_free_result(pSql); + return; + } et = taosGetTimestampUs(); if (error_no == 0) { @@ -299,7 +305,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { printf("Query interrupted (%s), %d row(s) in set (%.6fs)\n", taos_errstr(con), numOfRows, (et - st) / 1E6); } } else { - int num_rows_affacted = taos_affected_rows(con); + int num_rows_affacted = taos_affected_rows(pSql); et = taosGetTimestampUs(); printf("Query OK, %d row(s) affected (%.6fs)\n", num_rows_affacted, (et - st) / 1E6); } @@ -309,6 +315,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { if (fname != NULL) { wordfree(&full_path); } + + taos_free_result(pSql); } /* Function to do regular expression check */ @@ -461,6 +469,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* result) { } while( row != NULL); fclose(fp); + taos_free_result(result); return numOfRows; } @@ -548,15 +557,15 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le } -static int verticalPrintResult(TAOS_RES* result) { - TAOS_ROW row = taos_fetch_row(result); +static int verticalPrintResult(TAOS_RES* tres) { + TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - int num_fields = taos_num_fields(result); - TAOS_FIELD *fields = taos_fetch_fields(result); - int precision = taos_result_precision(result); + int num_fields = taos_num_fields(tres); + TAOS_FIELD *fields = taos_fetch_fields(tres); + int precision = taos_result_precision(tres); int maxColNameLen = 0; for (int col = 0; col < num_fields; col++) { @@ -569,7 +578,7 @@ static int verticalPrintResult(TAOS_RES* result) { int numOfRows = 0; do { printf("*************************** %d.row ***************************\n", numOfRows + 1); - int32_t* length = taos_fetch_lengths(result); + int32_t* length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { TAOS_FIELD* field = fields + i; @@ -581,7 +590,7 @@ static int verticalPrintResult(TAOS_RES* result) { } numOfRows++; - row = taos_fetch_row(result); + row = taos_fetch_row(tres); } while(row != NULL); return numOfRows; @@ -656,15 +665,15 @@ static void printHeader(TAOS_FIELD* fields, int* width, int num_fields) { } -static int horizontalPrintResult(TAOS_RES* result) { - TAOS_ROW row = taos_fetch_row(result); +static int horizontalPrintResult(TAOS_RES* tres) { + TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - int num_fields = taos_num_fields(result); - TAOS_FIELD *fields = taos_fetch_fields(result); - int precision = taos_result_precision(result); + int num_fields = taos_num_fields(tres); + TAOS_FIELD *fields = taos_fetch_fields(tres); + int precision = taos_result_precision(tres); int width[TSDB_MAX_COLUMNS]; for (int col = 0; col < num_fields; col++) { @@ -675,7 +684,7 @@ static int horizontalPrintResult(TAOS_RES* result) { int numOfRows = 0; do { - int32_t* length = taos_fetch_lengths(result); + int32_t* length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { putchar(' '); printField(row[i], fields + i, width[i], length[i], precision); @@ -684,32 +693,24 @@ static int horizontalPrintResult(TAOS_RES* result) { } putchar('\n'); numOfRows++; - row = taos_fetch_row(result); + row = taos_fetch_row(tres); } while(row != NULL); return numOfRows; } -int shellDumpResult(TAOS *con, char *fname, int *error_no, bool vertical) { +int shellDumpResult(TAOS_RES *tres, char *fname, int *error_no, bool vertical) { int numOfRows = 0; - - TAOS_RES* result = taos_use_result(con); - if (result == NULL) { - taos_error(con); - return -1; - } - if (fname != NULL) { - numOfRows = dumpResultToFile(fname, result); + numOfRows = dumpResultToFile(fname, tres); } else if(vertical) { - numOfRows = verticalPrintResult(result); + numOfRows = verticalPrintResult(tres); } else { - numOfRows = horizontalPrintResult(result); + numOfRows = horizontalPrintResult(tres); } - *error_no = taos_errno(con); - taos_free_result(result); + *error_no = taos_errno(tres); return numOfRows; } @@ -771,12 +772,11 @@ void write_history() { fclose(f); } -void taos_error(TAOS *con) { - fprintf(stderr, "\nDB error: %s\n", taos_errstr(con)); +void taos_error(TAOS_RES *tres) { + fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres)); /* free local resouce: allocated memory/metric-meta refcnt */ - TAOS_RES *pRes = taos_use_result(con); - taos_free_result(pRes); + taos_free_result(tres); } int isCommentLine(char *line) { @@ -858,8 +858,9 @@ void shellGetGrantInfo(void *con) { char sql[] = "show grants"; - int code = taos_query(con, sql); - + TAOS_RES* pSql = taos_query(con, sql); + int code = taos_errno(pSql); + if (code != TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_OPS_NOT_SUPPORT) { fprintf(stdout, "Server is Community Edition, version is %s\n\n", taos_get_server_info(con)); @@ -869,12 +870,11 @@ void shellGetGrantInfo(void *con) { return; } - int num_fields = taos_field_count(con); + int num_fields = taos_field_count(result); if (num_fields == 0) { fprintf(stderr, "\nInvalid grant information.\n"); exit(0); } else { - result = taos_use_result(con); if (result == NULL) { fprintf(stderr, "\nGrant information is null.\n"); exit(0); diff --git a/src/kit/shell/src/shellImport.c b/src/kit/shell/src/shellImport.c index 70660254a5..1dea6bca70 100644 --- a/src/kit/shell/src/shellImport.c +++ b/src/kit/shell/src/shellImport.c @@ -192,11 +192,14 @@ static void shellSourceFile(TAOS *con, char *fptr) { } memcpy(cmd + cmd_len, line, read_len); - if (taos_query(con, cmd)) { + + TAOS_RES* pSql = taos_query(con, cmd); + int32_t code = taos_errno(pSql); + + if (code != 0) { fprintf(stderr, "DB error: %s: %s (%d)\n", taos_errstr(con), fname, lineNo); /* free local resouce: allocated memory/metric-meta refcnt */ - TAOS_RES *pRes = taos_use_result(con); - taos_free_result(pRes); + taos_free_result(pSql); } memset(cmd, 0, MAX_COMMAND_SIZE); diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 3485a964d7..1fbe04208c 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -18,21 +18,21 @@ #include "tsclient.h" #include "tutil.h" -TAOS* con; +TAOS_RES* con; pthread_t pid; // TODO: IMPLEMENT INTERRUPT HANDLER. void interruptHandler(int signum) { #ifdef LINUX - TAOS_RES* res = taos_use_result(con); - taos_stop_query(res); - if (res != NULL) { + taos_stop_query(con); + if (con != NULL) { /* * we need to free result in async model, in order to avoid free * results while the master thread is waiting for server response. */ - tscQueueAsyncFreeResult(res); + tscQueueAsyncFreeResult(con); } + result = NULL; #else printf("\nReceive ctrl+c or other signal, quit shell.\n"); @@ -90,7 +90,6 @@ int main(int argc, char* argv[]) { /* Initialize the shell */ con = shellInit(&args); if (con == NULL) { - taos_error(con); exit(EXIT_FAILURE); } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index cf84ecad43..81426b683a 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -461,8 +461,7 @@ int main(int argc, char *argv[]) { taos_init(); TAOS *taos = taos_connect(ip_addr, user, pass, NULL, port); if (taos == NULL) { - fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(taos)); - taos_close(taos); + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); return 1; } char command[BUFFER_SIZE] = "\0"; @@ -708,27 +707,24 @@ void *readTable(void *sarg) { sprintf(command, "select %s from %s%d where ts>= %" PRId64, aggreFunc[j], tb_prefix, i, sTime); double t = getCurrentTime(); - if (taos_query(taos, command) != 0) { - fprintf(stderr, "Failed to query\n"); + TAOS_RES *pSql = taos_query(taos, command); + int32_t code = taos_errno(pSql); + + if (code != 0) { + fprintf(stderr, "Failed to query:%s\n", taos_errstr(taos)); + taos_free_result(pSql); taos_close(taos); exit(EXIT_FAILURE); } - TAOS_RES *result = taos_use_result(taos); - if (result == NULL) { - fprintf(stderr, "Failed to retreive results:%s\n", taos_errstr(taos)); - taos_close(taos); - exit(1); - } - - while (taos_fetch_row(result) != NULL) { + while (taos_fetch_row(pSql) != NULL) { count++; } t = getCurrentTime() - t; totalT += t; - taos_free_result(result); + taos_free_result(pSql); } fprintf(fp, "|%10s | %10d | %12.2f | %10.2f |\n", @@ -779,20 +775,18 @@ void *readMetric(void *sarg) { fprintf(fp, "%s\n", command); double t = getCurrentTime(); - if (taos_query(taos, command) != 0) { - fprintf(stderr, "Failed to query\n"); - taos_close(taos); - exit(EXIT_FAILURE); - } - TAOS_RES *result = taos_use_result(taos); - if (result == NULL) { - fprintf(stderr, "Failed to retreive results:%s\n", taos_errstr(taos)); + TAOS_RES *pSql = taos_query(taos, command); + int32_t code = taos_errno(pSql); + + if (code != 0) { + fprintf(stderr, "Failed to query:%s\n", taos_errstr(taos)); + taos_free_result(pSql); taos_close(taos); exit(1); } int count = 0; - while (taos_fetch_row(result) != NULL) { + while (taos_fetch_row(pSql) != NULL) { count++; } t = getCurrentTime() - t; @@ -800,7 +794,7 @@ void *readMetric(void *sarg) { fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", num_of_tables * num_of_DPT / t, t * 1000); printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t); - taos_free_result(result); + taos_free_result(pSql); } fprintf(fp, "\n"); } @@ -811,10 +805,19 @@ void *readMetric(void *sarg) { void queryDB(TAOS *taos, char *command) { int i = 5; - while (i > 0) { - if (taos_query(taos, command) == 0) break; + TAOS_RES *pSql = NULL; + int32_t code = -1; + while (i > 0 && code != 0) { + pSql = taos_query(taos, command); + code = taos_errno(pSql); + taos_free_result(pSql); + pSql = NULL; + if (code == 0) { + break; + } i--; } + if (i == 0) { fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(taos)); taos_close(taos); @@ -947,6 +950,7 @@ void callBack(void *param, TAOS_RES *res, int code) { break; } } + tb_info->timestamp = tmp_time; taos_query_a(tb_info->taos, buffer, callBack, tb_info); diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 9886a91f48..7875ef732c 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -372,14 +372,12 @@ int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo) { memset(pTableRecordInfo, 0, sizeof(STableRecordInfo)); sprintf(command, "show tables like %s", table); - if (taos_query(taos, command) != 0) { - fprintf(stderr, "failed to run command %s\n", command); - return -1; - } + TAOS_RES *result = taos_query(taos, command);\ + int32_t code = taos_errno(result); - result = taos_use_result(taos); - if (result == NULL) { - fprintf(stderr, "failed to use result\n"); + if (code != 0) { + fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); + taos_free_result(result); return -1; } @@ -400,14 +398,12 @@ int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo) { if (isSet) return 0; sprintf(command, "show stables like %s", table); - if (taos_query(taos, command) != 0) { - fprintf(stderr, "failed to run command %s\n", command); - return -1; - } - result = taos_use_result(taos); - if (result == NULL) { - fprintf(stderr, "failed to use result\n"); + result = taos_query(taos, command); + code = taos_errno(result); + if (code != 0) { + fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); + taos_free_result(result); return -1; } @@ -467,14 +463,11 @@ int taosDumpOut(SDumpArguments *arguments) { taosDumpCharset(fp); sprintf(command, "show databases"); - if (taos_query(taos, command) != 0) { + result = taos_query(taos, command); + int32_t code = taos_errno(result); + if (code != 0) { fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(taos)); - goto _exit_failure; - } - - result = taos_use_result(taos); - if (result == NULL) { - fprintf(stderr, "failed to use result\n"); + taos_free_result(result); goto _exit_failure; } @@ -551,7 +544,7 @@ int taosDumpOut(SDumpArguments *arguments) { taosDumpCreateDbClause(dbInfos[0], arguments->with_property, fp); sprintf(command, "use %s", dbInfos[0]->name); - if (taos_query(taos, command) != 0) { + if (taos_query(taos, command) == NULL ) { fprintf(stderr, "invalid database %s\n", dbInfos[0]->name); goto _exit_failure; } @@ -612,7 +605,7 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) { taosDumpCreateDbClause(dbInfo, arguments->with_property, fp); sprintf(command, "use %s", dbInfo->name); - if (taos_query(taos, command) != 0) { + if (taos_errno(taos_query(taos, command)) != 0) { fprintf(stderr, "invalid database %s\n", dbInfo->name); return -1; } @@ -620,14 +613,11 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) { fprintf(fp, "USE %s\n\n", dbInfo->name); sprintf(command, "show tables"); - if (taos_query(taos, command) != 0) { - fprintf(stderr, "failed to run command %s\n", command); - return -1; - } - - result = taos_use_result(taos); - if (result == NULL) { - fprintf(stderr, "failed to use result\n"); + result = taos_query(taos,command); + int32_t code = taos_errno(result); + if (code != 0) { + fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); + taos_free_result(result); return -1; } @@ -725,14 +715,11 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols TAOS_ROW row = NULL; sprintf(command, "select %s from %s limit 1", tableDes->cols[counter].field, tableDes->name); - if (taos_query(taos, command) != 0) { - fprintf(stderr, "failed to run command %s\n", command); - return; - } - result = taos_use_result(taos); - if (result == NULL) { - fprintf(stderr, "failed to use result\n"); + result = taos_query(taos, command); + int32_t code = taos_errno(result); + if (code != 0) { + fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); return; } @@ -806,14 +793,12 @@ int taosGetTableDes(char *table, STableDef *tableDes) { int count = 0; sprintf(command, "describe %s", table); - if (taos_query(taos, command) != 0) { - fprintf(stderr, "failed to run command %s\n", command); - return -1; - } - result = taos_use_result(taos); - if (result == NULL) { - fprintf(stderr, "failed to use result\n"); + result = taos_query(taos, command); + int32_t code = taos_errno(result); + if (code != 0) { + fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); + taos_free_result(result); return -1; } @@ -889,14 +874,11 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) { strcpy(tableRecord.metric, metric); sprintf(command, "select tbname from %s", metric); - if (taos_query(taos, command) != 0) { - fprintf(stderr, "failed to run command %s\n", command); - return -1; - } - - result = taos_use_result(taos); - if (result == NULL) { - fprintf(stderr, "failed to use result\n"); + result = taos_query(taos, command); + int32_t code = taos_errno(result); + if (code != 0) { + fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result)); + taos_free_result(result); return -1; } @@ -942,18 +924,16 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) { sprintf(command, "select * from %s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc", tbname, arguments->start_time, arguments->end_time); - if (taos_query(taos, command) != 0) { - fprintf(stderr, "failed to run command %s, reason: %s\n", command, taos_errstr(taos)); + + result = taos_query(taos, command); + int32_t code = taos_errno(result); + if (code != 0) { + fprintf(stderr, "failed to run command %s, reason: %s\n", command, taos_errstr(result)); + taos_free_result(result); return -1; } - result = taos_use_result(taos); - if (result == NULL) { - fprintf(stderr, "failed to use result\n"); - return -1; - } - - numFields = taos_field_count(taos); + numFields = taos_field_count(result); assert(numFields > 0); TAOS_FIELD *fields = taos_fetch_fields(result); tbuf = (char *)malloc(COMMAND_SIZE); @@ -1194,7 +1174,7 @@ int taosDumpIn(SDumpArguments *arguments) { tcommand = command; } taosReplaceCtrlChar(tcommand); - if (taos_query(taos, tcommand) != 0) + if (taos_query(taos, tcommand) == NULL) fprintf(stderr, "linenu: %" PRId64 " failed to run command %s reason:%s \ncontinue...\n", linenu, command, taos_errstr(taos)); @@ -1242,9 +1222,14 @@ int taosDumpIn(SDumpArguments *arguments) { tcommand = command; } taosReplaceCtrlChar(tcommand); - if (taos_query(taos, tcommand) != 0) + result = taos_query(taos, tcommand); + int32_t code = taos_errno(result); + if (code != 0) + { fprintf(stderr, "linenu:%" PRId64 " failed to run command %s reason: %s \ncontinue...\n", linenu, command, taos_errstr(taos)); + } + taos_free_result(result); } pstr = command; @@ -1265,7 +1250,7 @@ int taosDumpIn(SDumpArguments *arguments) { tcommand = command; } taosReplaceCtrlChar(lcommand); - if (taos_query(taos, tcommand) != 0) + if (taos_query(taos, tcommand) == NULL) fprintf(stderr, "linenu:%" PRId64 " failed to run command %s reason:%s \ncontinue...\n", linenu, command, taos_errstr(taos)); } diff --git a/src/os/linux/src/linuxSysPara.c b/src/os/linux/src/linuxSysPara.c index 5794ad0302..af2d3c2633 100644 --- a/src/os/linux/src/linuxSysPara.c +++ b/src/os/linux/src/linuxSysPara.c @@ -245,10 +245,10 @@ static void taosGetSystemLocale() { // get and set default locale strncpy(tsCharset, revisedCharset, tListLen(tsCharset)); free(revisedCharset); - uError("charset not configured, set to system default:%s", tsCharset); + uWarn("charset not configured, set to system default:%s", tsCharset); } else { strcpy(tsCharset, "UTF-8"); - uError("can't get locale and charset from system, set it to UTF-8"); + uWarn("can't get locale and charset from system, set it to UTF-8"); } } } diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 45cc3fcfdf..c82a235eba 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -58,10 +58,10 @@ void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numO if (numOfRows < 0) { httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(numOfRows), sql); - } else { - taos_free_result(result); - } - + } + + taos_free_result(result); + if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->stopJsonFp) { (encode->stopJsonFp)(pContext, singleCmd); } @@ -103,12 +103,15 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) { } multiCmds->pos++; httpProcessMultiSql(pContext); + + taos_free_result(result); return; } - if (result == NULL) { + int num_fields = taos_field_count(result); + if (num_fields == 0) { // not select or show commands - int affectRows = code; + int affectRows = taos_affected_rows(result); httpTrace("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, affect rows:%d, sql:%s", pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, affectRows, sql); @@ -132,6 +135,7 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) { multiCmds->pos++; } + taos_free_result(result); httpProcessMultiSql(pContext); } else { httpTrace("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, start retrieve, sql:%s", @@ -212,9 +216,9 @@ void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int num if (numOfRows < 0) { httpError("context:%p, fd:%d, ip:%s, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->ipstr, pContext->user, tstrerror(numOfRows)); - } else { - taos_free_result(result); - } + } + + taos_free_result(result); if (encode->stopJsonFp) { (encode->stopJsonFp)(pContext, &pContext->singleCmd); @@ -247,12 +251,14 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) { pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj); httpSendTaosdErrorResp(pContext, code); } + taos_free_result(result); return; } - if (result == NULL) { + int num_fields = taos_field_count(result); + if (num_fields == 0) { // not select or show commands - int affectRows = code; + int affectRows = taos_affected_rows(result); httpTrace("context:%p, fd:%d, ip:%s, user:%s, affect rows:%d, stop query, sqlObj:%p", pContext, pContext->fd, pContext->ipstr, pContext->user, affectRows, result); @@ -269,6 +275,7 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) { (encode->stopJsonFp)(pContext, &pContext->singleCmd); } + taos_free_result(result); httpCloseContextByApp(pContext); } else { httpTrace("context:%p, fd:%d, ip:%s, user:%s, start retrieve", pContext, pContext->fd, pContext->ipstr, diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 722a625526..7ea272b8e8 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -116,11 +116,14 @@ typedef struct SQueryCostInfo { uint64_t loadDataInCacheSize; uint64_t loadDataTime; - uint64_t dataInRows; - uint64_t checkRows; - uint32_t dataBlocks; + uint64_t totalRows; + uint64_t totalCheckedRows; + uint32_t totalBlocks; + uint32_t loadBlocks; uint32_t loadBlockStatis; uint32_t discardBlocks; + uint64_t elapsedTime; + uint64_t computTime; } SQueryCostInfo; typedef struct SGroupItem { @@ -168,7 +171,7 @@ typedef struct SQueryRuntimeEnv { SWindowResInfo windowResInfo; STSBuf* pTSBuf; STSCursor cur; - SQueryCostInfo summary; + SQueryCostInfo summary; bool stableQuery; // super table query or not void* pQueryHandle; void* pSecQueryHandle; // another thread for @@ -177,8 +180,6 @@ typedef struct SQueryRuntimeEnv { typedef struct SQInfo { void* signature; - TSKEY startTime; - TSKEY elapsedTime; int32_t pointsInterpo; int32_t code; // error code to returned to client sem_t dataReady; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index cf63c3a078..ef1ad10f2b 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1183,6 +1183,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl STableQueryInfo* pTableQInfo = pQuery->current; SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); @@ -1190,10 +1191,10 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } + // update the lastkey of current table TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; pTableQInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - // interval query with limit applied int32_t numOfRes = 0; @@ -2013,7 +2014,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, if (*pStatis == NULL) { // data block statistics does not exist, load data block pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); - pRuntimeEnv->summary.checkRows += pBlockInfo->rows; + pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; } } else { assert(r == BLK_DATA_ALL_NEEDED); @@ -2032,7 +2033,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, // return DISK_DATA_DISCARDED; } - pRuntimeEnv->summary.checkRows += pBlockInfo->rows; + pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); } @@ -2149,7 +2150,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { - pRuntimeEnv->summary.dataBlocks += 1; + pRuntimeEnv->summary.totalBlocks += 1; if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return 0; } @@ -2185,12 +2186,10 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { ensureOutputBuffer(pRuntimeEnv, &blockInfo); SDataStatis *pStatis = NULL; - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : blockInfo.rows - 1; - SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis); int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); - pRuntimeEnv->summary.dataInRows += blockInfo.rows; + pRuntimeEnv->summary.totalRows += blockInfo.rows; qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey); @@ -3253,7 +3252,7 @@ void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) free(pTableQueryInfo); } -void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) { +void setCurrentQueryTable(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->current = pTableQueryInfo; @@ -3322,7 +3321,7 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * int32_t setAdditionalInfo(SQInfo *pQInfo, STableId* pTableId, STableQueryInfo *pTableQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - assert(pTableQueryInfo->lastKey >= 0); + assert(pTableQueryInfo->lastKey >= TSKEY_INITIAL_VAL); setTagVal(pRuntimeEnv, pTableId, pQInfo->tsdb); @@ -3534,10 +3533,11 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableQueryIn } } -void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo, - SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray *pDataBlock, - __block_search_fn_t searchFn) { +void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, + SArray *pDataBlock, __block_search_fn_t searchFn) { SQuery * pQuery = pRuntimeEnv->pQuery; + STableQueryInfo* pTableQueryInfo = pQuery->current; + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; @@ -3670,10 +3670,8 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int } } -void queryCostStatis(SQInfo *pQInfo) { +static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; -// SQuery *pQuery = pRuntimeEnv->pQuery; - SQueryCostInfo *pSummary = &pRuntimeEnv->summary; // if (pRuntimeEnv->pResultBuf == NULL) { //// pSummary->tmpBufferInDisk = 0; @@ -3693,8 +3691,9 @@ void queryCostStatis(SQInfo *pQInfo) { // pQInfo, pSummary->readDiskBlocks, pSummary->totalBlockSize, pSummary->loadBlocksUs / 1000.0, // pSummary->skippedFileBlocks, pSummary->totalGenData); - qTrace("QInfo:%p cost: check blocks:%d, statis:%d, rows:%"PRId64", check rows:%"PRId64, pQInfo, pSummary->dataBlocks, - pSummary->loadBlockStatis, pSummary->dataInRows, pSummary->checkRows); + qTrace("QInfo:%p :cost summary: elpased time:%"PRId64" us, total blocks:%d, use block statis:%d, use block data:%d, " + "total rows:%"PRId64 ", check rows:%"PRId64, pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, + pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); // qTrace("QInfo:%p cost: temp file:%d Bytes", pQInfo, pSummary->tmpBufferInDisk); // @@ -4090,12 +4089,13 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - + SQueryCostInfo* summary = &pRuntimeEnv->summary; + int64_t st = taosGetTimestampMs(); TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; - while (tsdbNextDataBlock(pQueryHandle)) { + summary->totalBlocks += 1; if (isQueryKilled(pQInfo)) { break; } @@ -4127,10 +4127,9 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { } assert(pTableQueryInfo != NULL); - restoreIntervalQueryRange(pRuntimeEnv, pTableQueryInfo); + setCurrentQueryTable(pRuntimeEnv, pTableQueryInfo); SDataStatis *pStatis = NULL; - SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis); if (!isIntervalQuery(pQuery)) { @@ -4139,15 +4138,14 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { } else { // interval query TSKEY nextKey = blockInfo.window.skey; setIntervalQueryRange(pQInfo, nextKey); - int32_t ret = setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo); - - if (ret != TSDB_CODE_SUCCESS) { - pQInfo->code = ret; - return taosGetTimestampMs() - st; - } + /*int32_t ret = */setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo); } - stableApplyFunctionsOnBlock(pRuntimeEnv, pTableQueryInfo, &blockInfo, pStatis, pDataBlock, binarySearchForKey); + summary->totalRows += blockInfo.rows; + stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey); + + qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), + blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, pQuery->current->lastKey); } int64_t et = taosGetTimestampMs(); @@ -4509,10 +4507,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); } - if (pQuery->rec.rows == 0) { - // queryCostStatis(pSupporter); - } - qTrace("QInfo:%p current:%lld, total:%lld", pQInfo, pQuery->rec.rows, pQuery->rec.total); return; } @@ -4795,7 +4789,6 @@ static void tableQueryImpl(SQInfo *pQInfo) { } qTrace("QInfo:%p query over, %d rows are returned", pQInfo, pQuery->rec.total); - queryCostStatis(pQInfo); return; } @@ -4818,7 +4811,7 @@ static void tableQueryImpl(SQInfo *pQInfo) { } // record the total elapsed time - pQInfo->elapsedTime += (taosGetTimestampUs() - st); + pRuntimeEnv->summary.elapsedTime += (taosGetTimestampUs() - st); assert(pQInfo->groupInfo.numOfTables == 1); /* check if query is killed or not */ @@ -4827,10 +4820,6 @@ static void tableQueryImpl(SQInfo *pQInfo) { } else {// todo set the table uid and tid in log qTrace("QInfo:%p query paused, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); - - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - queryCostStatis(pQInfo); - } } } @@ -4852,13 +4841,10 @@ static void stableQueryImpl(SQInfo *pQInfo) { } // record the total elapsed time - pQInfo->elapsedTime += (taosGetTimestampUs() - st); - // taosFillSetStartInfo(&pQInfo->runtimeEnv.pFillInfo, pQuery->size, pQInfo->query.fillType); - + pQInfo->runtimeEnv.summary.elapsedTime += (taosGetTimestampUs() - st); + if (pQuery->rec.rows == 0) { - qTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, pQInfo->groupInfo.numOfTables, - pQuery->rec.total); - // queryCostStatis(pSupporter); + qTrace("QInfo:%p over, %d tables queried, %d rows are returned", pQInfo, pQInfo->groupInfo.numOfTables, pQuery->rec.total); } } @@ -5838,9 +5824,11 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi numOfGroupByCols = 0; } - // todo handle the error - /*int32_t ret =*/tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex, + code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex, numOfGroupByCols); + if (code != TSDB_CODE_SUCCESS) { + goto _over; + } } else { SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); @@ -5880,6 +5868,9 @@ _over: void qDestroyQueryInfo(qinfo_t pQInfo) { qTrace("QInfo:%p query completed", pQInfo); + + // print the query cost summary + queryCostStatis(pQInfo); freeQInfo(pQInfo); } @@ -5956,6 +5947,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co return TSDB_CODE_INVALID_QHANDLE; } + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; size_t size = getResultSize(pQInfo, &pQuery->rec.rows); size += sizeof(int32_t); @@ -5969,7 +5961,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co int32_t code = pQInfo->code; if (code == TSDB_CODE_SUCCESS) { (*pRsp)->offset = htobe64(pQuery->limit.offset); - (*pRsp)->useconds = htobe64(pQInfo->elapsedTime); + (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } else { (*pRsp)->offset = 0; (*pRsp)->useconds = 0; diff --git a/src/query/src/qfill.c b/src/query/src/qfill.c index 8c8a50a3d8..7b3ea5c1f0 100644 --- a/src/query/src/qfill.c +++ b/src/query/src/qfill.c @@ -110,6 +110,14 @@ void taosDestoryFillInfo(SFillInfo* pFillInfo) { tfree(pFillInfo->prevValues); tfree(pFillInfo->nextValues); tfree(pFillInfo->pTags); + + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + tfree(pFillInfo->pData[i]); + } + + tfree(pFillInfo->pData); + tfree(pFillInfo->pFillCol); + tfree(pFillInfo); } @@ -247,7 +255,7 @@ int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoi } static void setTagsValue(SFillInfo* pColInfo, tFilePage** data, char** pTags, int32_t start, int32_t num) { - for (int32_t j = 0, i = start; i < pColInfo->numOfCols + pColInfo->numOfTags; ++i, ++j) { + for (int32_t j = 0, i = start; i < pColInfo->numOfCols; ++i, ++j) { SFillColInfo* pCol = &pColInfo->pFillCol[i]; char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, num); @@ -344,7 +352,7 @@ static void doInterpoResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* setTagsValue(pFillInfo, data, pTags, numOfValCols, *num); } - } else { /* default value interpolation */ + } else { /* fill the default value */ for (int32_t i = 1; i < numOfValCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 46cc897eac..4a48bcefe7 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -193,7 +193,7 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable } } - uTrace("%p total numOfTable:%d in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo)); + tsdbTrace("%p total numOfTable:%d in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo)); tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); @@ -282,10 +282,10 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer - uTrace("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle, + tsdbTrace("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); } else { - uTrace("%p uid:%" PRId64 ", tid:%d no data in mem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid); + tsdbTrace("%p uid:%" PRId64 ", tid:%d no data in mem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid); } if (!imemEmpty) { @@ -294,10 +294,10 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer - uTrace("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle, + tsdbTrace("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); } else { - uTrace("%p uid:%"PRId64", tid:%d no data in imem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid); + tsdbTrace("%p uid:%"PRId64", tid:%d no data in imem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid); } return true; @@ -338,7 +338,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { SDataRow row = SL_GET_NODE_DATA(node); pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer - uTrace("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, %p", pHandle, + tsdbTrace("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qinfo); // all data in mem are checked already. @@ -1038,7 +1038,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->rows = numOfRows; cur->pos = pos; - uTrace("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" %p", pQueryHandle, cur->win.skey, + tsdbTrace("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" %p", pQueryHandle, cur->win.skey, cur->win.ekey, cur->rows, pQueryHandle->qinfo); } @@ -1138,7 +1138,7 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* if (pLeftBlockInfoEx->compBlock->offset == pRightBlockInfoEx->compBlock->offset && pLeftBlockInfoEx->compBlock->last == pRightBlockInfoEx->compBlock->last) { // todo add more information - uError("error in header file, two block with same offset:%p", pLeftBlockInfoEx->compBlock->offset); + tsdbError("error in header file, two block with same offset:%p", pLeftBlockInfoEx->compBlock->offset); } return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1; @@ -1200,7 +1200,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO numOfQualTables++; } - uTrace("%p create data blocks info struct completed, %d blocks in %d tables", pQueryHandle, cnt, numOfQualTables); + tsdbTrace("%p create data blocks info struct completed, %d blocks in %d tables", pQueryHandle, cnt, numOfQualTables); assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 sup.numOfTables = numOfQualTables; @@ -1236,7 +1236,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO * } */ - uTrace("%p %d data blocks sort completed", pQueryHandle, cnt); + tsdbTrace("%p %d data blocks sort completed", pQueryHandle, cnt); cleanBlockOrderSupporter(&sup, numOfTables); free(pTree); @@ -1257,7 +1257,7 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) { break; } - uTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, + tsdbTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, numOfTables, pQueryHandle->pFileGroup->fileId); assert(numOfBlocks >= 0); @@ -1583,7 +1583,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - uTrace("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, + tsdbTrace("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, pQueryHandle->window.ekey); break; @@ -1958,7 +1958,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC size_t size = taosArrayGetSize(pTableList); if (size == 0) { - uTrace("no qualified tables"); + tsdbTrace("no qualified tables"); return pTableGroup; } @@ -1970,7 +1970,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC } taosArrayPush(pTableGroup, &sa); - uTrace("all %d tables belong to one group", size); + tsdbTrace("all %d tables belong to one group", size); } else { STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter)); pSupp->tsdbMeta = tsdbGetMeta(tsdb); @@ -2069,12 +2069,12 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT* tsdb, uint64_t uid, const char* pTag SColIndex* pColIndex, int32_t numOfCols) { STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); if (pTable == NULL) { - uError("%p failed to get stable, uid:%" PRIu64, tsdb, uid); + tsdbError("%p failed to get stable, uid:%" PRIu64, tsdb, uid); return TSDB_CODE_INVALID_TABLE_ID; } if (pTable->type != TSDB_SUPER_TABLE) { - uError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", + tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid, pTable->name); return TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client @@ -2090,7 +2090,7 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT* tsdb, uint64_t uid, const char* pTag pGroupInfo->numOfTables = taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, tsdb); - uTrace("no tbname condition or tagcond, all tables belongs to one group, numOfTables:%d", pGroupInfo->numOfTables); + tsdbTrace("no tbname condition or tagcond, all tables belongs to one group, numOfTables:%d", pGroupInfo->numOfTables); } else { // todo add error } diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 6dd707b763..6a22bdea6a 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -47,7 +47,7 @@ typedef struct SCacheDataNode { typedef struct STrashElem { struct STrashElem *prev; struct STrashElem *next; - SCacheDataNode * pData; + SCacheDataNode *pData; } STrashElem; typedef struct { diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index 1960dc2a2a..4fc382a09b 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -118,7 +118,7 @@ extern "C" { int32_t strdequote(char *src); -void strtrim(char *src); +size_t strtrim(char *src); char *strnchr(char *haystack, char needle, int32_t len, bool skipquote); diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 93b8e30f1e..95d0ce7081 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -529,7 +529,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - size_t totalSize = dsize + sizeof(SHashNode) + keyLen + 1; // one extra byte for null + size_t totalSize = dsize + sizeof(SHashNode) + keyLen; SHashNode *pNewNode = calloc(1, totalSize); if (pNewNode == NULL) { @@ -544,7 +544,6 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s pNewNode->keyLen = keyLen; pNewNode->hashVal = hashVal; - return pNewNode; } @@ -559,7 +558,6 @@ SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t keyLen, co memcpy(pNewNode->data, pData, dsize); pNewNode->key = pNewNode->data + dsize; - assert(memcmp(pNewNode->key, key, keyLen) == 0 && keyLen == pNewNode->keyLen); memcpy(pNewNode->key, key, keyLen); diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 2641d2eacb..82873f9906 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -77,7 +77,7 @@ static FORCE_INLINE void taosFreeNode(void *data) { * @param lifespan total survial expiredTime from now * @return SCacheDataNode */ -static SCacheDataNode *taosCreateHashNode(const char *key, size_t keyLen, const char *pData, size_t size, +static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { size_t totalSize = size + sizeof(SCacheDataNode) + keyLen + 1; @@ -242,13 +242,14 @@ static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode // only a node is not referenced by any other object, in-place update it if (T_REF_VAL_GET(pNode) == 0) { - size_t newSize = sizeof(SCacheDataNode) + dataSize + keyLen; + size_t newSize = sizeof(SCacheDataNode) + dataSize + keyLen + 1; pNewNode = (SCacheDataNode *)realloc(pNode, newSize); if (pNewNode == NULL) { return NULL; } + memset(pNewNode, 0, newSize); pNewNode->signature = (uint64_t)pNewNode; memcpy(pNewNode->data, pData, dataSize); @@ -267,7 +268,7 @@ static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode } else { taosCacheMoveToTrash(pCacheObj, pNode); - pNewNode = taosCreateHashNode(key, keyLen, pData, dataSize, duration); + pNewNode = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); if (pNewNode == NULL) { return NULL; } @@ -293,7 +294,7 @@ static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode */ static FORCE_INLINE SCacheDataNode *taosAddToCacheImpl(SCacheObj *pCacheObj, const char *key, size_t keyLen, const void *pData, size_t dataSize, uint64_t duration) { - SCacheDataNode *pNode = taosCreateHashNode(key, keyLen, pData, dataSize, duration); + SCacheDataNode *pNode = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); if (pNode == NULL) { return NULL; } diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 6bc60615ce..6a5c8b6799 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -60,7 +60,7 @@ int32_t strdequote(char *z) { return j + 1; // only one quote, do nothing } -void strtrim(char *z) { +size_t strtrim(char *z) { int32_t i = 0; int32_t j = 0; @@ -71,7 +71,7 @@ void strtrim(char *z) { if (z[j] == 0) { z[0] = 0; - return; + return 0; } delta = j; @@ -89,9 +89,12 @@ void strtrim(char *z) { if (stop > 0) { z[stop - delta] = 0; + return (stop - delta); } else if (j != i) { z[i] = 0; } + + return i; } char **strsplit(char *z, const char *delim, int32_t *num) { diff --git a/src/util/tests/stringTest.cpp b/src/util/tests/stringTest.cpp index ef8df90e3e..0dc1ee7a77 100644 --- a/src/util/tests/stringTest.cpp +++ b/src/util/tests/stringTest.cpp @@ -20,10 +20,10 @@ TEST(testCase, string_dequote_test) { EXPECT_STRCASEEQ(t1, "abc"); char t21[] = " abc "; - strtrim(t21); + int32_t lx = strtrim(t21); EXPECT_STREQ("abc", t21); - EXPECT_EQ(3, strlen(t21)); + EXPECT_EQ(3, lx); } TEST(testCase, string_replace_test) { diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index b25e3c32fe..89dd3527c6 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -26,13 +26,12 @@ void taosMsleep(int mseconds); static int32_t doQuery(TAOS* taos, const char* sql) { - int32_t code = taos_query(taos, sql); - if (code != 0) { + TAOS_RES* res = taos_query(taos, sql); + if (taos_errno(res) != 0) { printf("failed to execute query, reason:%s\n", taos_errstr(taos)); return -1; } - TAOS_RES* res = taos_use_result(taos); TAOS_ROW row = NULL; char buf[512] = {0}; @@ -46,7 +45,6 @@ static int32_t doQuery(TAOS* taos, const char* sql) { } taos_free_result(res); - return 0; } @@ -81,6 +79,7 @@ static __attribute__((unused)) void multiThreadTest(int32_t numOfThreads, void* pthread_join(threadId[i], NULL); } + free(threadId); pthread_attr_destroy(&thattr); } @@ -114,19 +113,6 @@ int main(int argc, char *argv[]) { // doQuery(taos, "insert into t1 values(now, 2)"); // } // doQuery(taos, "create table t1(ts timestamp, k binary(12), f nchar(2))"); - -// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 'abc')"); -// doQuery(taos, "create table if not exists tm0 (ts timestamp, k int);"); -// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 1);"); -// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:2', 2);"); -// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:3', 3);"); -// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:4', 4);"); -// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:5', 5);"); -// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:6', 6);"); -// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:7', 7);"); -// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:8', 8);"); -// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:9', 9);"); -// doQuery(taos, "select sum(k),count(*) from m1 group by a"); taos_close(taos); return 0; @@ -173,10 +159,6 @@ int main(int argc, char *argv[]) { exit(1); } - - result = taos_use_result(taos); - - if (result == NULL) { printf("failed to get result, reason:%s\n", taos_errstr(taos)); exit(1); diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 2e5d84d45f..dfd2034f7e 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -609,7 +609,8 @@ class StateDbOnly(AnyState): ] def verifyTasksToState(self, tasks, newState): - self.assertAtMostOneSuccess(tasks, DropDbTask) # not true in massively parralel cases + if ( not self.hasTask(tasks, CreateDbTask) ): + self.assertAtMostOneSuccess(tasks, DropDbTask) # only if we don't create any more self.assertIfExistThenSuccess(tasks, DropDbTask) # self.assertAtMostOneSuccess(tasks, CreateFixedTableTask) # not true in massively parrallel cases # Nothing to be said about adding data task @@ -619,7 +620,8 @@ class StateDbOnly(AnyState): # self._state = self.STATE_EMPTY elif ( self.hasSuccess(tasks, CreateFixedSuperTableTask) ): # did not drop db, create table success # self.assertHasTask(tasks, CreateFixedTableTask) # tried to create table - self.assertAtMostOneSuccess(tasks, CreateFixedSuperTableTask) # at most 1 attempt is successful + if ( not self.hasTask(tasks, DropFixedSuperTableTask) ): + self.assertAtMostOneSuccess(tasks, CreateFixedSuperTableTask) # at most 1 attempt is successful, if we don't drop anything self.assertNoTask(tasks, DropDbTask) # should have have tried # if ( not self.hasSuccess(tasks, AddFixedDataTask) ): # just created table, no data yet # # can't say there's add-data attempts, since they may all fail @@ -674,7 +676,7 @@ class StateHasData(AnyState): if ( not self.hasTask(tasks, CreateDbTask)): # without a create_db task self.assertNoTask(tasks, DropDbTask) # we must have drop_db task self.hasSuccess(tasks, DropFixedSuperTableTask) - self.assertAtMostOneSuccess(tasks, DropFixedSuperTableTask) # TODO: dicy + # self.assertAtMostOneSuccess(tasks, DropFixedSuperTableTask) # TODO: dicy elif ( newState.equals(AnyState.STATE_TABLE_ONLY) ): # data deleted self.assertNoTask(tasks, DropDbTask) self.assertNoTask(tasks, DropFixedSuperTableTask) @@ -689,9 +691,9 @@ class StateHasData(AnyState): # State of the database as we believe it to be class DbState(): - def __init__(self): + def __init__(self, resetDb = True): self.tableNumQueue = LinearQueue() - self._lastTick = datetime.datetime(2019, 1, 1) # initial date time tick + self._lastTick = self.setupLastTick() # datetime.datetime(2019, 1, 1) # initial date time tick self._lastInt = 0 # next one is initial integer self._lock = threading.RLock() @@ -712,12 +714,32 @@ class DbState(): except: print("[=] Unexpected exception") raise - self._dbConn.resetDb() # drop and recreate DB - self._state = StateEmpty() # initial state, the result of above + + if resetDb : + self._dbConn.resetDb() # drop and recreate DB + self._state = self._findCurrentState() def getDbConn(self): return self._dbConn + def getState(self): + return self._state + + # We aim to create a starting time tick, such that, whenever we run our test here once + # We should be able to safely create 100,000 records, which will not have any repeated time stamp + # when we re-run the test in 3 minutes (180 seconds), basically we should expand time duration + # by a factor of 500. + # TODO: what if it goes beyond 10 years into the future + def setupLastTick(self): + t1 = datetime.datetime(2020, 5, 30) + t2 = datetime.datetime.now() + elSec = t2.timestamp() - t1.timestamp() + # print("elSec = {}".format(elSec)) + t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years + t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec * 500) # see explanation above + logger.info("Setting up TICKS to start from: {}".format(t4)) + return t4 + def pickAndAllocateTable(self): # pick any table, and "use" it return self.tableNumQueue.pickAndAllocate() @@ -743,7 +765,7 @@ class DbState(): return self._lastInt def getNextBinary(self): - return "Los_Angeles_{}".format(self.getNextInt()) + return "Beijing_Shanghai_Los_Angeles_New_York_San_Francisco_Chicago_Beijing_Shanghai_Los_Angeles_New_York_San_Francisco_Chicago_{}".format(self.getNextInt()) def getNextFloat(self): return 0.9 + self.getNextInt() @@ -1089,7 +1111,7 @@ class CreateFixedSuperTableTask(StateTransitionTask): def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): tblName = self._dbState.getFixedSuperTableName() - wt.execSql("create table db.{} (ts timestamp, speed int) tags (b binary(20), f float) ".format(tblName)) + wt.execSql("create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName)) # No need to create the regular tables, INSERT will do that automatically @@ -1148,12 +1170,13 @@ class AddFixedDataTask(StateTransitionTask): ds = self._dbState wt.execSql("use db") # TODO: seems to be an INSERT bug to require this for i in range(10): # 0 to 9 - sql = "insert into db.reg_table_{} using {} tags ('{}', {}) values ('{}', {});".format( - i, - ds.getFixedSuperTableName(), - ds.getNextBinary(), ds.getNextFloat(), - ds.getNextTick(), ds.getNextInt()) - wt.execSql(sql) + for j in range(10) : + sql = "insert into db.reg_table_{} using {} tags ('{}', {}) values ('{}', {});".format( + i, + ds.getFixedSuperTableName(), + ds.getNextBinary(), ds.getNextFloat(), + ds.getNextTick(), ds.getNextInt()) + wt.execSql(sql) #---------- Non State-Transition Related Tasks ----------# @@ -1301,7 +1324,9 @@ def main(): ch = logging.StreamHandler() logger.addHandler(ch) - dbState = DbState() + # resetDb = False # DEBUG only + # dbState = DbState(resetDb) # DBEUG only! + dbState = DbState() # Regular function Dice.seed(0) # initial seeding of dice tc = ThreadCoordinator( ThreadPool(dbState, gConfig.num_threads, gConfig.max_steps, 0), @@ -1309,6 +1334,43 @@ def main(): dbState ) + # # Hack to exercise reading from disk, imcreasing coverage. TODO: fix + # dbc = dbState.getDbConn() + # sTbName = dbState.getFixedSuperTableName() + # dbc.execute("create database if not exists db") + # if not dbState.getState().equals(StateEmpty()): + # dbc.execute("use db") + + # rTables = None + # try: # the super table may not exist + # sql = "select TBNAME from db.{}".format(sTbName) + # logger.info("Finding out tables in super table: {}".format(sql)) + # dbc.query(sql) # TODO: analyze result set later + # logger.info("Fetching result") + # rTables = dbc.getQueryResult() + # logger.info("Result: {}".format(rTables)) + # except taos.error.ProgrammingError as err: + # logger.info("Initial Super table OPS error: {}".format(err)) + + # # sys.exit() + # if ( not rTables == None): + # # print("rTables[0] = {}, type = {}".format(rTables[0], type(rTables[0]))) + # try: + # for rTbName in rTables : # regular tables + # ds = dbState + # logger.info("Inserting into table: {}".format(rTbName[0])) + # sql = "insert into db.{} values ('{}', {});".format( + # rTbName[0], + # ds.getNextTick(), ds.getNextInt()) + # dbc.execute(sql) + # for rTbName in rTables : # regular tables + # dbc.query("select * from db.{}".format(rTbName[0])) # TODO: check success failure + # logger.info("Initial READING operation is successful") + # except taos.error.ProgrammingError as err: + # logger.info("Initial WRITE/READ error: {}".format(err)) + + + # Sandbox testing code # dbc = dbState.getDbConn() # while True: diff --git a/tests/pytest/crash_gen_0519.py b/tests/pytest/crash_gen_0519.py deleted file mode 100755 index 44c978957e..0000000000 --- a/tests/pytest/crash_gen_0519.py +++ /dev/null @@ -1,831 +0,0 @@ -#!/usr/bin/python3.7 -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- -from __future__ import annotations # For type hinting before definition, ref: https://stackoverflow.com/questions/33533148/how-do-i-specify-that-the-return-type-of-a-method-is-the-same-as-the-class-itsel - -import sys -# Require Python 3 -if sys.version_info[0] < 3: - raise Exception("Must be using Python 3") - -import getopt -import argparse -import copy - -import threading -import random -import logging -import datetime -import textwrap - -from typing import List - -from util.log import * -from util.dnodes import * -from util.cases import * -from util.sql import * - -import crash_gen -import taos - -# Global variables, tried to keep a small number. -gConfig = None # Command-line/Environment Configurations, will set a bit later -logger = None - -def runThread(wt: WorkerThread): - wt.run() - -class WorkerThread: - def __init__(self, pool: ThreadPool, tid, - tc: ThreadCoordinator, - # te: TaskExecutor, - ): # note: main thread context! - # self._curStep = -1 - self._pool = pool - self._tid = tid - self._tc = tc - # self.threadIdent = threading.get_ident() - self._thread = threading.Thread(target=runThread, args=(self,)) - self._stepGate = threading.Event() - - # Let us have a DB connection of our own - if ( gConfig.per_thread_db_connection ): # type: ignore - self._dbConn = DbConn() - - def logDebug(self, msg): - logger.info(" t[{}] {}".format(self._tid, msg)) - - def logInfo(self, msg): - logger.info(" t[{}] {}".format(self._tid, msg)) - - - def getTaskExecutor(self): - return self._tc.getTaskExecutor() - - def start(self): - self._thread.start() # AFTER the thread is recorded - - def run(self): - # initialization after thread starts, in the thread context - # self.isSleeping = False - logger.info("Starting to run thread: {}".format(self._tid)) - - if ( gConfig.per_thread_db_connection ): # type: ignore - self._dbConn.open() - - self._doTaskLoop() - - # clean up - if ( gConfig.per_thread_db_connection ): # type: ignore - self._dbConn.close() - - def _doTaskLoop(self) : - # while self._curStep < self._pool.maxSteps: - # tc = ThreadCoordinator(None) - while True: - tc = self._tc # Thread Coordinator, the overall master - tc.crossStepBarrier() # shared barrier first, INCLUDING the last one - logger.debug("Thread task loop exited barrier...") - self.crossStepGate() # then per-thread gate, after being tapped - logger.debug("Thread task loop exited step gate...") - if not self._tc.isRunning(): - break - - task = tc.fetchTask() - task.execute(self) - tc.saveExecutedTask(task) - - def verifyThreadSelf(self): # ensure we are called by this own thread - if ( threading.get_ident() != self._thread.ident ): - raise RuntimeError("Unexpectly called from other threads") - - def verifyThreadMain(self): # ensure we are called by the main thread - if ( threading.get_ident() != threading.main_thread().ident ): - raise RuntimeError("Unexpectly called from other threads") - - def verifyThreadAlive(self): - if ( not self._thread.is_alive() ): - raise RuntimeError("Unexpected dead thread") - - # A gate is different from a barrier in that a thread needs to be "tapped" - def crossStepGate(self): - self.verifyThreadAlive() - self.verifyThreadSelf() # only allowed by ourselves - - # Wait again at the "gate", waiting to be "tapped" - # logger.debug("Worker thread {} about to cross the step gate".format(self._tid)) - self._stepGate.wait() - self._stepGate.clear() - - # self._curStep += 1 # off to a new step... - - def tapStepGate(self): # give it a tap, release the thread waiting there - self.verifyThreadAlive() - self.verifyThreadMain() # only allowed for main thread - - logger.debug("Tapping worker thread {}".format(self._tid)) - self._stepGate.set() # wake up! - time.sleep(0) # let the released thread run a bit - - def execSql(self, sql): # not "execute", since we are out side the DB context - if ( gConfig.per_thread_db_connection ): - return self._dbConn.execute(sql) - else: - return self._tc.getDbState().getDbConn().execute(sql) - - def querySql(self, sql): # not "execute", since we are out side the DB context - if ( gConfig.per_thread_db_connection ): - return self._dbConn.query(sql) - else: - return self._tc.getDbState().getDbConn().query(sql) - -class ThreadCoordinator: - def __init__(self, pool, wd: WorkDispatcher, dbState): - self._curStep = -1 # first step is 0 - self._pool = pool - self._wd = wd - self._te = None # prepare for every new step - self._dbState = dbState - self._executedTasks: List[Task] = [] # in a given step - self._lock = threading.RLock() # sync access for a few things - - self._stepBarrier = threading.Barrier(self._pool.numThreads + 1) # one barrier for all threads - - def getTaskExecutor(self): - return self._te - - def getDbState(self) -> DbState : - return self._dbState - - def crossStepBarrier(self): - self._stepBarrier.wait() - - def run(self): - self._pool.createAndStartThreads(self) - - # Coordinate all threads step by step - self._curStep = -1 # not started yet - maxSteps = gConfig.max_steps # type: ignore - while(self._curStep < maxSteps): - print(".", end="", flush=True) - logger.debug("Main thread going to sleep") - - # Now ready to enter a step - self.crossStepBarrier() # let other threads go past the pool barrier, but wait at the thread gate - self._stepBarrier.reset() # Other worker threads should now be at the "gate" - - # At this point, all threads should be pass the overall "barrier" and before the per-thread "gate" - self._dbState.transition(self._executedTasks) # at end of step, transiton the DB state - self.resetExecutedTasks() # clear the tasks after we are done - - # Get ready for next step - logger.info("<-- Step {} finished".format(self._curStep)) - self._curStep += 1 # we are about to get into next step. TODO: race condition here! - logger.debug("\r\n--> Step {} starts with main thread waking up".format(self._curStep)) # Now not all threads had time to go to sleep - - # A new TE for the new step - self._te = TaskExecutor(self._curStep) - - logger.debug("Main thread waking up at step {}, tapping worker threads".format(self._curStep)) # Now not all threads had time to go to sleep - self.tapAllThreads() - - logger.debug("Main thread ready to finish up...") - self.crossStepBarrier() # Cross it one last time, after all threads finish - self._stepBarrier.reset() - logger.debug("Main thread in exclusive zone...") - self._te = None # No more executor, time to end - logger.debug("Main thread tapping all threads one last time...") - self.tapAllThreads() # Let the threads run one last time - logger.debug("Main thread joining all threads") - self._pool.joinAll() # Get all threads to finish - - logger.info("All threads finished") - print("\r\nFinished") - - def tapAllThreads(self): # in a deterministic manner - wakeSeq = [] - for i in range(self._pool.numThreads): # generate a random sequence - if Dice.throw(2) == 1 : - wakeSeq.append(i) - else: - wakeSeq.insert(0, i) - logger.info("Waking up threads: {}".format(str(wakeSeq))) - # TODO: set dice seed to a deterministic value - for i in wakeSeq: - self._pool.threadList[i].tapStepGate() # TODO: maybe a bit too deep?! - time.sleep(0) # yield - - def isRunning(self): - return self._te != None - - def fetchTask(self) -> Task : - if ( not self.isRunning() ): # no task - raise RuntimeError("Cannot fetch task when not running") - # return self._wd.pickTask() - # Alternatively, let's ask the DbState for the appropriate task - dbState = self.getDbState() - tasks = dbState.getTasksAtState() - i = Dice.throw(len(tasks)) - # return copy.copy(tasks[i]) # Needs a fresh copy, to save execution results, etc. - return tasks[i].clone() - - def resetExecutedTasks(self): - self._executedTasks = [] # should be under single thread - - def saveExecutedTask(self, task): - with self._lock: - self._executedTasks.append(task) - -# We define a class to run a number of threads in locking steps. -class ThreadPool: - def __init__(self, dbState, numThreads, maxSteps, funcSequencer): - self.numThreads = numThreads - self.maxSteps = maxSteps - self.funcSequencer = funcSequencer - # Internal class variables - self.dispatcher = WorkDispatcher(dbState) - self.curStep = 0 - self.threadList = [] - # self.stepGate = threading.Condition() # Gate to hold/sync all threads - # self.numWaitingThreads = 0 - - # starting to run all the threads, in locking steps - def createAndStartThreads(self, tc: ThreadCoordinator): - for tid in range(0, self.numThreads): # Create the threads - workerThread = WorkerThread(self, tid, tc) - self.threadList.append(workerThread) - workerThread.start() # start, but should block immediately before step 0 - - def joinAll(self): - for workerThread in self.threadList: - logger.debug("Joining thread...") - workerThread._thread.join() - -# A queue of continguous POSITIVE integers -class LinearQueue(): - def __init__(self): - self.firstIndex = 1 # 1st ever element - self.lastIndex = 0 - self._lock = threading.RLock() # our functions may call each other - self.inUse = set() # the indexes that are in use right now - - def toText(self): - return "[{}..{}], in use: {}".format(self.firstIndex, self.lastIndex, self.inUse) - - # Push (add new element, largest) to the tail, and mark it in use - def push(self): - with self._lock: - # if ( self.isEmpty() ): - # self.lastIndex = self.firstIndex - # return self.firstIndex - # Otherwise we have something - self.lastIndex += 1 - self.allocate(self.lastIndex) - # self.inUse.add(self.lastIndex) # mark it in use immediately - return self.lastIndex - - def pop(self): - with self._lock: - if ( self.isEmpty() ): - # raise RuntimeError("Cannot pop an empty queue") - return False # TODO: None? - - index = self.firstIndex - if ( index in self.inUse ): - return False - - self.firstIndex += 1 - return index - - def isEmpty(self): - return self.firstIndex > self.lastIndex - - def popIfNotEmpty(self): - with self._lock: - if (self.isEmpty()): - return 0 - return self.pop() - - def allocate(self, i): - with self._lock: - # logger.debug("LQ allocating item {}".format(i)) - if ( i in self.inUse ): - raise RuntimeError("Cannot re-use same index in queue: {}".format(i)) - self.inUse.add(i) - - def release(self, i): - with self._lock: - # logger.debug("LQ releasing item {}".format(i)) - self.inUse.remove(i) # KeyError possible, TODO: why? - - def size(self): - return self.lastIndex + 1 - self.firstIndex - - def pickAndAllocate(self): - if ( self.isEmpty() ): - return None - with self._lock: - cnt = 0 # counting the interations - while True: - cnt += 1 - if ( cnt > self.size()*10 ): # 10x iteration already - # raise RuntimeError("Failed to allocate LinearQueue element") - return None - ret = Dice.throwRange(self.firstIndex, self.lastIndex+1) - if ( not ret in self.inUse ): - self.allocate(ret) - return ret - -class DbConn: - def __init__(self): - self._conn = None - self._cursor = None - self.isOpen = False - - def open(self): # Open connection - if ( self.isOpen ): - raise RuntimeError("Cannot re-open an existing DB connection") - - cfgPath = "../../build/test/cfg" - self._conn = taos.connect(host="127.0.0.1", config=cfgPath) # TODO: make configurable - self._cursor = self._conn.cursor() - - # Get the connection/cursor ready - self._cursor.execute('reset query cache') - # self._cursor.execute('use db') - - # Open connection - self._tdSql = TDSql() - self._tdSql.init(self._cursor) - self.isOpen = True - - def resetDb(self): # reset the whole database, etc. - if ( not self.isOpen ): - raise RuntimeError("Cannot reset database until connection is open") - # self._tdSql.prepare() # Recreate database, etc. - - self._cursor.execute('drop database if exists db') - logger.debug("Resetting DB, dropped database") - # self._cursor.execute('create database db') - # self._cursor.execute('use db') - - # tdSql.execute('show databases') - - def close(self): - if ( not self.isOpen ): - raise RuntimeError("Cannot clean up database until connection is open") - self._tdSql.close() - self.isOpen = False - - def execute(self, sql): - if ( not self.isOpen ): - raise RuntimeError("Cannot execute database commands until connection is open") - return self._tdSql.execute(sql) - - def query(self, sql) -> int : # return number of rows retrieved - if ( not self.isOpen ): - raise RuntimeError("Cannot query database until connection is open") - return self._tdSql.query(sql) - - -# State of the database as we believe it to be -class DbState(): - STATE_INVALID = -1 - STATE_EMPTY = 1 # nothing there, no even a DB - STATE_DB_ONLY = 2 # we have a DB, but nothing else - STATE_TABLE_ONLY = 3 # we have a table, but totally empty - STATE_HAS_DATA = 4 # we have some data in the table - - def __init__(self): - self.tableNumQueue = LinearQueue() - self._lastTick = datetime.datetime(2019, 1, 1) # initial date time tick - self._lastInt = 0 # next one is initial integer - self._lock = threading.RLock() - self._state = self.STATE_INVALID - - # self.openDbServerConnection() - self._dbConn = DbConn() - try: - self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected - except taos.error.ProgrammingError as err: - # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err)) - if ( err.msg == 'disconnected' ): # cannot open DB connection - print("Cannot establish DB connection, please re-run script without parameter, and follow the instructions.") - sys.exit() - else: - raise - except: - print("[=]Unexpected exception") - raise - self._dbConn.resetDb() # drop and recreate DB - self._state = self.STATE_EMPTY # initial state, the result of above - - def getDbConn(self): - return self._dbConn - - def pickAndAllocateTable(self): # pick any table, and "use" it - return self.tableNumQueue.pickAndAllocate() - - def addTable(self): - with self._lock: - tIndex = self.tableNumQueue.push() - return tIndex - - def getFixedTableName(self): - return "fixed_table" - - def releaseTable(self, i): # return the table back, so others can use it - self.tableNumQueue.release(i) - - def getNextTick(self): - with self._lock: # prevent duplicate tick - self._lastTick += datetime.timedelta(0, 1) # add one second to it - return self._lastTick - - def getNextInt(self): - with self._lock: - self._lastInt += 1 - return self._lastInt - - def getTableNameToDelete(self): - tblNum = self.tableNumQueue.pop() # TODO: race condition! - if ( not tblNum ): # maybe false - return False - - return "table_{}".format(tblNum) - - def execSql(self, sql): # using the main DB connection - return self._dbConn.execute(sql) - - def cleanUp(self): - self._dbConn.close() - - def getTasksAtState(self): - tasks = [] - tasks.append(ReadFixedDataTask(self)) # always - if ( self._state == self.STATE_EMPTY ): - tasks.append(CreateDbTask(self)) - tasks.append(CreateFixedTableTask(self)) - elif ( self._state == self.STATE_DB_ONLY ): - tasks.append(DropDbTask(self)) - tasks.append(CreateFixedTableTask(self)) - tasks.append(AddFixedDataTask(self)) - elif ( self._state == self.STATE_TABLE_ONLY ): - tasks.append(DropFixedTableTask(self)) - tasks.append(AddFixedDataTask(self)) - elif ( self._state == self.STATE_HAS_DATA ) : # same as above. TODO: adjust - tasks.append(DropFixedTableTask(self)) - tasks.append(AddFixedDataTask(self)) - else: - raise RuntimeError("Unexpected DbState state: {}".format(self._state)) - return tasks - - def transition(self, tasks): - if ( len(tasks) == 0 ): # before 1st step, or otherwise empty - return # do nothing - if ( self._state == self.STATE_EMPTY ): - # self.assertNoSuccess(tasks, ReadFixedDataTask) # some read may be successful, since we might be creating a table - if ( self.hasSuccess(tasks, CreateDbTask) ): - self.assertAtMostOneSuccess(tasks, CreateDbTask) # param is class - self._state = self.STATE_DB_ONLY - if ( self.hasSuccess(tasks, CreateFixedTableTask )): - self._state = self.STATE_TABLE_ONLY - # else: # no successful table creation, not much we can say, as it is step 2 - else: # did not create db - self.assertNoTask(tasks, CreateDbTask) # because we did not have such task - # self.assertNoSuccess(tasks, CreateDbTask) # not necessary, since we just verified no such task - self.assertNoSuccess(tasks, CreateFixedTableTask) - - elif ( self._state == self.STATE_DB_ONLY ): - self.assertAtMostOneSuccess(tasks, DropDbTask) - self.assertIfExistThenSuccess(tasks, DropDbTask) - self.assertAtMostOneSuccess(tasks, CreateFixedTableTask) - # Nothing to be said about adding data task - if ( self.hasSuccess(tasks, DropDbTask) ): # dropped the DB - # self.assertHasTask(tasks, DropDbTask) # implied by hasSuccess - self.assertAtMostOneSuccess(tasks, DropDbTask) - self._state = self.STATE_EMPTY - elif ( self.hasSuccess(tasks, CreateFixedTableTask) ): # did not drop db, create table success - # self.assertHasTask(tasks, CreateFixedTableTask) # tried to create table - self.assertAtMostOneSuccess(tasks, CreateFixedTableTask) # at most 1 attempt is successful - self.assertNoTask(tasks, DropDbTask) # should have have tried - if ( not self.hasSuccess(tasks, AddFixedDataTask) ): # just created table, no data yet - # can't say there's add-data attempts, since they may all fail - self._state = self.STATE_TABLE_ONLY - else: - self._state = self.STATE_HAS_DATA - else: # no success in dropping db tasks, no success in create fixed table, not acceptable - raise RuntimeError("Unexpected no-success scenario") - - elif ( self._state == self.STATE_TABLE_ONLY ): - if ( self.hasSuccess(tasks, DropFixedTableTask) ): - self.assertAtMostOneSuccess(tasks, DropFixedTableTask) - self._state = self.STATE_DB_ONLY - elif ( self.hasSuccess(tasks, AddFixedDataTask) ): # no success dropping the table - self.assertNoTask(tasks, DropFixedTableTask) - self._state = self.STATE_HAS_DATA - else: # did not drop table, did not insert data, that is impossible - raise RuntimeError("Unexpected no-success scenarios") - - elif ( self._state == self.STATE_HAS_DATA ): # Same as above, TODO: adjust - if ( self.hasSuccess(tasks, DropFixedTableTask) ): - self.assertAtMostOneSuccess(tasks, DropFixedTableTask) - self._state = self.STATE_DB_ONLY - elif ( self.hasSuccess(tasks, AddFixedDataTask) ): # no success dropping the table - self.assertNoTask(tasks, DropFixedTableTask) - self._state = self.STATE_HAS_DATA - else: # did not drop table, did not insert data, that is impossible - raise RuntimeError("Unexpected no-success scenarios") - - else: - raise RuntimeError("Unexpected DbState state: {}".format(self._state)) - logger.debug("New DB state is: {}".format(self._state)) - - def assertAtMostOneSuccess(self, tasks, cls): - sCnt = 0 - for task in tasks : - if not isinstance(task, cls): - continue - if task.isSuccess(): - task.logDebug("Task success found") - sCnt += 1 - if ( sCnt >= 2 ): - raise RuntimeError("Unexpected more than 1 success with task: {}".format(cls)) - - def assertIfExistThenSuccess(self, tasks, cls): - sCnt = 0 - exists = False - for task in tasks : - if not isinstance(task, cls): - continue - exists = True # we have a valid instance - if task.isSuccess(): - sCnt += 1 - if ( exists and sCnt <= 0 ): - raise RuntimeError("Unexpected zero success for task: {}".format(cls)) - - def assertNoTask(self, tasks, cls): - for task in tasks : - if isinstance(task, cls): - raise RuntimeError("Unexpected task: {}".format(cls)) - - def assertNoSuccess(self, tasks, cls): - for task in tasks : - if isinstance(task, cls): - if task.isSuccess(): - raise RuntimeError("Unexpected successful task: {}".format(cls)) - - def hasSuccess(self, tasks, cls): - for task in tasks : - if not isinstance(task, cls): - continue - if task.isSuccess(): - return True - return False - -class TaskExecutor(): - def __init__(self, curStep): - self._curStep = curStep - - def getCurStep(self): - return self._curStep - - def execute(self, task: Task, wt: WorkerThread): # execute a task on a thread - task.execute(wt) - - # def logInfo(self, msg): - # logger.info(" T[{}.x]: ".format(self._curStep) + msg) - - # def logDebug(self, msg): - # logger.debug(" T[{}.x]: ".format(self._curStep) + msg) - -class Task(): - taskSn = 100 - - @classmethod - def allocTaskNum(cls): - cls.taskSn += 1 - return cls.taskSn - - def __init__(self, dbState: DbState): - self._dbState = dbState - self._workerThread = None - self._err = None - self._curStep = None - self._numRows = None # Number of rows affected - - # Assign an incremental task serial number - self._taskNum = self.allocTaskNum() - - def isSuccess(self): - return self._err == None - - def clone(self): - newTask = self.__class__(self._dbState) - return newTask - - def logDebug(self, msg): - self._workerThread.logDebug("s[{}.{}] {}".format(self._curStep, self._taskNum, msg)) - - def logInfo(self, msg): - self._workerThread.logInfo("s[{}.{}] {}".format(self._curStep, self._taskNum, msg)) - - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - raise RuntimeError("To be implemeted by child classes, class name: {}".format(self.__class__.__name__)) - - def execute(self, wt: WorkerThread): - wt.verifyThreadSelf() - self._workerThread = wt # type: ignore - - te = wt.getTaskExecutor() - self._curStep = te.getCurStep() - self.logDebug("[-] executing task {}...".format(self.__class__.__name__)) - - self._err = None - try: - self._executeInternal(te, wt) # TODO: no return value? - except taos.error.ProgrammingError as err: - self.logDebug("[=]Taos Execution exception: {0}".format(err)) - self._err = err - except: - self.logDebug("[=]Unexpected exception") - raise - - self.logDebug("[X] task execution completed, {}, status: {}".format(self.__class__.__name__, "Success" if self.isSuccess() else "Failure")) - - def execSql(self, sql): - return self._dbState.execute(sql) - -class CreateDbTask(Task): - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - wt.execSql("create database db") - -class DropDbTask(Task): - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - wt.execSql("drop database db") - -class CreateTableTask(Task): - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - tIndex = self._dbState.addTable() - self.logDebug("Creating a table {} ...".format(tIndex)) - wt.execSql("create table db.table_{} (ts timestamp, speed int)".format(tIndex)) - self.logDebug("Table {} created.".format(tIndex)) - self._dbState.releaseTable(tIndex) - -class CreateFixedTableTask(Task): - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - tblName = self._dbState.getFixedTableName() - wt.execSql("create table db.{} (ts timestamp, speed int)".format(tblName)) - -class ReadFixedDataTask(Task): - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - tblName = self._dbState.getFixedTableName() - self._numRows = wt.querySql("select * from db.{}".format(tblName)) # save the result for later - # tdSql.query(" cars where tbname in ('carzero', 'carone')") - -class DropTableTask(Task): - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - tableName = self._dbState.getTableNameToDelete() - if ( not tableName ): # May be "False" - self.logInfo("Cannot generate a table to delete, skipping...") - return - self.logInfo("Dropping a table db.{} ...".format(tableName)) - wt.execSql("drop table db.{}".format(tableName)) - -class DropFixedTableTask(Task): - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - tblName = self._dbState.getFixedTableName() - wt.execSql("drop table db.{}".format(tblName)) - -class AddDataTask(Task): - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - ds = self._dbState - self.logInfo("Adding some data... numQueue={}".format(ds.tableNumQueue.toText())) - tIndex = ds.pickAndAllocateTable() - if ( tIndex == None ): - self.logInfo("No table found to add data, skipping...") - return - sql = "insert into db.table_{} values ('{}', {});".format(tIndex, ds.getNextTick(), ds.getNextInt()) - self.logDebug("Executing SQL: {}".format(sql)) - wt.execSql(sql) - ds.releaseTable(tIndex) - self.logDebug("Finished adding data") - -class AddFixedDataTask(Task): - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - ds = self._dbState - sql = "insert into db.table_{} values ('{}', {});".format(ds.getFixedTableName(), ds.getNextTick(), ds.getNextInt()) - wt.execSql(sql) - -# Deterministic random number generator -class Dice(): - seeded = False # static, uninitialized - - @classmethod - def seed(cls, s): # static - if (cls.seeded): - raise RuntimeError("Cannot seed the random generator more than once") - cls.verifyRNG() - random.seed(s) - cls.seeded = True # TODO: protect against multi-threading - - @classmethod - def verifyRNG(cls): # Verify that the RNG is determinstic - random.seed(0) - x1 = random.randrange(0, 1000) - x2 = random.randrange(0, 1000) - x3 = random.randrange(0, 1000) - if ( x1 != 864 or x2!=394 or x3!=776 ): - raise RuntimeError("System RNG is not deterministic") - - @classmethod - def throw(cls, stop): # get 0 to stop-1 - return cls.throwRange(0, stop) - - @classmethod - def throwRange(cls, start, stop): # up to stop-1 - if ( not cls.seeded ): - raise RuntimeError("Cannot throw dice before seeding it") - return random.randrange(start, stop) - - -# Anyone needing to carry out work should simply come here -class WorkDispatcher(): - def __init__(self, dbState): - # self.totalNumMethods = 2 - self.tasks = [ - CreateTableTask(dbState), - DropTableTask(dbState), - AddDataTask(dbState), - ] - - def throwDice(self): - max = len(self.tasks) - 1 - dRes = random.randint(0, max) - # logger.debug("Threw the dice in range [{},{}], and got: {}".format(0,max,dRes)) - return dRes - - def pickTask(self): - dice = self.throwDice() - return self.tasks[dice] - - def doWork(self, workerThread): - task = self.pickTask() - task.execute(workerThread) - -def main(): - # Super cool Python argument library: https://docs.python.org/3/library/argparse.html - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent('''\ - TDengine Auto Crash Generator (PLEASE NOTICE the Prerequisites Below) - --------------------------------------------------------------------- - 1. You build TDengine in the top level ./build directory, as described in offical docs - 2. You run the server there before this script: ./build/bin/taosd -c test/cfg - - ''')) - parser.add_argument('-p', '--per-thread-db-connection', action='store_true', - help='Use a single shared db connection (default: false)') - parser.add_argument('-d', '--debug', action='store_true', - help='Turn on DEBUG mode for more logging (default: false)') - parser.add_argument('-s', '--max-steps', action='store', default=100, type=int, - help='Maximum number of steps to run (default: 100)') - parser.add_argument('-t', '--num-threads', action='store', default=10, type=int, - help='Number of threads to run (default: 10)') - - global gConfig - gConfig = parser.parse_args() - if len(sys.argv) == 1: - parser.print_help() - sys.exit() - - global logger - logger = logging.getLogger('myApp') - if ( gConfig.debug ): - logger.setLevel(logging.DEBUG) # default seems to be INFO - ch = logging.StreamHandler() - logger.addHandler(ch) - - dbState = DbState() - Dice.seed(0) # initial seeding of dice - tc = ThreadCoordinator( - ThreadPool(dbState, gConfig.num_threads, gConfig.max_steps, 0), - WorkDispatcher(dbState), - dbState - ) - tc.run() - dbState.cleanUp() - logger.info("Finished running thread pool") - -if __name__ == "__main__": - main() diff --git a/tests/pytest/crash_gen_0519.sh b/tests/pytest/crash_gen_0519.sh deleted file mode 100755 index 3e744d7926..0000000000 --- a/tests/pytest/crash_gen_0519.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# This is the script for us to try to cause the TDengine server or client to crash -# -# PREPARATION -# -# 1. Build an compile the TDengine source code that comes with this script, in the same directory tree -# 2. Please follow the direction in our README.md, and build TDengine in the build/ directory -# 3. Adjust the configuration file if needed under build/test/cfg/taos.cfg -# 4. Run the TDengine server instance: cd build; ./build/bin/taosd -c test/cfg -# 5. Make sure you have a working Python3 environment: run /usr/bin/python3 --version, and you should get 3.6 or above -# 6. Make sure you have the proper Python packages: # sudo apt install python3-setuptools python3-pip python3-distutils -# -# RUNNING THIS SCRIPT -# -# This script assumes the source code directory is intact, and that the binaries has been built in the -# build/ directory, as such, will will load the Python libraries in the directory tree, and also load -# the TDengine client shared library (so) file, in the build/directory, as evidenced in the env -# variables below. -# -# Running the script is simple, no parameter is needed (for now, but will change in the future). -# -# Happy Crashing... - - -# Due to the heavy path name assumptions/usage, let us require that the user be in the current directory -EXEC_DIR=`dirname "$0"` -if [[ $EXEC_DIR != "." ]] -then - echo "ERROR: Please execute `basename "$0"` in its own directory (for now anyway, pardon the dust)" - exit -1 -fi - -# First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. -export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3 - -# Then let us set up the library path so that our compiled SO file can be loaded by Python -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/../../build/build/lib - -# Now we are all let, and let's see if we can find a crash. Note we pass all params -./crash_gen_0519.py $@ diff --git a/tests/script/general/field/binary.sim b/tests/script/general/field/binary.sim index 8bf1c84b93..b51f023efe 100644 --- a/tests/script/general/field/binary.sim +++ b/tests/script/general/field/binary.sim @@ -60,17 +60,10 @@ if $rows != 75 then return -1 endi -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' group by tgcol -x step13 - return -1 -step13: - -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m and tbcol = '1' group by tgcol -x step14 - return -1 -step14: - -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' interval(1d) group by tgcol -x step15 - return -1 -step15: +print select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' +sql_error select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' group by tgcol +sql_error select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m and tbcol = '1' group by tgcol +sql_error select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' interval(1d) group by tgcol #can't filter binary fields diff --git a/tests/script/general/parser/limit2_query.sim b/tests/script/general/parser/limit2_query.sim index cbff2e946c..1dbd0f32bd 100644 --- a/tests/script/general/parser/limit2_query.sim +++ b/tests/script/general/parser/limit2_query.sim @@ -133,6 +133,7 @@ $limit = $rowNum $offset = $limit / 2 sql select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1, -2) limit $limit offset $offset if $rows != $limit then + print expect $limit, actual $rows return -1 endi if $data01 != 0 then @@ -147,6 +148,7 @@ $limit = $rowNum $offset = $limit / 2 sql select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(linear) limit $limit offset $offset if $rows != $limit then + print expect $limit, actual $rows return -1 endi if $data01 != 0 then @@ -182,6 +184,7 @@ $limit = $rowNum $offset = $limit / 2 sql select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(prev) limit $limit offset $offset if $rows != $limit then + print expect $limit, actual: $rows return -1 endi diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index cc95246f06..4a3da2f2a0 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -40,45 +40,47 @@ #run general/parser/nchar.sim #sleep 2000 ##run general/parser/null_char.sim -sleep 2000 -run general/parser/single_row_in_tb.sim -sleep 2000 -run general/parser/select_from_cache_disk.sim -sleep 2000 -run general/parser/selectResNum.sim -sleep 2000 -run general/parser/mixed_blocks.sim -sleep 2000 -run general/parser/limit1.sim -sleep 2000 -run general/parser/limit.sim -sleep 2000 -run general/parser/limit1_tblocks100.sim -sleep 2000 -run general/parser/select_across_vnodes.sim -sleep 2000 -run general/parser/slimit1.sim -sleep 2000 -run general/parser/tbnameIn.sim -sleep 2000 +#sleep 2000 +#run general/parser/single_row_in_tb.sim +#sleep 2000 +#run general/parser/select_from_cache_disk.sim +#sleep 2000 +#run general/parser/selectResNum.sim +#sleep 2000 +#run general/parser/mixed_blocks.sim +#sleep 2000 +#run general/parser/limit1.sim +#sleep 2000 +#run general/parser/limit.sim +#sleep 2000 +#run general/parser/limit1_tblocks100.sim +#sleep 2000 +#run general/parser/select_across_vnodes.sim +#sleep 2000 +#run general/parser/slimit1.sim +#sleep 2000 +#run general/parser/tbnameIn.sim +#sleep 2000 run general/parser/projection_limit_offset.sim sleep 2000 run general/parser/limit2.sim sleep 2000 -run general/parser/fill.sim -sleep 2000 -run general/parser/fill_stb.sim -sleep 2000 -run general/parser/where.sim -sleep 2000 -run general/parser/slimit.sim -sleep 2000 -run general/parser/select_with_tags.sim -sleep 2000 -run general/parser/interp.sim - +#run general/parser/fill.sim +#sleep 2000 +#run general/parser/fill_stb.sim +#sleep 2000 +#run general/parser/where.sim +#sleep 2000 +#run general/parser/slimit.sim +#sleep 2000 +#run general/parser/select_with_tags.sim +#sleep 2000 +#run general/parser/interp.sim sleep 2000 run general/parser/tags_dynamically_specifiy.sim +sleep 2000 +run general/parser/groupby.sim + sleep 2000 run general/parser/set_tag_vals.sim @@ -97,9 +99,6 @@ sleep 2000 run general/parser/join.sim sleep 2000 run general/parser/join_multivnode.sim - -sleep 2000 -run general/parser/groupby.sim sleep 2000 run general/parser/binary_escapeCharacter.sim sleep 2000 diff --git a/tests/script/unique/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim index 827f02bc1f..19d1c48c3d 100644 --- a/tests/script/unique/http/opentsdb.sim +++ b/tests/script/unique/http/opentsdb.sim @@ -75,7 +75,7 @@ endi system_content curl -u root:taosdata -d '[{"metric": "ab1234567890123456789012345678ab1234567890123456789012345678","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":-2147483389}}],"failed":1,"success":0,"affected_rows":0}@ then +if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":-2147482999}}],"failed":1,"success":0,"affected_rows":0}@ then return -1 endi @@ -125,7 +125,7 @@ endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","group1": "1","group1": "1","group1": "1","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbbbbbb","table":"sys_cpu_d_bbbbbbb_lga_1_1_1_1_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","group1":"1","group1":"1","group1":"1","group1":"1","host":"web01"},"status":"error","code":-2147483445}}],"failed":1,"success":0,"affected_rows":0}@ then +if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbbbbbb","table":"sys_cpu_d_bbbbbbb_lga_1_1_1_1_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","group1":"1","group1":"1","group1":"1","group1":"1","host":"web01"},"status":"error","code":-2147483135}}],"failed":1,"success":0,"affected_rows":0}@ then return -1 endi diff --git a/tests/test/c/importOneRow.c b/tests/test/c/importOneRow.c index 902cf46c3a..0690d89837 100644 --- a/tests/test/c/importOneRow.c +++ b/tests/test/c/importOneRow.c @@ -122,10 +122,26 @@ void* taos_execute(void *param) { int64_t timestamp = 1530374400000L; sprintf(sql, "insert into db.t%d values(%ld, %d, %d, %d)", pThread->index, timestamp, 0, 0, 0); - int code = taos_query(taos, sql); - if (code != 0) printf("error code:%d, sql:%s\n", code, sql); + TAOS_RES *pSql = taos_query(taos, sql); + int code = taos_errno(pSql); + if (code != 0) + { + printf("error code:%d, sql:%s\n", code, sql); + taos_free_result(pSql); + taos_close(taos); + return NULL; + } int affectrows = taos_affected_rows(taos); - if (affectrows != 1) printf("affect rows:%d, sql:%s\n", affectrows, sql); + if (affectrows != 1) + { + printf("affect rows:%d, sql:%s\n", affectrows, sql); + taos_free_result(pSql); + taos_close(taos); + return NULL; + } + taos_free_result(pSql); + pSql = NULL; + timestamp -= 1000; @@ -133,17 +149,34 @@ void* taos_execute(void *param) { for (int i = 1; i < rowNum; ++i) { sprintf(sql, "import into db.t%d values(%ld, %d, %d, %d)", pThread->index, timestamp, i, i, i); - code = taos_query(taos, sql); - if (code != 0) printf("error code:%d, sql:%s\n", code, sql); + + pSql = taos_query(taos, sql); + code = taos_errno(pSql); + if (code != 0) + { + printf("error code:%d, sql:%s\n", code, sql); + taos_free_result(pSql); + pSql = NULL; + taos_close(taos); + return NULL; + } int affectrows = taos_affected_rows(taos); - if (affectrows != 1) printf("affect rows:%d, sql:%s\n", affectrows, sql); + if (affectrows != 1) { + printf("affect rows:%d, sql:%s\n", affectrows, sql); + taos_free_result(pSql); + pSql = NULL; + taos_close(taos); + } total_affect_rows += affectrows; + taos_free_result(pSql); + pSql = NULL; timestamp -= 1000; } printf("thread:%d run finished total_affect_rows:%d\n", pThread->index, total_affect_rows); + taos_close(taos); return NULL; } diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 0c35e3668c..1230f52f63 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -635,17 +635,20 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { SCmdLine *line = &script->lines[script->linePos]; int ret = -1; + TAOS_RES* pSql = NULL; + for (int attempt = 0; attempt < 3; ++attempt) { simLogSql(rest); - ret = taos_query(script->taos, rest); - if (ret == TSDB_CODE_TABLE_ALREADY_EXIST || - ret == TSDB_CODE_DB_ALREADY_EXIST) { + pSql = taos_query(script->taos, rest); + ret = taos_errno(pSql); + + if (ret == TSDB_CODE_TABLE_ALREADY_EXIST || ret == TSDB_CODE_DB_ALREADY_EXIST) { simTrace("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret, tstrerror(ret)); ret = 0; break; } else if (ret != 0) { simTrace("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s", - script->fileName, script->taos, rest, ret, tstrerror(ret), taos_errstr(script->taos)); + script->fileName, script->taos, rest, ret, tstrerror(ret), taos_errstr(pSql)); if (line->errorJump == SQL_JUMP_TRUE) { script->linePos = line->jump; @@ -655,6 +658,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } else { break; } + + taos_free_result(pSql); } if (ret) { @@ -663,10 +668,9 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } int numOfRows = 0; - int num_fields = taos_field_count(script->taos); + int num_fields = taos_field_count(pSql); if (num_fields != 0) { - TAOS_RES *result = taos_use_result(script->taos); - if (result == NULL) { + if (pSql == NULL) { simTrace("script:%s, taos:%p, %s failed, result is null", script->fileName, script->taos, rest); if (line->errorJump == SQL_JUMP_TRUE) { script->linePos = line->jump; @@ -679,10 +683,10 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { TAOS_ROW row; - while ((row = taos_fetch_row(result))) { + while ((row = taos_fetch_row(pSql))) { if (numOfRows < MAX_QUERY_ROW_NUM) { - TAOS_FIELD *fields = taos_fetch_fields(result); - int* length = taos_fetch_lengths(result); + TAOS_FIELD *fields = taos_fetch_fields(pSql); + int* length = taos_fetch_lengths(pSql); for (int i = 0; i < num_fields; i++) { char *value = NULL; @@ -768,11 +772,11 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } } - taos_free_result(result); } else { - numOfRows = taos_affected_rows(script->taos); + numOfRows = taos_affected_rows(pSql); } + taos_free_result(pSql); sprintf(script->rows, "%d", numOfRows); script->linePos++; @@ -911,13 +915,16 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { } int ret; + TAOS_RES* pSql = NULL; if (simAsyncQuery) { char command[4096]; sprintf(command, "curl -H 'Authorization: Taosd %s' -d '%s' 127.0.0.1:6020/rest/sql", script->auth, rest); ret = simExecuteRestFulCommand(script, command); } else { - ret = taos_query(script->taos, rest); + pSql = taos_query(script->taos, rest); + ret = taos_errno(pSql); + taos_free_result(pSql); } if (ret != TSDB_CODE_SUCCESS) { @@ -926,6 +933,7 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { script->linePos++; return true; } + sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret, tstrerror(ret)); return false;