xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feature/3_liaohj' into feature/3_liaohj

This commit is contained in:
Haojun Liao 2022-07-19 10:28:51 +08:00
parent 2f199bd80b 5e754a7662
commit c9e39e722a
64 changed files with 1322 additions and 420 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
examples/c/tmq.c
View File

@ -28,8 +28,9 @@ static void msg_process(TAOS_RES* msg) {
printf("db: %s\n", tmq_get_db_name(msg)); printf("db: %s\n", tmq_get_db_name(msg));
printf("vg: %d\n", tmq_get_vgroup_id(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg));
if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) {
tmq_raw_data* raw = tmq_get_raw_meta(msg); tmq_raw_data raw = {0};
if (raw) { int32_t code = tmq_get_raw_meta(msg, &raw);
if (code == 0) {
TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", NULL, 0);
if (pConn == NULL) { if (pConn == NULL) {
return; return;
@ -53,7 +54,6 @@ static void msg_process(TAOS_RES* msg) {
printf("write raw data: %s\n", tmq_err2str(ret)); printf("write raw data: %s\n", tmq_err2str(ret));
taos_close(pConn); taos_close(pConn);
} }
tmq_free_raw_meta(raw);
char* result = tmq_get_json_meta(msg); char* result = tmq_get_json_meta(msg);
if (result) { if (result) {
printf("meta result: %s\n", result); printf("meta result: %s\n", result);

12
include/client/taos.h
View File

@ -259,13 +259,17 @@ enum tmq_res_t {
TMQ_RES_TABLE_META = 2, TMQ_RES_TABLE_META = 2,
}; };
typedef struct {
void* raw_meta;
uint32_t raw_meta_len;
uint16_t raw_meta_type;
} tmq_raw_data;
typedef enum tmq_res_t tmq_res_t; typedef enum tmq_res_t tmq_res_t;
typedef struct tmq_raw_data tmq_raw_data;
DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res);
DLL_EXPORT tmq_raw_data *tmq_get_raw_meta(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, tmq_raw_data *raw_meta);
DLL_EXPORT int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta); DLL_EXPORT int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data raw_meta);
DLL_EXPORT void tmq_free_raw_meta(tmq_raw_data *rawMeta);
DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); // Returning null means error. Returned result need to be freed by tmq_free_json_meta DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); // Returning null means error. Returned result need to be freed by tmq_free_json_meta
DLL_EXPORT void tmq_free_json_meta(char* jsonMeta); DLL_EXPORT void tmq_free_json_meta(char* jsonMeta);
DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);

2
packaging/deb/makedeb.sh
View File

@ -73,7 +73,7 @@ cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_pat
cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include
cp ${compile_dir}/../include/util/taoserror.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/util/taoserror.h ${pkg_dir}${install_home_path}/include
cp ${compile_dir}/../include/libs/function/taosudf.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/libs/function/taosudf.h ${pkg_dir}${install_home_path}/include
cp ${compile_dir}/../src/inc/taosws.h ${pkg_dir}${install_home_path}/include ||: cp ${compile_dir}/build/include/taosws.h ${pkg_dir}${install_home_path}/include ||:
cp -r ${top_dir}/examples/* ${pkg_dir}${install_home_path}/examples cp -r ${top_dir}/examples/* ${pkg_dir}${install_home_path}/examples
#cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector #cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector
#cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector #cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector

2
packaging/rpm/tdengine.spec
View File

@ -80,7 +80,7 @@ cp %{_compiledir}/../include/client/taos.h %{buildroot}%{homepath}/incl
cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include
cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include
cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepath}/include
cp %{_compiledir}/../src/inc/taosws.h %{buildroot}%{homepath}/include ||: cp %{_compiledir}/build/include/taosws.h %{buildroot}%{homepath}/include ||:
#cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector #cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector
#cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector #cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector
#cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector #cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector

4
packaging/tools/install.sh
View File

@ -229,13 +229,13 @@ function install_lib() {
${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
${csudo}ln -s ${lib_link_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so || : [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
${csudo}ln -s ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : ${csudo}ln -s ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
fi fi
${csudo}ldconfig ${csudo}ldconfig

12
packaging/tools/install_client.sh
View File

@ -116,6 +116,7 @@ function install_bin() {
function clean_lib() { function clean_lib() {
sudo rm -f /usr/lib/libtaos.* || : sudo rm -f /usr/lib/libtaos.* || :
sudo rm -f /usr/lib/libtaosws.* || :
sudo rm -rf ${lib_dir} || : sudo rm -rf ${lib_dir} || :
} }
@ -123,6 +124,9 @@ function install_lib() {
# Remove links # Remove links
${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
#${csudo}rm -rf ${v15_java_app_dir} || : #${csudo}rm -rf ${v15_java_app_dir} || :
${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/*
@ -131,13 +135,19 @@ function install_lib() {
${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
[ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so
if [ -d "${lib64_link_dir}" ]; then if [ -d "${lib64_link_dir}" ]; then
${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
[ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
fi fi
else else
${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib
${csudo}ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib ${csudo}ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib
[ -f ${install_main_dir}/driver/libtaosws.dylib ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib
fi fi
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then
@ -154,6 +164,8 @@ function install_header() {
${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h
[ -f ${install_main_dir}/include/taosws.h ] && ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taos.h
} }
function install_jemalloc() { function install_jemalloc() {

7
packaging/tools/makeclient.sh
View File

@ -57,12 +57,16 @@ if [ "$osType" != "Darwin" ]; then
${script_dir}/get_client.sh" ${script_dir}/get_client.sh"
fi fi
lib_files="${build_dir}/lib/libtaos.so.${version}" lib_files="${build_dir}/lib/libtaos.so.${version}"
wslib_files="${build_dir}/lib/libtaosws.so"
else else
bin_files="${build_dir}/bin/${clientName} ${script_dir}/remove_client.sh" bin_files="${build_dir}/bin/${clientName} ${script_dir}/remove_client.sh"
lib_files="${build_dir}/lib/libtaos.${version}.dylib" lib_files="${build_dir}/lib/libtaos.${version}.dylib"
wslib_files="${build_dir}/lib/libtaosws.dylib"
fi fi
header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h"
wsheader_files="${build_dir}/include/taosws.h"
if [ "$dbName" != "taos" ]; then if [ "$dbName" != "taos" ]; then
cfg_dir="${top_dir}/../enterprise/packaging/cfg" cfg_dir="${top_dir}/../enterprise/packaging/cfg"
else else
@ -74,6 +78,8 @@ install_files="${script_dir}/install_client.sh"
# make directories. # make directories.
mkdir -p ${install_dir} mkdir -p ${install_dir}
mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
[ -f ${wsheader_files} ] && cp ${wsheader_files} ${install_dir}/inc
mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile}
mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/*
@ -187,6 +193,7 @@ cp ${lib_files} ${install_dir}/driver
# Copy connector # Copy connector
connector_dir="${code_dir}/connector" connector_dir="${code_dir}/connector"
mkdir -p ${install_dir}/connector mkdir -p ${install_dir}/connector
[ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then

4
packaging/tools/makepkg.sh
View File

@ -96,7 +96,7 @@ lib_files="${build_dir}/lib/libtaos.so.${version}"
wslib_files="${build_dir}/lib/libtaosws.so." wslib_files="${build_dir}/lib/libtaosws.so."
header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h"
wsheader_files="${code_dir}/inc/taosws.h" wsheader_files="${build_dir}/include/taosws.h"
if [ "$dbName" != "taos" ]; then if [ "$dbName" != "taos" ]; then
cfg_dir="${top_dir}/../enterprise/packaging/cfg" cfg_dir="${top_dir}/../enterprise/packaging/cfg"
@ -114,7 +114,7 @@ init_file_rpm=${script_dir}/../rpm/taosd
mkdir -p ${install_dir} mkdir -p ${install_dir}
mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
${wsheader_files} ${install_dir}/inc || : [ -f ${wsheader_files} ] && cp ${wsheader_files} ${install_dir}/inc || :
mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile}

11
packaging/tools/post.sh
View File

@ -82,22 +82,33 @@ function kill_taosd() {
function install_include() { function install_include() {
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
${csudo}rm -f ${inc_link_dir}/taosws.h
${csudo}ln -s ${inc_dir}/taos.h ${inc_link_dir}/taos.h ${csudo}ln -s ${inc_dir}/taos.h ${inc_link_dir}/taos.h
${csudo}ln -s ${inc_dir}/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${inc_dir}/taosdef.h ${inc_link_dir}/taosdef.h
${csudo}ln -s ${inc_dir}/taoserror.h ${inc_link_dir}/taoserror.h ${csudo}ln -s ${inc_dir}/taoserror.h ${inc_link_dir}/taoserror.h
${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h
[ -f ${inc_dir}/taosws.h ] && ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h ||:
} }
function install_lib() { function install_lib() {
${csudo}rm -f ${lib_link_dir}/libtaos* || : ${csudo}rm -f ${lib_link_dir}/libtaos* || :
${csudo}rm -f ${lib64_link_dir}/libtaos* || : ${csudo}rm -f ${lib64_link_dir}/libtaos* || :
${csudo}rm -f ${lib_link_dir}/libtaosws* || :
${csudo}rm -f ${lib64_link_dir}/libtaosws* || :
${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1
${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
[ -f ${lib_dir}/libtaosws.so ]${csudo}ln -s ${lib_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so
if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
[ -ff ${lib_dir}/libtaosws.so ] && ${csudo}ln -s ${lib_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
fi fi
${csudo}ldconfig ${csudo}ldconfig

25
source/client/src/clientSml.c
View File

@ -268,7 +268,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm
*actionNeeded = true; *actionNeeded = true;
} }
if (*actionNeeded) { if (*actionNeeded) {
uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, colField->name, uDebug("SML:0x%" PRIx64 " generate schema action. kv->name: %s, action: %d", info->id, kv->key,
action->action); action->action);
} }
return 0; return 0;
@ -436,6 +436,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
SSchemaAction *action, bool isTag) { SSchemaAction *action, bool isTag) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
for (int j = 0; j < taosArrayGetSize(cols); ++j) { for (int j = 0; j < taosArrayGetSize(cols); ++j) {
if(j == 0 && !isTag) continue;
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j);
bool actionNeeded = false; bool actionNeeded = false;
code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, &actionNeeded, info); code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, &actionNeeded, info);
@ -452,18 +453,25 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols) { static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) {
SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
for (uint16_t i = 0; i < length; i++) { int32_t i = 0;
for ( ;i < length; i++) {
taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES);
} }
for (int32_t i = 0; i < taosArrayGetSize(cols); i++) { if (isTag){
i = 0;
} else {
i = 1;
}
for (; i < taosArrayGetSize(cols); i++) {
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) {
return -1; return -1;
} }
} }
taosHashCleanup(hashTmp);
return 0; return 0;
} }
@ -523,7 +531,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
} }
taosHashClear(hashTmp); taosHashClear(hashTmp);
for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { for (uint16_t i = 1; i < pTableMeta->tableInfo.numOfColumns; i++) {
taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES);
} }
code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &schemaAction, false); code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &schemaAction, false);
@ -551,12 +559,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
if (needCheckMeta) { if (needCheckMeta) {
code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags,
sTableData->tags); sTableData->tags, true);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, (char *)superTable); uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, (char *)superTable);
goto end; goto end;
} }
code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols); code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, (char *)superTable); uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, (char *)superTable);
goto end; goto end;
@ -832,6 +840,7 @@ static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t
static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) { static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) {
int64_t ts = 0; int64_t ts = 0;
if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
// uError("SML:data:%s,len:%d", data, len);
ts = smlParseInfluxTime(info, data, len); ts = smlParseInfluxTime(info, data, len);
} else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
ts = smlParseOpenTsdbTime(info, data, len); ts = smlParseOpenTsdbTime(info, data, len);
@ -2031,6 +2040,8 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *
static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) {
SSmlLineInfo elements = {0}; SSmlLineInfo elements = {0};
uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql);
int ret = smlParseInfluxString(sql, &elements, &info->msgBuf); int ret = smlParseInfluxString(sql, &elements, &info->msgBuf);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id);

49
source/client/src/tmq.c
View File

@ -113,12 +113,6 @@ struct tmq_t {
tsem_t rspSem; tsem_t rspSem;
}; };
struct tmq_raw_data {
void* raw_meta;
int32_t raw_meta_len;
int16_t raw_meta_type;
};
enum { enum {
TMQ_VG_STATUS__IDLE = 0, TMQ_VG_STATUS__IDLE = 0,
TMQ_VG_STATUS__WAIT, TMQ_VG_STATUS__WAIT,
@ -363,7 +357,11 @@ tmq_list_t* tmq_list_new() {
int32_t tmq_list_append(tmq_list_t* list, const char* src) { int32_t tmq_list_append(tmq_list_t* list, const char* src) {
SArray* container = &list->container; SArray* container = &list->container;
char* topic = strdup(src); if (src == NULL || src[0] == 0) return -1;
char* topic = strdup(src);
if (topic[0] != '`') {
strtolower(topic, src);
}
if (taosArrayPush(container, &topic) == NULL) return -1; if (taosArrayPush(container, &topic) == NULL) return -1;
return 0; return 0;
} }
@ -1914,16 +1912,15 @@ const char* tmq_get_table_name(TAOS_RES* res) {
return NULL; return NULL;
} }
tmq_raw_data* tmq_get_raw_meta(TAOS_RES* res) { int32_t tmq_get_raw_meta(TAOS_RES* res, tmq_raw_data *raw) {
if (TD_RES_TMQ_META(res)) { if (TD_RES_TMQ_META(res) && raw) {
tmq_raw_data* raw = taosMemoryCalloc(1, sizeof(tmq_raw_data));
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
raw->raw_meta = pMetaRspObj->metaRsp.metaRsp; raw->raw_meta = pMetaRspObj->metaRsp.metaRsp;
raw->raw_meta_len = pMetaRspObj->metaRsp.metaRspLen; raw->raw_meta_len = pMetaRspObj->metaRsp.metaRspLen;
raw->raw_meta_type = pMetaRspObj->metaRsp.resMsgType; raw->raw_meta_type = pMetaRspObj->metaRsp.resMsgType;
return raw; return TSDB_CODE_SUCCESS;
} }
return NULL; return TSDB_CODE_INVALID_PARA;
} }
static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
@ -2931,23 +2928,23 @@ end:
return code; return code;
} }
int32_t taos_write_raw_meta(TAOS* taos, tmq_raw_data* raw_meta) { int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data raw_meta){
if (!taos || !raw_meta) { if (!taos) {
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
} }
if (raw_meta->raw_meta_type == TDMT_VND_CREATE_STB) { if(raw_meta.raw_meta_type == TDMT_VND_CREATE_STB) {
return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); return taosCreateStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
} else if (raw_meta->raw_meta_type == TDMT_VND_ALTER_STB) { }else if(raw_meta.raw_meta_type == TDMT_VND_ALTER_STB){
return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); return taosCreateStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
} else if (raw_meta->raw_meta_type == TDMT_VND_DROP_STB) { }else if(raw_meta.raw_meta_type == TDMT_VND_DROP_STB){
return taosDropStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); return taosDropStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
} else if (raw_meta->raw_meta_type == TDMT_VND_CREATE_TABLE) { }else if(raw_meta.raw_meta_type == TDMT_VND_CREATE_TABLE){
return taosCreateTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); return taosCreateTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
} else if (raw_meta->raw_meta_type == TDMT_VND_ALTER_TABLE) { }else if(raw_meta.raw_meta_type == TDMT_VND_ALTER_TABLE){
return taosAlterTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); return taosAlterTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
} else if (raw_meta->raw_meta_type == TDMT_VND_DROP_TABLE) { }else if(raw_meta.raw_meta_type == TDMT_VND_DROP_TABLE){
return taosDropTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); return taosDropTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
} }
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
} }

4
source/common/src/tdatablock.c
View File

@ -228,7 +228,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui
uint32_t finalNumOfRows = numOfRow1 + numOfRow2; uint32_t finalNumOfRows = numOfRow1 + numOfRow2;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
// Handle the bitmap // Handle the bitmap
if (finalNumOfRows > *capacity || numOfRow1 == 0) { if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) {
char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
if (p == NULL) { if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -262,7 +262,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui
memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len);
pColumnInfoData->varmeta.length = len + oldLen; pColumnInfoData->varmeta.length = len + oldLen;
} else { } else {
if (finalNumOfRows > *capacity || numOfRow1 == 0) { if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) {
ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); ASSERT(finalNumOfRows * pColumnInfoData->info.bytes);
char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes);
if (tmp == NULL) { if (tmp == NULL) {

2
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -49,9 +49,9 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps); dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps);
} }
} }
rpcFreeCont(pRsp->pCont);
tFreeSStatusRsp(&statusRsp); tFreeSStatusRsp(&statusRsp);
} }
rpcFreeCont(pRsp->pCont);
} }
void dmSendStatusReq(SDnodeMgmt *pMgmt) { void dmSendStatusReq(SDnodeMgmt *pMgmt) {

10
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -179,6 +179,16 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
} else { } else {
dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pWriteQ, pMsg); taosWriteQitem(pVnode->pWriteQ, pMsg);
#if 0 // tests for batch writes
if (pMsg->msgType == TDMT_VND_CREATE_TABLE) {
SRpcMsg *pDup = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
memcpy(pDup, pMsg, sizeof(SRpcMsg));
pDup->pCont = rpcMallocCont(pMsg->contLen);
memcpy(pDup->pCont, pMsg->pCont, pMsg->contLen);
pDup->info.handle = NULL;
taosWriteQitem(pVnode->pWriteQ, pDup);
}
#endif
} }
break; break;
case SYNC_QUEUE: case SYNC_QUEUE:

43
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -35,6 +35,46 @@
extern "C" { extern "C" {
#endif #endif
typedef enum {
MND_OPER_CONNECT = 1,
MND_OPER_CREATE_ACCT,
MND_OPER_DROP_ACCT,
MND_OPER_ALTER_ACCT,
MND_OPER_CREATE_USER,
MND_OPER_DROP_USER,
MND_OPER_ALTER_USER,
MND_OPER_CREATE_BNODE,
MND_OPER_DROP_BNODE,
MND_OPER_CREATE_DNODE,
MND_OPER_DROP_DNODE,
MND_OPER_CONFIG_DNODE,
MND_OPER_CREATE_MNODE,
MND_OPER_DROP_MNODE,
MND_OPER_CREATE_QNODE,
MND_OPER_DROP_QNODE,
MND_OPER_CREATE_SNODE,
MND_OPER_DROP_SNODE,
MND_OPER_REDISTRIBUTE_VGROUP,
MND_OPER_MERGE_VGROUP,
MND_OPER_SPLIT_VGROUP,
MND_OPER_BALANCE_VGROUP,
MND_OPER_CREATE_FUNC,
MND_OPER_DROP_FUNC,
MND_OPER_KILL_TRANS,
MND_OPER_KILL_CONN,
MND_OPER_KILL_QUERY,
MND_OPER_CREATE_DB,
MND_OPER_ALTER_DB,
MND_OPER_DROP_DB,
MND_OPER_COMPACT_DB,
MND_OPER_TRIM_DB,
MND_OPER_USE_DB,
MND_OPER_WRITE_DB,
MND_OPER_READ_DB,
MND_OPER_READ_OR_WRITE_DB,
MND_OPER_SHOW_VARIBALES,
} EOperType;
typedef enum { typedef enum {
MND_AUTH_ACCT_START = 0, MND_AUTH_ACCT_START = 0,
MND_AUTH_ACCT_USER, MND_AUTH_ACCT_USER,
@ -109,9 +149,9 @@ typedef struct {
ETrnPolicy policy; ETrnPolicy policy;
ETrnConflct conflict; ETrnConflct conflict;
ETrnExec exec; ETrnExec exec;
EOperType oper;
int32_t code; int32_t code;
int32_t failedTimes; int32_t failedTimes;
SRpcHandleInfo rpcInfo;
void* rpcRsp; void* rpcRsp;
int32_t rpcRspLen; int32_t rpcRspLen;
int32_t redoActionPos; int32_t redoActionPos;
@ -130,6 +170,7 @@ typedef struct {
int32_t stopFunc; int32_t stopFunc;
int32_t paramLen; int32_t paramLen;
void* param; void* param;
SArray* pRpcArray;
} STrans; } STrans;
typedef struct { typedef struct {

40
source/dnode/mnode/impl/inc/mndPrivilege.h
View File

@ -22,46 +22,6 @@
extern "C" { extern "C" {
#endif #endif
typedef enum {
MND_OPER_CONNECT = 1,
MND_OPER_CREATE_ACCT,
MND_OPER_DROP_ACCT,
MND_OPER_ALTER_ACCT,
MND_OPER_CREATE_USER,
MND_OPER_DROP_USER,
MND_OPER_ALTER_USER,
MND_OPER_CREATE_BNODE,
MND_OPER_DROP_BNODE,
MND_OPER_CREATE_DNODE,
MND_OPER_DROP_DNODE,
MND_OPER_CONFIG_DNODE,
MND_OPER_CREATE_MNODE,
MND_OPER_DROP_MNODE,
MND_OPER_CREATE_QNODE,
MND_OPER_DROP_QNODE,
MND_OPER_CREATE_SNODE,
MND_OPER_DROP_SNODE,
MND_OPER_REDISTRIBUTE_VGROUP,
MND_OPER_MERGE_VGROUP,
MND_OPER_SPLIT_VGROUP,
MND_OPER_BALANCE_VGROUP,
MND_OPER_CREATE_FUNC,
MND_OPER_DROP_FUNC,
MND_OPER_KILL_TRANS,
MND_OPER_KILL_CONN,
MND_OPER_KILL_QUERY,
MND_OPER_CREATE_DB,
MND_OPER_ALTER_DB,
MND_OPER_DROP_DB,
MND_OPER_COMPACT_DB,
MND_OPER_TRIM_DB,
MND_OPER_USE_DB,
MND_OPER_WRITE_DB,
MND_OPER_READ_DB,
MND_OPER_READ_OR_WRITE_DB,
MND_OPER_SHOW_VARIBALES,
} EOperType;
int32_t mndInitPrivilege(SMnode *pMnode); int32_t mndInitPrivilege(SMnode *pMnode);
void mndCleanupPrivilege(SMnode *pMnode); void mndCleanupPrivilege(SMnode *pMnode);

2
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -73,12 +73,14 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen); void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen);
void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2); void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2);
void mndTransSetSerial(STrans *pTrans); void mndTransSetSerial(STrans *pTrans);
void mndTransSetOper(STrans *pTrans, EOperType oper);
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
int32_t mndTransProcessRsp(SRpcMsg *pRsp); int32_t mndTransProcessRsp(SRpcMsg *pRsp);
void mndTransPullup(SMnode *pMnode); void mndTransPullup(SMnode *pMnode);
int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans);
void mndTransExecute(SMnode *pMnode, STrans *pTrans); void mndTransExecute(SMnode *pMnode, STrans *pTrans);
int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname);
#ifdef __cplusplus #ifdef __cplusplus
} }

9
source/dnode/mnode/impl/src/mndDb.c
View File

@ -487,6 +487,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db);
mndTransSetDbName(pTrans, dbObj.name, NULL); mndTransSetDbName(pTrans, dbObj.name, NULL);
mndTransSetOper(pTrans, MND_OPER_CREATE_DB);
if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
@ -534,6 +535,14 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
terrno = TSDB_CODE_MND_DB_ALREADY_EXIST; terrno = TSDB_CODE_MND_DB_ALREADY_EXIST;
goto _OVER; goto _OVER;
} }
} else if (terrno == TSDB_CODE_SDB_OBJ_CREATING) {
if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) {
mDebug("db:%s, is creating and response after trans finished", createReq.db);
code = TSDB_CODE_ACTION_IN_PROGRESS;
goto _OVER;
} else {
goto _OVER;
}
} else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) { } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) {
goto _OVER; goto _OVER;
} }

98
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -122,6 +122,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
SDB_SET_INT8(pRaw, dataPos, pTrans->policy, _OVER) SDB_SET_INT8(pRaw, dataPos, pTrans->policy, _OVER)
SDB_SET_INT8(pRaw, dataPos, pTrans->conflict, _OVER) SDB_SET_INT8(pRaw, dataPos, pTrans->conflict, _OVER)
SDB_SET_INT8(pRaw, dataPos, pTrans->exec, _OVER) SDB_SET_INT8(pRaw, dataPos, pTrans->exec, _OVER)
SDB_SET_INT8(pRaw, dataPos, pTrans->oper, _OVER)
SDB_SET_INT8(pRaw, dataPos, 0, _OVER)
SDB_SET_INT8(pRaw, dataPos, 0, _OVER)
SDB_SET_INT8(pRaw, dataPos, 0, _OVER)
SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER)
@ -269,15 +273,22 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
int8_t policy = 0; int8_t policy = 0;
int8_t conflict = 0; int8_t conflict = 0;
int8_t exec = 0; int8_t exec = 0;
int8_t oper = 0;
int8_t reserved = 0;
int8_t actionType = 0; int8_t actionType = 0;
SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
SDB_GET_INT8(pRaw, dataPos, &policy, _OVER) SDB_GET_INT8(pRaw, dataPos, &policy, _OVER)
SDB_GET_INT8(pRaw, dataPos, &conflict, _OVER) SDB_GET_INT8(pRaw, dataPos, &conflict, _OVER)
SDB_GET_INT8(pRaw, dataPos, &exec, _OVER) SDB_GET_INT8(pRaw, dataPos, &exec, _OVER)
SDB_GET_INT8(pRaw, dataPos, &oper, _OVER)
SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER)
SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER)
SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER)
pTrans->stage = stage; pTrans->stage = stage;
pTrans->policy = policy; pTrans->policy = policy;
pTrans->conflict = conflict; pTrans->conflict = conflict;
pTrans->exec = exec; pTrans->exec = exec;
pTrans->oper = oper;
SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER)
@ -495,6 +506,10 @@ static void mndTransDropData(STrans *pTrans) {
mndTransDropActions(pTrans->commitActions); mndTransDropActions(pTrans->commitActions);
pTrans->commitActions = NULL; pTrans->commitActions = NULL;
} }
if (pTrans->pRpcArray != NULL) {
taosArrayDestroy(pTrans->pRpcArray);
pTrans->pRpcArray = NULL;
}
if (pTrans->rpcRsp != NULL) { if (pTrans->rpcRsp != NULL) {
taosMemoryFree(pTrans->rpcRsp); taosMemoryFree(pTrans->rpcRsp);
pTrans->rpcRsp = NULL; pTrans->rpcRsp = NULL;
@ -585,14 +600,18 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict,
pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo));
if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL) { if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL ||
pTrans->pRpcArray == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to create transaction since %s", terrstr()); mError("failed to create transaction since %s", terrstr());
return NULL; return NULL;
} }
if (pReq != NULL) pTrans->rpcInfo = pReq->info; if (pReq != NULL) {
taosArrayPush(pTrans->pRpcArray, &pReq->info);
}
mTrace("trans:%d, local object is created, data:%p", pTrans->id, pTrans); mTrace("trans:%d, local object is created, data:%p", pTrans->id, pTrans);
return pTrans; return pTrans;
} }
@ -677,6 +696,31 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *
pTrans->paramLen = paramLen; pTrans->paramLen = paramLen;
} }
int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname) {
STrans *pTrans = NULL;
void *pIter = NULL;
int32_t code = -1;
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans);
if (pIter == NULL) break;
if (pTrans->oper == oper) {
if (strcasecmp(dbname, pTrans->dbname1) == 0) {
mDebug("trans:%d, db:%s oper:%d matched with input", pTrans->id, dbname, oper);
if (taosArrayPush(pTrans->pRpcArray, &pMsg->info) != NULL) {
code = 0;
}
sdbRelease(pMnode->pSdb, pTrans);
break;
}
}
sdbRelease(pMnode->pSdb, pTrans);
}
return code;
}
void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2) { void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2) {
if (dbname1 != NULL) { if (dbname1 != NULL) {
memcpy(pTrans->dbname1, dbname1, TSDB_DB_FNAME_LEN); memcpy(pTrans->dbname1, dbname1, TSDB_DB_FNAME_LEN);
@ -688,6 +732,8 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2)
void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; } void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; }
void mndTransSetOper(STrans *pTrans, EOperType oper) { pTrans->oper = oper; }
static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
SSdbRaw *pRaw = mndTransActionEncode(pTrans); SSdbRaw *pRaw = mndTransActionEncode(pTrans);
if (pRaw == NULL) { if (pRaw == NULL) {
@ -711,7 +757,7 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
static bool mndCheckDbConflict(const char *db, STrans *pTrans) { static bool mndCheckDbConflict(const char *db, STrans *pTrans) {
if (db[0] == 0) return false; if (db[0] == 0) return false;
if (strcmp(db, pTrans->dbname1) == 0 || strcmp(db, pTrans->dbname2) == 0) return true; if (strcasecmp(db, pTrans->dbname1) == 0 || strcasecmp(db, pTrans->dbname2) == 0) return true;
return false; return false;
} }
@ -784,9 +830,10 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
return -1; return -1;
} }
pNew->rpcInfo = pTrans->rpcInfo; pNew->pRpcArray = pTrans->pRpcArray;
pNew->rpcRsp = pTrans->rpcRsp; pNew->rpcRsp = pTrans->rpcRsp;
pNew->rpcRspLen = pTrans->rpcRspLen; pNew->rpcRspLen = pTrans->rpcRspLen;
pTrans->pRpcArray = NULL;
pTrans->rpcRsp = NULL; pTrans->rpcRsp = NULL;
pTrans->rpcRspLen = 0; pTrans->rpcRspLen = 0;
@ -835,29 +882,34 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
} }
} }
if (sendRsp && pTrans->rpcInfo.handle != NULL) { if (!sendRsp) return;
mDebug("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage),
pTrans->rpcInfo.ahandle);
if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL;
}
SRpcMsg rspMsg = {.code = code, .info = pTrans->rpcInfo};
if (pTrans->rpcRspLen != 0) { int32_t size = taosArrayGetSize(pTrans->pRpcArray);
void *rpcCont = rpcMallocCont(pTrans->rpcRspLen); if (size <= 0) return;
if (rpcCont != NULL) {
memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); for (int32_t i = 0; i < size; ++i) {
rspMsg.pCont = rpcCont; SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i);
rspMsg.contLen = pTrans->rpcRspLen; if (pInfo->handle != NULL) {
mDebug("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage),
pInfo->ahandle);
if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL;
} }
taosMemoryFree(pTrans->rpcRsp); SRpcMsg rspMsg = {.code = code, .info = *pInfo};
}
tmsgSendRsp(&rspMsg); if (pTrans->rpcRspLen != 0) {
pTrans->rpcInfo.handle = NULL; void *rpcCont = rpcMallocCont(pTrans->rpcRspLen);
pTrans->rpcRsp = NULL; if (rpcCont != NULL) {
pTrans->rpcRspLen = 0; memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen);
rspMsg.pCont = rpcCont;
rspMsg.contLen = pTrans->rpcRspLen;
}
}
tmsgSendRsp(&rspMsg);
}
} }
taosArrayClear(pTrans->pRpcArray);
} }
int32_t mndTransProcessRsp(SRpcMsg *pRsp) { int32_t mndTransProcessRsp(SRpcMsg *pRsp) {

27
source/dnode/vnode/src/inc/sma.h
View File

@ -43,9 +43,9 @@ typedef struct SRSmaInfo SRSmaInfo;
typedef struct SRSmaInfoItem SRSmaInfoItem; typedef struct SRSmaInfoItem SRSmaInfoItem;
struct SSmaEnv { struct SSmaEnv {
SRWLatch lock; SRWLatch lock;
int8_t type; int8_t type;
SSmaStat *pStat; SSmaStat *pStat;
}; };
typedef struct { typedef struct {
@ -103,10 +103,15 @@ struct SRSmaInfoItem {
}; };
struct SRSmaInfo { struct SRSmaInfo {
STSchema *pTSchema; STSchema *pTSchema;
int64_t suid; int64_t suid;
int8_t delFlag;
T_REF_DECLARE()
SRSmaInfoItem items[TSDB_RETENTION_L2]; SRSmaInfoItem items[TSDB_RETENTION_L2];
}; };
#define RSMA_INFO_HEAD_LEN 24
#define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1)
#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1)
enum { enum {
TASK_TRIGGER_STAT_INIT = 0, TASK_TRIGGER_STAT_INIT = 0,
@ -120,8 +125,8 @@ enum {
enum { enum {
RSMA_ROLE_CREATE = 0, RSMA_ROLE_CREATE = 0,
RSMA_ROLE_DROP = 1, RSMA_ROLE_DROP = 1,
RSMA_ROLE_FETCH = 2, RSMA_ROLE_SUBMIT = 2,
RSMA_ROLE_SUBMIT = 3, RSMA_ROLE_FETCH = 3,
RSMA_ROLE_ITERATE = 4, RSMA_ROLE_ITERATE = 4,
}; };
@ -134,6 +139,8 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg);
int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat); int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat);
int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat);
int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo);
int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo);
void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
@ -193,6 +200,7 @@ void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t le
static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree);
void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid);
int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash);
int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
@ -258,8 +266,9 @@ void tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
void tdCloseTFile(STFile *pTFile); void tdCloseTFile(STFile *pTFile);
void tdDestroyTFile(STFile *pTFile); void tdDestroyTFile(STFile *pTFile);
void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName); void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version,
void tdGetVndDirName(int32_t vgId,const char *pdname, const char *dname, bool endWithSep, char *outputName); char *outputName);
void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
source/dnode/vnode/src/inc/tsdb.h
View File

@ -186,6 +186,7 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData);
typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT; typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
void tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]); void tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]);
bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype); bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype);
bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype); int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype);
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype); int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype); int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype);

2
source/dnode/vnode/src/meta/metaTable.c
View File

@ -277,7 +277,7 @@ _drop_super_table:
_exit: _exit:
tdbFree(pKey); tdbFree(pKey);
tdbFree(pData); tdbFree(pData);
metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
return 0; return 0;
} }

4
source/dnode/vnode/src/sma/smaCommit.c
View File

@ -191,7 +191,7 @@ static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) {
if ((pDir = taosOpenDir(dir)) == NULL) { if ((pDir = taosOpenDir(dir)) == NULL) {
regfree(&regex); regfree(&regex);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); smaDebug("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -392,5 +392,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
// step 2: cleanup outdated qtaskinfo files // step 2: cleanup outdated qtaskinfo files
tdCleanupQTaskInfoFiles(pSma, pRSmaStat); tdCleanupQTaskInfoFiles(pSma, pRSmaStat);
atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

20
source/dnode/vnode/src/sma/smaEnv.c
View File

@ -169,6 +169,26 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
return 0; return 0;
} }
int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) {
if (!pRSmaInfo) return 0;
int ref = T_REF_INC(pRSmaInfo);
smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref);
return 0;
}
int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) {
if (!pRSmaInfo) return 0;
int ref = T_REF_DEC(pRSmaInfo);
smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref);
if (ref == 0) {
tdRemoveRSmaInfoBySuid(pSma, pRSmaInfo->suid);
}
return 0;
}
static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) { static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) {
ASSERT(pSmaStat != NULL); ASSERT(pSmaStat != NULL);

110
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -24,6 +24,7 @@ SSmaMgmt smaMgmt = {
}; };
#define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver" #define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver"
#define TD_RSMAINFO_DEL_FILE "rsmainfo.del"
typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter; typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
@ -48,14 +49,11 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables);
static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed); static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed);
static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed); static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed);
static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) { static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) {
// adapt accordingly if definition of SRSmaInfo update // adapt accordingly if definition of SRSmaInfo update
SRSmaInfo *pResult = NULL; SRSmaInfo *pResult = NULL;
int32_t rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *);
ASSERT(pItem->level == TSDB_RETENTION_L1 || pItem->level == TSDB_RETENTION_L2); ASSERT(pItem->level == TSDB_RETENTION_L1 || pItem->level == TSDB_RETENTION_L2);
pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + rsmaInfoHeadLen)); pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + RSMA_INFO_HEAD_LEN));
ASSERT(pResult->pTSchema->numOfCols > 1); ASSERT(pResult->pTSchema->numOfCols > 1);
return pResult; return pResult;
} }
@ -116,8 +114,8 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) {
SRSmaInfoItem *pItem = &pInfo->items[i]; SRSmaInfoItem *pItem = &pInfo->items[i];
if (pItem->taskInfo) { if (pItem->taskInfo) {
if (isDeepFree && pItem->tmrId) { if (isDeepFree && pItem->tmrId) {
smaDebug("vgId:%d, table %" PRIi64 " stop fetch timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId, smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid,
i + 1); pItem->tmrId, i + 1);
taosTmrStopA(&pItem->tmrId); taosTmrStopA(&pItem->tmrId);
} }
tdFreeQTaskInfo(&pItem->taskInfo, SMA_VID(pSma), i + 1); tdFreeQTaskInfo(&pItem->taskInfo, SMA_VID(pSma), i + 1);
@ -337,6 +335,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
} }
pRSmaInfo->pTSchema = pTSchema; pRSmaInfo->pTSchema = pTSchema;
pRSmaInfo->suid = suid; pRSmaInfo->suid = suid;
T_REF_INIT_VAL(pRSmaInfo, 1);
if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) { if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) {
goto _err; goto _err;
@ -392,11 +391,33 @@ int32_t tdProcessRSmaCreate(SSma *pSma, SVCreateStbReq *pReq) {
int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) {
SVnode *pVnode = pSma->pVnode; SVnode *pVnode = pSma->pVnode;
if (!VND_IS_RSMA(pVnode)) { if (!VND_IS_RSMA(pVnode)) {
smaTrace("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name, smaTrace("vgId:%d, not drop rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name,
pReq->suid); pReq->suid);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
if (!pSmaEnv) {
return TSDB_CODE_SUCCESS;
}
SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv);
SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pReq->suid);
if (!pRSmaInfo) {
smaWarn("vgId:%d, drop rsma for stable %s %" PRIi64 " failed no rsma in hash", TD_VID(pVnode), pReq->name,
pReq->suid);
return TSDB_CODE_SUCCESS;
}
// set del flag for data in mem
RSMA_INFO_SET_DEL(pRSmaInfo);
tdUnRefRSmaInfo(pSma, pRSmaInfo);
// save to file
smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid); smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -661,7 +682,6 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
SRSmaInfo *pRSmaInfo = NULL; SRSmaInfo *pRSmaInfo = NULL;
if (!pEnv) { if (!pEnv) {
// only applicable when rsma env exists
return NULL; return NULL;
} }
@ -683,18 +703,21 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
SRSmaInfo *pCowRSmaInfo = NULL; SRSmaInfo *pCowRSmaInfo = NULL;
// lock // lock
taosWLockLatch(SMA_ENV_LOCK(pEnv)); taosWLockLatch(SMA_ENV_LOCK(pEnv));
void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (!taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t))) { // 2-phase lock
if (iRSmaInfo) { void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; if (iRSmaInfo) {
if (pIRSmaInfo) { SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo;
if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) { if (pIRSmaInfo) {
taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) {
smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
return NULL; smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr());
} return NULL;
if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) { }
taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); smaDebug("vgId:%d, clone rsma info succeed for suid:%" PRIu64, SMA_VID(pSma), suid);
return NULL; if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) {
taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
return NULL;
}
} }
} }
} }
@ -703,21 +726,56 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
return pCowRSmaInfo; return pCowRSmaInfo;
} }
/**
* @brief During the drop procedure, only need to delete the object in rsmaInfoHash.
*
* @param pSma
* @param suid
* @return SRSmaInfo*
*/
void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid) {
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
SRSmaStat *pStat = NULL;
SRSmaInfo *pRSmaInfo = NULL;
if (!pEnv) {
return;
}
pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
if (!pStat || !RSMA_INFO_HASH(pStat)) {
return;
}
pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
if (pRSmaInfo) {
if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
tdFreeRSmaInfo(pSma, pRSmaInfo, true);
}
taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
smaDebug("vgId:%d, remove from infoHash for table:%" PRIu64 " succeed", SMA_VID(pSma), suid);
}
}
static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid); SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid);
if (!pRSmaInfo) { if (!pRSmaInfo) {
smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (!pRSmaInfo->items[0].taskInfo) { if (!pRSmaInfo->items[0].taskInfo) {
smaDebug("vgId:%d, return as no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); smaDebug("vgId:%d, execute rsma, no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (inputType == STREAM_INPUT__DATA_SUBMIT) { if (inputType == STREAM_INPUT__DATA_SUBMIT) {
tdRefRSmaInfo(pSma, pRSmaInfo);
tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1); tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1);
tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2); tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2);
tdUnRefRSmaInfo(pSma, pRSmaInfo);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1271,6 +1329,9 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
} }
SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem); SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem);
if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
goto _end;
}
int8_t fetchTriggerStat = int8_t fetchTriggerStat =
atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
@ -1279,13 +1340,14 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma), smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
pItem->level, pRSmaInfo->suid); pItem->level, pRSmaInfo->suid);
tdRefSmaStat(pSma, (SSmaStat *)pStat); // sync procedure => async process
tdRefRSmaInfo(pSma, pRSmaInfo);
SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false); qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK); tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK);
tdUnRefSmaStat(pSma, (SSmaStat *)pStat); tdUnRefRSmaInfo(pSma, pRSmaInfo);
} break; } break;
case TASK_TRIGGER_STAT_PAUSED: { case TASK_TRIGGER_STAT_PAUSED: {
smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused", smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",

4
source/dnode/vnode/src/tq/tqExec.c
View File

@ -68,7 +68,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa
pRsp->rspOffset = *pOffset; pRsp->rspOffset = *pOffset;
return 0; return 0;
} else { } else {
tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1); tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
if (qStreamPrepareScan(task, pOffset) < 0) { if (qStreamPrepareScan(task, pOffset) < 0) {
pRsp->rspOffset = *pOffset; pRsp->rspOffset = *pOffset;
return 0; return 0;
@ -106,7 +106,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa
} }
if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1); tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
qStreamPrepareScan(task, pOffset); qStreamPrepareScan(task, pOffset);
continue; continue;
} }

2
source/dnode/vnode/src/tq/tqPush.c
View File

@ -243,7 +243,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
void* data = taosMemoryMalloc(msgLen); void* data = taosMemoryMalloc(msgLen);
if (data == NULL) { if (data == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
qError("failed to copy data for stream since out of memory"); tqError("failed to copy data for stream since out of memory");
return -1; return -1;
} }
memcpy(data, msg, msgLen); memcpy(data, msg, msgLen);

2
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -245,7 +245,7 @@ static int32_t tsdbApplyDelFileChange(STsdbFS *pFS, SDelFile *pFrom, SDelFile *p
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
if (pFrom && pTo) { if (pFrom && pTo) {
if (pFrom != pTo) { if (!tsdbDelFileIsSame(pFrom, pTo)) {
tsdbDelFileName(pFS->pTsdb, pFrom, fname); tsdbDelFileName(pFS->pTsdb, pFrom, fname);
if (taosRemoveFile(fname) < 0) { if (taosRemoveFile(fname) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);

2
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -140,6 +140,8 @@ bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT fty
} }
} }
bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; }
int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype) { int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype) {
int32_t code = 0; int32_t code = 0;
int64_t n; int64_t n;

69
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -1453,38 +1453,71 @@ static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVer
(pBlock->minVersion <= pVerRange->maxVer); (pBlock->minVersion <= pVerRange->maxVer);
} }
static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) {
if (pBlockScanInfo->delSkyline == NULL) {
return false;
}
TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0); static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) {
TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);
// ts is not overlap
if (pBlock->minKey.ts > pLast->ts || pBlock->maxKey.ts < pFirst->ts) {
return false;
}
int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1;
// version is not overlap
size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);
for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += step) {
for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) {
TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);
if (p->ts >= pBlock->minKey.ts && p->ts <= pBlock->maxKey.ts) { if (p->ts >= pBlock->minKey.ts && p->ts <= pBlock->maxKey.ts) {
if (p->version >= pBlock->minVersion) { if (p->version >= pBlock->minVersion) {
return true; return true;
} }
} else if (p->ts > pBlock->maxKey.ts) { } else if (p->ts < pBlock->minKey.ts) { // p->ts < pBlock->minKey.ts
if (p->version >= pBlock->minVersion) {
if (i < num - 1) {
TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);
if (i + 1 == num - 1) { // pnext is the last point
if (pnext->ts >= pBlock->minKey.ts) {
return true;
}
} else {
if (pnext->ts >= pBlock->minKey.ts && pnext->version >= pBlock->minVersion) {
return true;
}
}
} else { // it must be the last point
ASSERT(p->version == 0);
}
}
} else { // (p->ts > pBlock->maxKey.ts) {
return false; return false;
} }
} }
ASSERT(0);
return false; return false;
} }
static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) {
if (pBlockScanInfo->delSkyline == NULL) {
return false;
}
// ts is not overlap
TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);
TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);
if (pBlock->minKey.ts > pLast->ts || pBlock->maxKey.ts < pFirst->ts) {
return false;
}
// version is not overlap
if (ASCENDING_TRAVERSE(order)) {
return doCheckforDatablockOverlap(pBlockScanInfo, pBlock);
} else {
int32_t index = pBlockScanInfo->fileDelIndex;
while(1) {
TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);
if (p->ts > pBlock->minKey.ts && index > 0) {
index -= 1;
} else { // find the first point that is smaller than the minKey.ts of dataBlock.
break;
}
}
return doCheckforDatablockOverlap(pBlockScanInfo, pBlock);
}
}
// 1. the version of all rows should be less than the endVersion // 1. the version of all rows should be less than the endVersion
// 2. current block should not overlap with next neighbor block // 2. current block should not overlap with next neighbor block
// 3. current timestamp should not be overlap with each other // 3. current timestamp should not be overlap with each other

2
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -246,7 +246,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
tsdbDelFileName(pTsdb, pFile, fname); tsdbDelFileName(pTsdb, pFile, fname);
pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ); pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ);
if (pDelFReader == NULL) { if (pDelFReader->pReadH == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pDelFReader); taosMemoryFree(pDelFReader);
goto _err; goto _err;

3
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -53,6 +53,7 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
*(int64_t *)(dc.data + dc.pos) = uid; *(int64_t *)(dc.data + dc.pos) = uid;
*(int64_t *)(dc.data + dc.pos + 8) = ctime; *(int64_t *)(dc.data + dc.pos + 8) = ctime;
vTrace("vgId:%d, table:%s uid:%" PRId64 " is generated", pVnode->config.vgId, name, uid);
tEndDecode(&dc); tEndDecode(&dc);
} }
@ -381,7 +382,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
goto end; goto end;
} }
vInfo("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp); vDebug("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp);
int32_t ret = metaTtlDropTable(pVnode->pMeta, ttlReq.timestamp, tbUids); int32_t ret = metaTtlDropTable(pVnode->pMeta, ttlReq.timestamp, tbUids);
if (ret != 0) { if (ret != 0) {
goto end; goto end;

49
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -17,35 +17,22 @@
#include "vnd.h" #include "vnd.h"
static inline bool vnodeIsMsgBlock(tmsg_t type) { static inline bool vnodeIsMsgBlock(tmsg_t type) {
return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_ALTER_REPLICA); return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) ||
(type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL);
} }
static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; } static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; }
static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) { static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
if (!vnodeIsMsgBlock(type)) return; if (vnodeIsMsgBlock(pMsg->msgType)) {
vTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType));
int32_t count = atomic_add_fetch_32(&pVnode->blockCount, 1); tsem_wait(&pVnode->syncSem);
vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type)); }
} }
static inline void vnodeWaitBlockMsg(SVnode *pVnode) { static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
int32_t count = atomic_load_32(&pVnode->blockCount); if (vnodeIsMsgBlock(pMsg->msgType)) {
if (count <= 0) return; vTrace("vgId:%d, msg:%p post block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType));
vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count);
tsem_wait(&pVnode->syncSem);
}
static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) {
if (!vnodeIsMsgBlock(type)) return;
int32_t count = atomic_load_32(&pVnode->blockCount);
if (count <= 0) return;
count = atomic_sub_fetch_32(&pVnode->blockCount, 1);
vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
if (count <= 0) {
tsem_post(&pVnode->syncSem); tsem_post(&pVnode->syncSem);
} }
} }
@ -143,6 +130,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
int32_t code = 0; int32_t code = 0;
SRpcMsg *pMsg = NULL; SRpcMsg *pMsg = NULL;
vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs);
for (int32_t m = 0; m < numOfMsgs; m++) { for (int32_t m = 0; m < numOfMsgs; m++) {
if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
@ -165,13 +154,14 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
if (rsp.info.handle != NULL) { if (rsp.info.handle != NULL) {
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} }
} else if (code == 0) {
vnodeWaitBlockMsg(pVnode, pMsg);
} else {
} }
} }
} }
if (code == 0) { if (code < 0) {
vnodeAccumBlockMsg(pVnode, pMsg->msgType);
} else if (code < 0) {
if (terrno == TSDB_CODE_SYN_NOT_LEADER) { if (terrno == TSDB_CODE_SYN_NOT_LEADER) {
vnodeRedirectRpcMsg(pVnode, pMsg); vnodeRedirectRpcMsg(pVnode, pMsg);
} else { } else {
@ -182,15 +172,12 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} }
} }
} else {
} }
vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code); vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
vnodeWaitBlockMsg(pVnode);
} }
void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
@ -213,7 +200,7 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
} }
} }
vnodePostBlockMsg(pVnode, pMsg->msgType); vnodePostBlockMsg(pVnode, pMsg);
if (rsp.info.handle != NULL) { if (rsp.info.handle != NULL) {
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} }
@ -418,7 +405,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon
tmsgSendRsp(&rpcMsg); tmsgSendRsp(&rpcMsg);
} }
vnodePostBlockMsg(pVnode, TDMT_VND_ALTER_REPLICA); vnodePostBlockMsg(pVnode, pMsg);
} }
static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {

8
source/libs/function/src/builtins.c
View File

@ -1423,14 +1423,6 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
} }
static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
for (int32_t i = 0; i < numOfParams; ++i) {
SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
if (QUERY_NODE_VALUE == nodeType(pParamNode)) {
return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
}
}
pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

11
source/libs/parser/src/parInsert.c
View File

@ -2161,7 +2161,7 @@ static void smlDestroyTableHandle(void* pHandle) {
tdDestroySVCreateTbReq(&handle->createTblReq); tdDestroySVCreateTbReq(&handle->createTblReq);
} }
static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema) { static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema, bool isTag) {
col_id_t nCols = pColList->numOfCols; col_id_t nCols = pColList->numOfCols;
pColList->numOfBound = 0; pColList->numOfBound = 0;
@ -2177,7 +2177,8 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS
SSmlKv* kv = taosArrayGetP(cols, i); SSmlKv* kv = taosArrayGetP(cols, i);
SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key};
col_id_t t = lastColIdx + 1; col_id_t t = lastColIdx + 1;
col_id_t index = findCol(&sToken, t, nCols, pSchema); col_id_t index = ((t == 0 && !isTag) ? 0 : findCol(&sToken, t, nCols, pSchema));
uDebug("SML, index:%d, t:%d, ncols:%d, kv->name:%s", index, t, nCols, kv->key);
if (index < 0 && t > 0) { if (index < 0 && t > 0) {
index = findCol(&sToken, 0, t, pSchema); index = findCol(&sToken, 0, t, pSchema);
isOrdered = false; isOrdered = false;
@ -2312,7 +2313,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table
SSchema* pTagsSchema = getTableTagSchema(pTableMeta); SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
setBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta)); setBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta));
int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema); int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, true);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
buildInvalidOperationMsg(&pBuf, "bound tags error"); buildInvalidOperationMsg(&pBuf, "bound tags error");
return ret; return ret;
@ -2343,7 +2344,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
SSchema* pSchema = getTableColumnSchema(pTableMeta); SSchema* pSchema = getTableColumnSchema(pTableMeta);
ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema); ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema, false);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
buildInvalidOperationMsg(&pBuf, "bound cols error"); buildInvalidOperationMsg(&pBuf, "bound cols error");
return ret; return ret;
@ -2401,7 +2402,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
} else { } else {
int32_t colLen = kv->length; int32_t colLen = kv->length;
if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
// uError("SML:data before:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision);
kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision);
// uError("SML:data after:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision);
} }
if (IS_VAR_DATA_TYPE(kv->type)) { if (IS_VAR_DATA_TYPE(kv->type)) {

1
source/libs/sync/inc/syncInt.h
View File

@ -192,6 +192,7 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms);
int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode); int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode);
int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode);
int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode);
int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode);
// utils -------------- // utils --------------
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);

6
source/libs/sync/src/syncMain.c
View File

@ -1298,6 +1298,12 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) {
return ret; return ret;
} }
int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) {
syncNodeStopHeartbeatTimer(pSyncNode);
syncNodeStartHeartbeatTimer(pSyncNode);
return 0;
}
// utils -------------- // utils --------------
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
SEpSet epSet; SEpSet epSet;

4
source/libs/sync/src/syncRaftLog.c
View File

@ -313,7 +313,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
index, err, err, errStr, sysErr, sysErrStr); index, err, err, errStr, sysErr, sysErrStr);
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
syncNodeEventLog(pData->pSyncNode, logBuf); // syncNodeEventLog(pData->pSyncNode, logBuf);
} else { } else {
syncNodeErrorLog(pData->pSyncNode, logBuf); syncNodeErrorLog(pData->pSyncNode, logBuf);
} }
@ -499,7 +499,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
index, err, err, errStr, sysErr, sysErrStr); index, err, err, errStr, sysErr, sysErrStr);
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
syncNodeEventLog(pData->pSyncNode, logBuf); // syncNodeEventLog(pData->pSyncNode, logBuf);
} else { } else {
syncNodeErrorLog(pData->pSyncNode, logBuf); syncNodeErrorLog(pData->pSyncNode, logBuf);
} }

5
source/libs/tdb/src/db/tdbBtree.c
View File

@ -103,10 +103,9 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
// if pgno == 0 fetch new btree root leaf page // if pgno == 0 fetch new btree root leaf page
if (pgno == 0) { if (pgno == 0) {
// fetch page & insert into main db // fetch page & insert into main db
// allocate a new child page
SPage *pPage; SPage *pPage;
TXN txn; TXN txn;
tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
pPager->inTran = 1; pPager->inTran = 1;
@ -118,8 +117,6 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
return -1; return -1;
} }
// TODO: Need to zero the page
ret = tdbPagerWrite(pPager, pPage); ret = tdbPagerWrite(pPager, pPage);
if (ret < 0) { if (ret < 0) {
return -1; return -1;

22
source/libs/tdb/src/db/tdbPager.c
View File

@ -473,12 +473,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
return -1; return -1;
} }
TXN txn;
tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
SBtreeInitPageArg iArg;
iArg.pBt = pBt;
iArg.flags = 0;
for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) { for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) {
// read pgno & the page from journal // read pgno & the page from journal
SPgno pgno; SPgno pgno;
@ -494,20 +488,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
return -1; return -1;
} }
/*
ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &iArg, &txn);
if (ret < 0) {
return -1;
}
// write the page to db
ret = tdbPagerWritePageToDB(pPager, pPage);
if (ret < 0) {
return -1;
}
tdbPCacheRelease(pPager->pCache, pPage, &txn);
*/
i64 offset = pPager->pageSize * (pgno - 1); i64 offset = pPager->pageSize * (pgno - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
ASSERT(0); ASSERT(0);
@ -523,8 +503,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
tdbOsFSync(pPager->fd); tdbOsFSync(pPager->fd);
tdbTxnClose(&txn);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
tdbOsClose(jfd); tdbOsClose(jfd);

11
tests/pytest/crash_gen/crash_gen_main.py
View File

@ -30,6 +30,7 @@ import argparse
import sys import sys
import os import os
import io import io
import datetime
import signal import signal
import traceback import traceback
import requests import requests
@ -1107,14 +1108,20 @@ class Database:
# TODO: fix the error as result of above: "tsdb timestamp is out of range" # TODO: fix the error as result of above: "tsdb timestamp is out of range"
@classmethod @classmethod
def setupLastTick(cls): def setupLastTick(cls):
t1 = datetime.datetime(2020, 6, 1) # start time will be auto generated , start at 10 years ago local time
local_time = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-16]
local_epoch_time = [int(i) for i in local_time.split("-")]
#local_epoch_time will be such as : [2022, 7, 18]
t1 = datetime.datetime(local_epoch_time[0]-5, local_epoch_time[1], local_epoch_time[2])
t2 = datetime.datetime.now() t2 = datetime.datetime.now()
# maybe a very large number, takes 69 years to exceed Python int range # maybe a very large number, takes 69 years to exceed Python int range
elSec = int(t2.timestamp() - t1.timestamp()) elSec = int(t2.timestamp() - t1.timestamp())
elSec2 = (elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500)) * \ elSec2 = (elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500)) * \
500 # a number representing seconds within 10 years 500 # a number representing seconds within 10 years
# print("elSec = {}".format(elSec)) # print("elSec = {}".format(elSec))
t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years
t3 = datetime.datetime(local_epoch_time[0]-10, local_epoch_time[1], local_epoch_time[2]) # default "keep" is 10 years
t4 = datetime.datetime.fromtimestamp( t4 = datetime.datetime.fromtimestamp(
t3.timestamp() + elSec2) # see explanation above t3.timestamp() + elSec2) # see explanation above
Logging.debug("Setting up TICKS to start from: {}".format(t4)) Logging.debug("Setting up TICKS to start from: {}".format(t4))

5
tests/script/jenkins/basic.txt
View File

@ -248,8 +248,8 @@
./test.sh -f tsim/stream/sliding.sim ./test.sh -f tsim/stream/sliding.sim
# ---- transaction # ---- transaction
./test.sh -f tsim/trans/lossdata1.sim ./test.sh -f tsim/trans/lossdata1.sim
./test.sh -f tsim/trans/create_db.sim ./test.sh -f tsim/trans/create_db.sim
# ---- tmq # ---- tmq
./test.sh -f tsim/tmq/basic1.sim ./test.sh -f tsim/tmq/basic1.sim
@ -312,6 +312,7 @@
./test.sh -f tsim/valgrind/checkError3.sim ./test.sh -f tsim/valgrind/checkError3.sim
./test.sh -f tsim/valgrind/checkError4.sim ./test.sh -f tsim/valgrind/checkError4.sim
./test.sh -f tsim/valgrind/checkError5.sim ./test.sh -f tsim/valgrind/checkError5.sim
./test.sh -f tsim/valgrind/checkError6.sim
# --- vnode # --- vnode
# unsupport ./test.sh -f tsim/vnode/replica3_basic.sim # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim

125
tests/script/tmp/data.sim
View File

@ -1,18 +1,125 @@
system sh/stop_dnodes.sh system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c telemetryReporting -v 1 system sh/deploy.sh -n dnode2 -i 2
system sh/cfg.sh -n dnode1 -c telemetryInterval -v 1 system sh/deploy.sh -n dnode3 -i 3
system sh/cfg.sh -n dnode1 -c telemetryServer -v localhost system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c telemetryPort -v 80
return
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
sql connect sql connect
sql create database db print =============== step1: create dnodes
sql create table db.tb (ts timestamp, i int) sql create dnode $hostname port 7200
sql insert into db.tb values(now, 1) sql create dnode $hostname port 7300
sql create dnode $hostname port 7400
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> rows: $rows
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data24 $data35
if $rows != 4 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
if $data(3)[4] != ready then
goto step1
endi
if $data(4)[4] != ready then
goto step1
endi
print =============== step2: create database
sql create database db vgroups 1 replica 3
sql show databases
if $rows != 3 then
return -1
endi
if $data(db)[4] != 3 then
return -1
endi
sql show dnodes
if $data(2)[2] != 1 then
return -1
endi
if $data(3)[2] != 1 then
return -1
endi
if $data(4)[2] != 1 then
return -1
endi
# vnodes
sql show dnodes
if $data(2)[2] != 1 then
return -1
endi
if $data(3)[2] != 1 then
return -1
endi
if $data(4)[2] != 1 then
return -1
endi
# v1_dnode
$hasleader = 0
$x = 0
step2:
$x = $x + 1
sleep 1000
if $x == 20 then
print ====> dnode not ready!
return -1
endi
sql show db.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08
if $data(2)[3] != 2 then
return -1
endi
if $data(2)[5] != 3 then
return -1
endi
if $data(2)[7] != 4 then
return -1
endi
if $data(2)[4] == leader then
$hasleader = 1
endi
if $data(2)[6] == leader then
$hasleader = 1
endi
if $data(2)[8] == leader then
$hasleader = 1
endi
if $hasleader != 1 then
goto step2
endi
sql use db;
sql create table stb (ts timestamp, c int) tags (t int);
sql create table t0 using stb tags (0);
sql insert into t0 values(now, 1);
sql show db.stables;
sql show db.tables;
sql show db.vgroups;
return
print ======== start back print ======== start back
run_back tmp/back.sim run_back tmp/back.sim

21
tests/script/tmp/monitor.sim
View File

@ -6,32 +6,21 @@ system sh/cfg.sh -n dnode1 -c monitorInterval -v 1
system sh/cfg.sh -n dnode1 -c monitorComp -v 1 system sh/cfg.sh -n dnode1 -c monitorComp -v 1
#system sh/cfg.sh -n dnode1 -c supportVnodes -v 128 #system sh/cfg.sh -n dnode1 -c supportVnodes -v 128
#system sh/cfg.sh -n dnode1 -c telemetryReporting -v 1
#system sh/cfg.sh -n dnode1 -c telemetryInterval -v 1
#system sh/cfg.sh -n dnode1 -c telemetryServer -v localhost
#system sh/cfg.sh -n dnode1 -c telemetryPort -v 80
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sql connect sql connect
print =============== show dnodes print =============== show dnodes
sleep 2000
sql create database db vgroups 2; sql create database db vgroups 2;
sql use db; sql use db;
sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd"; sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd";
sleep 2000
print =============== create drop qnode 1 print =============== create drop qnode 1
sql create qnode on dnode 1 sql create qnode on dnode 1
sql create snode on dnode 1 sql create snode on dnode 1
sql create bnode on dnode 1 sql create bnode on dnode 1
return
print =============== restart
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
return
system sh/deploy.sh -n dnode2 -i 2
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT

2
tests/script/tsim/valgrind/checkError1.sim
View File

@ -152,7 +152,7 @@ endi
system_content sh/checkValgrind.sh -n dnode2 system_content sh/checkValgrind.sh -n dnode2
print cmd return result ----> [ $system_content ] print cmd return result ----> [ $system_content ]
if $system_content > 4 then if $system_content > 0 then
return -1 return -1
endi endi

33
tests/script/tsim/valgrind/checkError5.sim
View File

@ -26,6 +26,7 @@ print =============== step2: create db
sql create database db sql create database db
sql use db sql use db
sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd"
sql create table db.c1 using db.stb tags(101, 102, "103")
print =============== step3: alter stb print =============== step3: alter stb
sql_error alter table db.stb add column ts int sql_error alter table db.stb add column ts int
@ -42,9 +43,8 @@ sql alter table db.stb drop tag c1
sql alter table db.stb drop tag t5 sql alter table db.stb drop tag t5
sql alter table db.stb MODIFY tag t3 binary(32) sql alter table db.stb MODIFY tag t3 binary(32)
sql alter table db.stb rename tag t1 tx sql alter table db.stb rename tag t1 tx
sql alter table db.stb comment 'abcde' ; sql alter table db.stb comment 'abcde' ;
goto _OVER sql drop table db.stb
print =============== step4: alter tb print =============== step4: alter tb
sql create table tb (ts timestamp, a int) sql create table tb (ts timestamp, a int)
@ -66,6 +66,35 @@ sql alter table tb add column h binary(10)
sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb
sql select * from tb order by ts desc sql select * from tb order by ts desc
print =============== step5: alter stb and insert data
sql create table stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd"
sql show db.stables
sql describe stb
sql_error alter table stb add column ts int
sql create table db.ctb using db.stb tags(101, 102, "103")
sql insert into db.ctb values(now, 1, "2")
sql show db.tables
sql select * from db.stb
sql select * from tb
sql alter table stb add column c3 int
sql describe stb
sql select * from db.stb
sql select * from tb
sql insert into db.ctb values(now+1s, 1, 2, 3)
sql select * from db.stb
sql alter table db.stb add column c4 bigint
sql select * from db.stb
sql insert into db.ctb values(now+2s, 1, 2, 3, 4)
sql alter table db.stb drop column c1
sql reset query cache
sql select * from tb
sql insert into db.ctb values(now+3s, 2, 3, 4)
sql select * from db.stb
_OVER: _OVER:
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
print =============== check print =============== check

2
tests/script/tsim/valgrind/checkError6.sim
View File

@ -68,7 +68,7 @@ $null=
system_content sh/checkValgrind.sh -n dnode1 system_content sh/checkValgrind.sh -n dnode1
print cmd return result ----> [ $system_content ] print cmd return result ----> [ $system_content ]
if $system_content > 0 then if $system_content > 3 then
return -1 return -1
endi endi

86
tests/system-test/0-others/cachelast.py → tests/system-test/0-others/cachemodel.py
View File

@ -56,6 +56,16 @@ class TDTestCase:
} }
return numbers.get(value, 'other') return numbers.get(value, 'other')
def getCacheModelNum(self,str):
numbers = {
"none" : 0,
"last_row" : 1,
"last_value" : 2,
"both" : 3
}
return numbers.get(str, 'other')
def prepare_datas(self): def prepare_datas(self):
for i in range(4): for i in range(4):
str = self.getCacheModelStr(i) str = self.getCacheModelStr(i)
@ -69,7 +79,7 @@ class TDTestCase:
tdSql.execute(" insert into tb1 values(now , %d, %f)" %(k,k*10) ) tdSql.execute(" insert into tb1 values(now , %d, %f)" %(k,k*10) )
tdSql.execute(" insert into tb2 values(now , %d, %f)" %(k,k*10) ) tdSql.execute(" insert into tb2 values(now , %d, %f)" %(k,k*10) )
def check_cache_last_sets(self): def check_cachemodel_sets(self):
# check cache_last value for database # check cache_last value for database
@ -84,52 +94,54 @@ class TDTestCase:
# print(cache_last_value) # print(cache_last_value)
if dbname in ["information_schema" , "performance_schema"]: if dbname in ["information_schema" , "performance_schema"]:
continue continue
cache_lasts[dbname]=cache_last_value cache_lasts[dbname]=self.getCacheModelNum(cache_last_value)
# cache_last_set value # cache_last_set value
for k , v in cache_lasts.items(): for k , v in cache_lasts.items():
if k=="testdb_"+str(v): if k=="testdb_"+str(self.getCacheModelStr(v)):
tdLog.info(" database %s cache_last value check pass, value is %s "%(k,v) ) tdLog.info(" database %s cache_last value check pass, value is %s "%(k,self.getCacheModelStr(v)) )
else: else:
tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,v) ) tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,self.getCacheModelStr(v)) )
# # check storage layer implementation # # check storage layer implementation
# buildPath = self.getBuildPath() buildPath = self.getBuildPath()
# if (buildPath == ""): if (buildPath == ""):
# tdLog.exit("taosd not found!") tdLog.exit("taosd not found!")
# else: else:
# tdLog.info("taosd found in %s" % buildPath) tdLog.info("taosd found in %s" % buildPath)
# dataPath = buildPath + "/../sim/dnode1/data" dataPath = buildPath + "/../sim/dnode1/data"
# abs_vnodePath = os.path.abspath(dataPath)+"/vnode/" abs_vnodePath = os.path.abspath(dataPath)+"/vnode/"
# tdLog.info("abs_vnodePath: %s" % abs_vnodePath) tdLog.info("abs_vnodePath: %s" % abs_vnodePath)
# tdSql.query(" show dnodes ") tdSql.query(" show dnodes ")
# dnode_id = tdSql.queryResult[0][0] dnode_id = tdSql.queryResult[0][0]
# for dbname in cache_lasts.keys(): for dbname in cache_lasts.keys():
# print(dbname) # print(dbname)
# tdSql.execute(" use %s" % dbname) tdSql.execute(" use %s" % dbname)
# tdSql.query(" show vgroups ") tdSql.query(" show vgroups ")
# vgroups_infos = tdSql.queryResult vgroups_infos = tdSql.queryResult
# for vgroup_info in vgroups_infos: for vgroup_info in vgroups_infos:
# vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json" vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json"
# vnode_info_of_db = f"cat {vnode_json}" vnode_info_of_db = f"cat {vnode_json}"
# vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8") vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8")
# infoDict = json.loads(vnode_info) infoDict = json.loads(vnode_info)
# vnode_json_of_dbname = f"{dnode_id}."+ dbname vnode_json_of_dbname = f"{dnode_id}."+ dbname
# config = infoDict["config"] config = infoDict["config"]
# if infoDict["config"]["dbname"] == vnode_json_of_dbname: if infoDict["config"]["dbname"] == vnode_json_of_dbname:
# if "cachelast" in infoDict["config"]: if "cacheLast" in infoDict["config"]:
# if int(infoDict["config"]["cachelast"]) != cache_lasts[dbname]: if int(infoDict["config"]["cacheLast"]) != cache_lasts[dbname]:
# tdLog.exit("cachelast value is error in vnode.json of vnode%d "%(vgroup_info[0])) tdLog.exit("cachemodel value is error in vnode.json of vnode%d "%(vgroup_info[0]))
# else: else:
# tdLog.exit("cachelast not found in vnode.json of vnode%d "%(vgroup_info[0])) tdLog.info("cachemodel value is success in vnode.json of vnode%d "%(vgroup_info[0]))
else:
tdLog.exit("cacheLast not found in vnode.json of vnode%d "%(vgroup_info[0]))
def restart_check_cache_last_sets(self): def restart_check_cachemodel_sets(self):
for i in range(3): for i in range(3):
tdSql.query("show dnodes") tdSql.query("show dnodes")
@ -137,14 +149,14 @@ class TDTestCase:
tdDnodes.stop(index) tdDnodes.stop(index)
tdDnodes.start(index) tdDnodes.start(index)
time.sleep(3) time.sleep(3)
self.check_cache_last_sets() self.check_cachemodel_sets()
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
self.illegal_params() self.illegal_params()
self.prepare_datas() self.prepare_datas()
self.check_cache_last_sets() self.check_cachemodel_sets()
self.restart_check_cache_last_sets() self.restart_check_cachemodel_sets()
def stop(self): def stop(self):
tdSql.close() tdSql.close()

190
tests/system-test/1-insert/delete_data.py Normal file
View File

@ -0,0 +1,190 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import random
import string
from numpy import logspace
from util import constant
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import TDSetSql
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(),logSql)
self.dbname = 'db_test'
self.setsql = TDSetSql()
self.ntbname = 'ntb'
self.rowNum = 10
self.tbnum = 20
self.ts = 1537146000000
self.binary_str = 'taosdata'
self.nchar_str = '涛思数据'
self.str_length = 20
self.column_dict = {
'col1': 'tinyint',
'col2': 'smallint',
'col3': 'int',
'col4': 'bigint',
'col5': 'tinyint unsigned',
'col6': 'smallint unsigned',
'col7': 'int unsigned',
'col8': 'bigint unsigned',
'col9': 'float',
'col10': 'double',
'col11': 'bool',
'col12': f'binary({self.str_length})',
'col13': f'nchar({self.str_length})',
}
self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX)
self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX)
self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX)
self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX)
self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX)
self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX)
self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX)
self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX)
self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX)
self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300))
self.bool_val = random.randint(0,100)%2
self.binary_val = tdCom.getLongName(random.randint(0,self.str_length))
self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length))
self.base_data = {
'tinyint':self.tinyint_val,
'smallint':self.smallint_val,
'int':self.int_val,
'bigint':self.bigint_val,
'tinyint unsigned':self.untingint_val,
'smallint unsigned':self.unsmallint_val,
'int unsigned':self.unint_val,
'bigint unsigned':self.unbigint_val,
'bool':self.bool_val,
'float':self.float_val,
'double':self.double_val,
'binary':self.binary_val,
'nchar':self.nchar_val
}
def insert_base_data(self,col_type,tbname,rows,base_data):
for i in range(rows):
if col_type.lower() == 'tinyint':
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})')
elif col_type.lower() == 'smallint':
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})')
elif col_type.lower() == 'int':
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})')
elif col_type.lower() == 'bigint':
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})')
elif col_type.lower() == 'tinyint unsigned':
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})')
elif col_type.lower() == 'smallint unsigned':
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})')
elif col_type.lower() == 'int unsigned':
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})')
elif col_type.lower() == 'bigint unsigned':
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})')
elif col_type.lower() == 'bool':
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})')
elif col_type.lower() == 'float':
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})')
elif col_type.lower() == 'double':
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})')
elif 'binary' in col_type.lower():
tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''')
elif 'nchar' in col_type.lower():
tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''')
def delete_all_data(self,tbname,col_type,row_num,base_data,dbname):
tdSql.execute(f'delete from {tbname}')
tdSql.execute(f'flush database {dbname}')
tdSql.execute('reset query cache')
tdSql.query(f'select * from {tbname}')
tdSql.checkRows(0)
self.insert_base_data(col_type,tbname,row_num,base_data)
tdSql.execute(f'flush database {dbname}')
tdSql.execute('reset query cache')
tdSql.query(f'select * from {tbname}')
tdSql.checkRows(row_num)
def delete_one_row(self,tbname,column_type,column_name,base_data,dbname):
tdSql.execute(f'delete from {tbname} where ts={self.ts}')
tdSql.execute(f'flush database {dbname}')
tdSql.execute('reset query cache')
tdSql.query(f'select {column_name} from {tbname}')
tdSql.checkRows(self.rowNum-1)
tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}')
tdSql.checkRows(0)
if 'binary' in column_type.lower():
tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''')
elif 'nchar' in column_type.lower():
tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''')
else:
tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})')
tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}')
if column_type.lower() == 'float' or column_type.lower() == 'double':
if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001:
tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0])
else:
tdLog.exit(f'{column_type} data check failure')
elif 'binary' in column_type.lower():
tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary'])
elif 'nchar' in column_type.lower():
tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar'])
else:
tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type])
def delete_rows(self):
pass
def delete_error(self,tbname,column_name,column_type,base_data):
for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']:
if 'binary' in column_type.lower():
tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''')
elif 'nchar' in column_type.lower():
tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''')
else:
tdSql.error('delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}')
def delete_data_ntb(self):
tdSql.execute(f'create database if not exists {self.dbname}')
tdSql.execute(f'use {self.dbname}')
for col_name,col_type in self.column_dict.items():
tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})')
self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data)
self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.dbname)
self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname)
self.delete_error(self.ntbname,col_name,col_type,self.base_data)
for i in range(self.rowNum):
tdSql.execute(f'delete from {self.ntbname} where ts>{self.ts+i}')
tdSql.execute(f'flush database {self.dbname}')
tdSql.execute('reset query cache')
tdSql.query(f'select {col_name} from {self.ntbname}')
tdSql.checkRows(i+1)
self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data)
tdSql.execute(f'drop table {self.ntbname}')
def run(self):
self.delete_data_ntb()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

49
tests/system-test/1-insert/insert_drop.py Normal file
View File

@ -0,0 +1,49 @@
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
import threading
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def genMultiThreadSeq(self, sql_list):
tlist = list()
for insert_sql in sql_list:
t = threading.Thread(target=tdSql.execute, args=(insert_sql,))
tlist.append(t)
return tlist
def multiThreadRun(self, tlist):
for t in tlist:
t.start()
for t in tlist:
t.join()
def run(self):
tdSql.prepare()
tdSql.execute('create database if not exists test;')
tdSql.execute('create table test.stb (ts timestamp, c11 int, c12 float ) TAGS(t11 int, t12 int );')
tdSql.execute('create table test.tb using test.stb TAGS (1, 1);')
sql_list = list()
for i in range(5):
sql = f'insert into test.tb values (now-{i}m, {i}, {i});'
sql_list.append(sql)
sql_list.append(f'drop database test;')
tlist = self.genMultiThreadSeq(sql_list)
self.multiThreadRun(tlist)
tdSql.query(f'show databases')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

229
tests/system-test/1-insert/update_data.py Normal file
View File

@ -0,0 +1,229 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import random
import string
from util import constant
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import TDSetSql
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(),logSql)
self.setsql = TDSetSql()
self.dbname = 'db_test'
self.ntbname = 'ntb'
self.stbname = 'stb'
self.ctbname = 'ctb'
self.ts = 1537146000000
self.str_length = 20
self.column_dict = {
'col1': 'tinyint',
'col2': 'smallint',
'col3': 'int',
'col4': 'bigint',
'col5': 'tinyint unsigned',
'col6': 'smallint unsigned',
'col7': 'int unsigned',
'col8': 'bigint unsigned',
'col9': 'float',
'col10': 'double',
'col11': 'bool',
'col12': f'binary({self.str_length})',
'col13': f'nchar({self.str_length})',
'col_ts' : 'timestamp'
}
def data_check(self,tbname,col_name,col_type,value):
tdSql.query(f'select {col_name} from {tbname}')
if col_type.lower() == 'float' or col_type.lower() == 'double':
if abs(tdSql.queryResult[0][0] - value) / value <= 0.0001:
tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0])
else:
tdLog.exit(f'{col_name} data check failure')
elif col_type.lower() == 'timestamp':
tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f")))
else:
tdSql.checkEqual(tdSql.queryResult[0][0],value)
def update_and_check_data(self,tbname,col_name,col_type,value,dbname):
if 'binary' in col_type.lower() or 'nchar' in col_type.lower():
tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")')
else:
tdSql.execute(f'insert into {tbname} values({self.ts},{value})')
self.data_check(tbname,col_name,col_type,value)
tdSql.execute(f'flush database {dbname}')
tdSql.execute('reset query cache')
self.data_check(tbname,col_name,col_type,value)
for func in ['first','last']:
tdSql.execute(f'select {func}({col_name}) from {tbname}')
def error_check(self,tbname,column_dict,tb_type=None,stbname=None):
str_length = self.str_length+1
for col_name,col_type in column_dict.items():
if tb_type == 'ntb':
tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})')
elif tb_type == 'ctb':
tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)')
tdSql.execute(f'create table {tbname} using {stbname} tags(1)')
tdSql.execute(f'insert into {tbname} values({self.ts},null)')
if col_type.lower() == 'double':
for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]:
tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
elif col_type.lower() == 'float':
for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]:
tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
elif 'binary' in col_type.lower() or 'nchar' in col_type.lower():
for error_value in [tdCom.getLongName(str_length)]:
tdSql.error(f'insert into {tbname} values({self.ts},"{error_value}")')
elif col_type.lower() == 'bool':
for error_value in [tdCom.getLongName(self.str_length)]:
tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
elif col_type.lower() == 'tinyint':
for error_value in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
elif col_type.lower() == 'smallint':
for error_value in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
elif col_type.lower() == 'int':
for error_value in [constant.INT_MIN-1,constant.INT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
elif col_type.lower() == 'bigint':
for error_value in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
elif col_type.lower() == 'tinyint unsigned':
for error_value in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
elif col_type.lower() == 'smallint unsigned':
for error_value in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
elif col_type.lower() == 'int unsigned':
for error_value in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
elif col_type.lower() == 'bigint unsigned':
for error_value in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
tdSql.execute(f'drop table {tbname}')
if tb_type == 'ctb':
tdSql.execute(f'drop table {stbname}')
def update_data_check(self,tbname,column_dict,dbname,tb_type=None,stbname=None):
up_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX)
up_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX)
up_int = random.randint(constant.INT_MIN,constant.INT_MAX)
up_bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX)
up_untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX)
up_unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX)
up_unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX)
up_unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX)
up_bool = random.randint(0,100)%2
up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX)
up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300))
binary_length = random.randint(0,self.str_length)
nchar_length = random.randint(0,self.str_length)
up_binary = tdCom.getLongName(binary_length)
up_nchar = tdCom.getLongName(nchar_length)
for col_name,col_type in column_dict.items():
if tb_type == 'ntb':
tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})')
elif tb_type == 'ctb':
tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)')
tdSql.execute(f'create table {tbname} using {stbname} tags(1)')
tdSql.execute(f'insert into {tbname} values({self.ts},null)')
if col_type.lower() == 'tinyint':
self.update_and_check_data(tbname,col_name,col_type,up_tinyint,dbname)
elif col_type.lower() == 'smallint':
self.update_and_check_data(tbname,col_name,col_type,up_smallint,dbname)
elif col_type.lower() == 'int':
self.update_and_check_data(tbname,col_name,col_type,up_int,dbname)
elif col_type.lower() == 'bigint':
self.update_and_check_data(tbname,col_name,col_type,up_bigint,dbname)
elif col_type.lower() == 'tinyint unsigned':
self.update_and_check_data(tbname,col_name,col_type,up_untinyint,dbname)
elif col_type.lower() == 'smallint unsigned':
self.update_and_check_data(tbname,col_name,col_type,up_unsmallint,dbname)
elif col_type.lower() == 'int unsigned':
self.update_and_check_data(tbname,col_name,col_type,up_unint,dbname)
elif col_type.lower() == 'bigint unsigned':
self.update_and_check_data(tbname,col_name,col_type,up_unbigint,dbname)
elif col_type.lower() == 'bool':
self.update_and_check_data(tbname,col_name,col_type,up_bool,dbname)
elif col_type.lower() == 'float':
self.update_and_check_data(tbname,col_name,col_type,up_float,dbname)
elif col_type.lower() == 'double':
self.update_and_check_data(tbname,col_name,col_type,up_double,dbname)
elif 'binary' in col_type.lower():
self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname)
elif 'nchar' in col_type.lower():
self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname)
elif col_type.lower() == 'timestamp':
self.update_and_check_data(tbname,col_name,col_type,self.ts+1,dbname)
tdSql.execute(f'insert into {tbname} values({self.ts},null)')
tdSql.query(f'select {col_name} from {tbname}')
tdSql.checkEqual(tdSql.queryResult[0][0],None)
tdSql.execute(f'flush database {self.dbname}')
tdSql.execute('reset query cache')
tdSql.query(f'select {col_name} from {tbname}')
tdSql.checkEqual(tdSql.queryResult[0][0],None)
tdSql.execute(f'drop table {tbname}')
if tb_type == 'ctb':
tdSql.execute(f'drop table {stbname}')
def update_check(self):
tdSql.execute(f'drop database if exists {self.dbname}')
tdSql.execute(f'create database {self.dbname}')
tdSql.execute(f'use {self.dbname}')
self.update_data_check(self.ntbname,self.column_dict,self.dbname,'ntb')
for col_name,col_type in self.column_dict.items():
tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})')
tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)')
if 'binary' in col_type.lower():
up_binary = tdCom.getLongName(self.str_length+1)
tdSql.execute(f'alter table {self.ntbname} modify column {col_name} binary({self.str_length+1})')
self.update_and_check_data(self.ntbname,col_name,col_type,up_binary,self.dbname)
elif 'nchar' in col_type.lower():
up_nchar = tdCom.getLongName(self.str_length+1)
tdSql.execute(f'alter table {self.ntbname} modify column {col_name} nchar({self.str_length+1})')
self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname)
tdSql.execute(f'drop table {self.ntbname}')
self.update_data_check(self.ctbname,self.column_dict,self.dbname,'ctb',self.stbname)
for col_name,col_type in self.column_dict.items():
tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)')
tdSql.execute(f'create table {self.ctbname} using {self.stbname} tags(1)')
tdSql.execute(f'insert into {self.ctbname} values({self.ts},null)')
if 'binary' in col_type.lower():
up_binary = tdCom.getLongName(self.str_length+1)
tdSql.execute(f'alter table {self.stbname} modify column {col_name} binary({self.str_length+1})')
self.update_and_check_data(self.ctbname,col_name,col_type,up_binary,self.dbname)
elif 'nchar' in col_type.lower():
up_nchar = tdCom.getLongName(self.str_length+1)
tdSql.execute(f'alter table {self.stbname} modify column {col_name} nchar({self.str_length+1})')
self.update_and_check_data(self.ctbname,col_name,col_type,up_nchar,self.dbname)
tdSql.execute(f'drop table {self.stbname}')
def update_check_error(self):
tdSql.execute(f'drop database if exists {self.dbname}')
tdSql.execute(f'create database {self.dbname}')
tdSql.execute(f'use {self.dbname}')
self.error_check(self.ntbname,self.column_dict,'ntb')
self.error_check(self.ctbname,self.column_dict,'ctb',self.stbname)
def run(self):
self.update_check()
self.update_check_error()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

3
tests/system-test/2-query/abs.py
View File

@ -555,6 +555,9 @@ class TDTestCase:
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
tdSql.query("select abs(c1) from (select ts , c1 ,t1 from stb1)")
tdSql.checkRows(25)
tdSql.query( tdSql.query(
"select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1")

4
tests/system-test/2-query/csum.py
View File

@ -435,8 +435,8 @@ class TDTestCase:
tdSql.checkRows(40) tdSql.checkRows(40)
# # bug need fix # # bug need fix
# tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ") tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ")
# tdSql.checkRows(4) tdSql.checkRows(4)
# tdSql.error("select csum(st1+c1) from stb1 partition by tbname limit 1 ") # tdSql.error("select csum(st1+c1) from stb1 partition by tbname limit 1 ")

14
tests/system-test/2-query/last_row.py
View File

@ -221,21 +221,21 @@ class TDTestCase:
tdSql.execute("use testdb") tdSql.execute("use testdb")
# bug need fix # bug need fix
tdSql.error("select last_row(c1 ,NULL) from testdb.t1") tdSql.query("select last_row(c1 ,NULL) from testdb.t1")
error_sql_lists = [ error_sql_lists = [
"select last_row from testdb.t1", "select last_row from testdb.t1",
"select last_row(-+--+c1) from testdb.t1", "select last_row(-+--+c1) from testdb.t1",
"select last_row(123--123)==1 from testdb.t1", "select last_row(123--123)==1 from testdb.t1",
"select last_row(c1) as 'd1' from testdb.t1", "select last_row(c1) as 'd1' from testdb.t1",
"select last_row(c1 ,NULL) from testdb.t1", #"select last_row(c1 ,NULL) from testdb.t1",
"select last_row(,) from testdb.t1;", "select last_row(,) from testdb.t1;",
"select last_row(abs(c1) ab from testdb.t1)", "select last_row(abs(c1) ab from testdb.t1)",
"select last_row(c1) as int from testdb.t1", "select last_row(c1) as int from testdb.t1",
"select last_row from testdb.stb1", "select last_row from testdb.stb1",
"select last_row(123--123)==1 from testdb.stb1", "select last_row(123--123)==1 from testdb.stb1",
"select last_row(c1) as 'd1' from testdb.stb1", "select last_row(c1) as 'd1' from testdb.stb1",
"select last_row(c1 ,NULL) from testdb.stb1", #"select last_row(c1 ,NULL) from testdb.stb1",
"select last_row(,) from testdb.stb1;", "select last_row(,) from testdb.stb1;",
"select last_row(abs(c1) ab from testdb.stb1)", "select last_row(abs(c1) ab from testdb.stb1)",
"select last_row(c1) as int from testdb.stb1" "select last_row(c1) as int from testdb.stb1"
@ -722,10 +722,18 @@ class TDTestCase:
tdSql.query("select last_row(ceil(c1-2)) , abs(floor(t1+1)) ,floor(c2-c1) from testdb.stb1 partition by abs(floor(c1)) order by abs(c1)") tdSql.query("select last_row(ceil(c1-2)) , abs(floor(t1+1)) ,floor(c2-c1) from testdb.stb1 partition by abs(floor(c1)) order by abs(c1)")
tdSql.checkRows(11) tdSql.checkRows(11)
tdSql.query("select max(c1) from stb1 interval(50s) sliding(30s)")
tdSql.checkRows(13)
tdSql.query("select unique(c1) from stb1 partition by tbname")
# interval # interval
tdSql.query("select last_row(c1) from testdb.stb1 interval(50s) sliding(30s)") tdSql.query("select last_row(c1) from testdb.stb1 interval(50s) sliding(30s)")
tdSql.checkRows(27) tdSql.checkRows(27)
tdSql.query("select last_row(c1) from testdb.ct1 interval(50s) sliding(30s)") tdSql.query("select last_row(c1) from testdb.ct1 interval(50s) sliding(30s)")
tdSql.checkRows(5) tdSql.checkRows(5)
last_row_result = tdSql.queryResult last_row_result = tdSql.queryResult

35
tests/system-test/2-query/max_partition.py
View File

@ -162,10 +162,45 @@ class TDTestCase:
tdSql.query("select tbname , max(c1) from stb partition by tbname interval(10s)") tdSql.query("select tbname , max(c1) from stb partition by tbname interval(10s)")
tdSql.checkRows(self.row_nums*2) tdSql.checkRows(self.row_nums*2)
tdSql.query("select unique(c1) from stb partition by tbname order by tbname")
tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)") tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)")
tdSql.checkData(0,0,'sub_stb_1') tdSql.checkData(0,0,'sub_stb_1')
tdSql.checkData(0,1,self.row_nums) tdSql.checkData(0,1,self.row_nums)
tdSql.query("select c1 , mavg(c1 ,2 ) from stb partition by c1")
tdSql.checkRows(72)
tdSql.query("select c1 , diff(c1 , 0) from stb partition by c1")
tdSql.checkRows(72)
tdSql.query("select c1 , csum(c1) from stb partition by c1")
tdSql.checkRows(80)
tdSql.query("select c1 , sample(c1,2) from stb partition by c1 order by c1")
tdSql.checkRows(21)
# bug need fix
# tdSql.checkData(0,1,None)
tdSql.query("select c1 , twa(c1) from stb partition by c1 order by c1")
tdSql.checkRows(11)
tdSql.checkData(0,1,0.000000000)
tdSql.query("select c1 , irate(c1) from stb partition by c1 order by c1")
tdSql.checkRows(11)
tdSql.checkData(0,1,None)
tdSql.query("select c1 , DERIVATIVE(c1,2,1) from stb partition by c1 order by c1")
tdSql.checkRows(72)
# bug need fix
# tdSql.checkData(0,1,None)
# bug need fix # bug need fix
# tdSql.query(" select tbname , max(c1) from stb partition by tbname order by tbname slimit 5 soffset 0 ") # tdSql.query(" select tbname , max(c1) from stb partition by tbname order by tbname slimit 5 soffset 0 ")
# tdSql.checkRows(5) # tdSql.checkRows(5)

5
tests/system-test/2-query/sample.py
View File

@ -870,7 +870,10 @@ class TDTestCase:
tdSql.query("select sample(c1 ,1000) from st") tdSql.query("select sample(c1 ,1000) from st")
tdSql.checkRows(1000) tdSql.checkRows(1000)
# bug need fix
tdSql.query("select c1 ,t1, sample(c1,2) from db.stb1 partition by c1 ")
tdSql.query("select sample(c1,2) from db.stb1 partition by c1 ")
# tdSql.query("select c1 ,ind, sample(c1,2) from sample_db.st partition by c1 ")
def run(self): def run(self):
import traceback import traceback

22
tests/system-test/7-tmq/stbTagFilter.py
View File

@ -97,18 +97,18 @@ class TDTestCase:
paraDict['rowsPerTbl'] = self.rowsPerTbl paraDict['rowsPerTbl'] = self.rowsPerTbl
# update to half tables # update to half tables
paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) # paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2)
# tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx",
# ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
# startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
tdLog.info("create topics from stb1") tdLog.info("create topics from stb1")
topicFromStb1 = 'topic_stb1' topicFromStb1 = 'topic_stb1'
# queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as %s" %(topicFromStb1, queryString) sqlString = "create topic %s as %s" %(topicFromStb1, queryString)
tdLog.info("create topic sql: %s"%sqlString) tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString) tdSql.execute(sqlString)
@ -242,11 +242,11 @@ class TDTestCase:
self.tmqCase1() self.tmqCase1()
# self.tmqCase2() # self.tmqCase2()
self.prepareTestEnv() # self.prepareTestEnv()
tdLog.printNoPrefix("====================================================================") # tdLog.printNoPrefix("====================================================================")
tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") # tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal")
self.snapshot = 1 # self.snapshot = 1
self.tmqCase1() # self.tmqCase1()
# self.tmqCase2() # self.tmqCase2()

7
tests/system-test/7-tmq/tmqCommon.py
View File

@ -205,6 +205,13 @@ class TMQCom:
tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName))
return return
def drop_ctable(self, tsql, dbname=None, count=1, default_ctbname_prefix="ctb",ctbStartIdx=0):
for _ in range(count):
create_ctable_sql = f'drop table {dbname}.{default_ctbname_prefix}{ctbStartIdx};'
ctbStartIdx += 1
tdLog.info("drop ctb sql: %s"%create_ctable_sql)
tsql.execute(create_ctable_sql)
# schema: (ts timestamp, c1 int, c2 binary(16)) # schema: (ts timestamp, c1 int, c2 binary(16))
def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None): def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None):
tdLog.debug("start to insert data ............") tdLog.debug("start to insert data ............")

13
tests/system-test/7-tmq/tmqUpdate-1ctb.py
View File

@ -116,7 +116,12 @@ class TDTestCase:
# paraDict['ctbNum'] = self.ctbNum # paraDict['ctbNum'] = self.ctbNum
paraDict['rowsPerTbl'] = self.rowsPerTbl paraDict['rowsPerTbl'] = self.rowsPerTbl
consumerId = 0 consumerId = 0
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 3/2)
if self.snapshot == 0:
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/2))
elif self.snapshot == 1:
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1))
topicList = topicFromStb1 topicList = topicFromStb1
ifcheckdata = 1 ifcheckdata = 1
ifManualCommit = 1 ifManualCommit = 1
@ -199,7 +204,11 @@ class TDTestCase:
# paraDict['ctbNum'] = self.ctbNum # paraDict['ctbNum'] = self.ctbNum
paraDict['rowsPerTbl'] = self.rowsPerTbl paraDict['rowsPerTbl'] = self.rowsPerTbl
consumerId = 1 consumerId = 1
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) if self.snapshot == 0:
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (2))
elif self.snapshot == 1:
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1))
topicList = topicFromStb1 topicList = topicFromStb1
ifcheckdata = 1 ifcheckdata = 1
ifManualCommit = 1 ifManualCommit = 1

2
tests/system-test/99-TDcase/TD-17255.py
View File

@ -320,7 +320,7 @@ class TDTestCase:
tdSql.prepare() tdSql.prepare()
self.tmqCase1() self.tmqCase1()
# self.tmqCase2() self.tmqCase2()
self.tmqCase3() self.tmqCase3()
def stop(self): def stop(self):

7
tests/system-test/fulltest.sh
View File

@ -10,7 +10,7 @@ python3 ./test.py -f 0-others/taosdMonitor.py
python3 ./test.py -f 0-others/udfTest.py python3 ./test.py -f 0-others/udfTest.py
python3 ./test.py -f 0-others/udf_create.py python3 ./test.py -f 0-others/udf_create.py
python3 ./test.py -f 0-others/udf_restart_taosd.py python3 ./test.py -f 0-others/udf_restart_taosd.py
python3 ./test.py -f 0-others/cachelast.py python3 ./test.py -f 0-others/cachemodel.py
python3 ./test.py -f 0-others/udf_cfg1.py python3 ./test.py -f 0-others/udf_cfg1.py
python3 ./test.py -f 0-others/udf_cfg2.py python3 ./test.py -f 0-others/udf_cfg2.py
@ -32,7 +32,10 @@ python3 ./test.py -f 1-insert/block_wise.py
python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/create_retentions.py
python3 ./test.py -f 1-insert/table_param_ttl.py python3 ./test.py -f 1-insert/table_param_ttl.py
python3 ./test.py -f 1-insert/update_data.py
python3 ./test.py -f 2-query/db.py python3 ./test.py -f 2-query/db.py
python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/between.py
python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/distinct.py
python3 ./test.py -f 2-query/varchar.py python3 ./test.py -f 2-query/varchar.py
@ -182,7 +185,7 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
#python3 ./test.py -f 7-tmq/tmqDnodeRestart.py #python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
#python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py
python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb.py python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb.py
#python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py #python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py

10
tests/test/c/tmqSim.c
View File

@ -587,9 +587,10 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn
tmq_get_topic_name(msg), vgroupId); tmq_get_topic_name(msg), vgroupId);
{ {
tmq_raw_data *raw = tmq_get_raw_meta(msg); tmq_raw_data raw = {0};
int32_t code = tmq_get_raw_meta(msg, &raw);
if(raw){ if(code == TSDB_CODE_SUCCESS){
TAOS_RES* pRes = taos_query(pInfo->taos, "use metadb"); TAOS_RES* pRes = taos_query(pInfo->taos, "use metadb");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
pError("error when use metadb, reason:%s\n", taos_errstr(pRes)); pError("error when use metadb, reason:%s\n", taos_errstr(pRes));
@ -599,10 +600,9 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn
exit(-1); exit(-1);
} }
taos_free_result(pRes); taos_free_result(pRes);
taosFprintfFile(g_fp, "raw:%p\n", raw); taosFprintfFile(g_fp, "raw:%p\n", &raw);
int32_t ret = taos_write_raw_meta(pInfo->taos, raw); taos_write_raw_meta(pInfo->taos, raw);
taosMemoryFree(raw);
} }
char* result = tmq_get_json_meta(msg); char* result = tmq_get_json_meta(msg);

5
tests/tsim/src/simParse.c
View File

@ -175,7 +175,7 @@ SScript *simBuildScriptObj(char *fileName) {
SScript *simParseScript(char *fileName) { SScript *simParseScript(char *fileName) {
TdFilePtr pFile; TdFilePtr pFile;
int32_t tokenLen, lineNum = 0; int32_t tokenLen, lineNum = 0;
char *buffer = NULL, name[128], *token, *rest; char buffer[10*1024], name[128], *token, *rest;
SCommand *pCmd; SCommand *pCmd;
SScript *script; SScript *script;
@ -195,7 +195,7 @@ SScript *simParseScript(char *fileName) {
simResetParser(); simResetParser();
while (!taosEOFFile(pFile)) { while (!taosEOFFile(pFile)) {
if (taosGetLineFile(pFile, &buffer) == -1) continue; if (taosGetsFile(pFile, sizeof(buffer) - 1, buffer) == -1) continue;
lineNum++; lineNum++;
int32_t cmdlen = (int32_t)strlen(buffer); int32_t cmdlen = (int32_t)strlen(buffer);
@ -240,7 +240,6 @@ SScript *simParseScript(char *fileName) {
return NULL; return NULL;
} }
} }
if(buffer != NULL) taosMemoryFree(buffer);
taosCloseFile(&pFile); taosCloseFile(&pFile);
script = simBuildScriptObj(fileName); script = simBuildScriptObj(fileName);

26
tools/shell/src/shellEngine.c
View File

@ -737,13 +737,6 @@ int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool ver
void shellReadHistory() { void shellReadHistory() {
SShellHistory *pHistory = &shell.history; SShellHistory *pHistory = &shell.history;
int64_t file_size;
if (taosStatFile(pHistory->file, &file_size, NULL) != 0) {
return;
} else if (file_size > SHELL_MAX_COMMAND_SIZE) {
taosRemoveFile(pHistory->file);
return;
}
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM); TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) return; if (pFile == NULL) return;
@ -763,10 +756,29 @@ void shellReadHistory() {
if (line != NULL) taosMemoryFree(line); if (line != NULL) taosMemoryFree(line);
taosCloseFile(&pFile); taosCloseFile(&pFile);
int64_t file_size;
if (taosStatFile(pHistory->file, &file_size, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) {
fprintf(stdout,"%s(%d) %s %08" PRId64 "\n", __FILE__, __LINE__,__func__,taosGetSelfPthreadId());fflush(stdout);
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC);
if (pFile == NULL) return;
int32_t endIndex = pHistory->hstart;
if (endIndex != 0) {
endIndex = pHistory->hend;
}
for (int32_t i = (pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; i != endIndex;) {
taosFprintfFile(pFile, "%s\n", pHistory->hist[i]);
i = (i + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE;
}
taosFprintfFile(pFile, "%s\n", pHistory->hist[endIndex]);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
}
pHistory->hend = pHistory->hstart;
} }
void shellWriteHistory() { void shellWriteHistory() {
SShellHistory *pHistory = &shell.history; SShellHistory *pHistory = &shell.history;
if (pHistory->hend == pHistory->hstart) return;
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND); TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND);
if (pFile == NULL) return; if (pFile == NULL) return;