xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/tsdb_optimize

This commit is contained in:
Hongze Cheng 2023-05-24 09:31:20 +08:00
parent 0c09d6f0d2 693bac60a0
commit 166a86db69
9 changed files with 59 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/taosadapter_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taosadapter # taosadapter
ExternalProject_Add(taosadapter ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
GIT_TAG 565ca21 GIT_TAG 283b50d
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

10
packaging/docker/dockerbuild.sh
View File

@ -123,6 +123,16 @@ else
echo "Unknown cpuType: ${cpuType}" echo "Unknown cpuType: ${cpuType}"
exit 1 exit 1
fi fi
# check the tdengine cloud base image existed or not
if [ "$cloudBuild" == "y" ]; then
CloudBase=$(docker images | grep tdengine/tdengine-cloud-base ||:)
if [[ "$CloudBase" == "" ]]; then
echo "Rebuild tdengine cloud base image..."
docker build --rm -f "${communityDir}/packaging/docker/DockerfileCloud.base" -t tdengine/tdengine-cloud-base "." --build-arg cpuType=${cpuTypeAlias}
else
echo "Already found tdengine cloud base image"
fi
fi
docker build --rm -f "${Dockerfile}" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} --build-arg cpuType=${cpuTypeAlias} docker build --rm -f "${Dockerfile}" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} --build-arg cpuType=${cpuTypeAlias}
if [ "$cloudBuild" != "y" ]; then if [ "$cloudBuild" != "y" ]; then

4
packaging/tools/makeclient.sh
View File

@ -175,7 +175,7 @@ if [ "$pagMode" == "lite" ]; then
fi fi
chmod a+x ${install_dir}/install_client.sh chmod a+x ${install_dir}/install_client.sh
if [[ $productName == "TDengine" ]]; then if [[ $productName == "TDengine" ]] && [ "$verMode" != "cloud" ]; then
# Copy example code # Copy example code
mkdir -p ${install_dir}/examples mkdir -p ${install_dir}/examples
examples_dir="${top_dir}/examples" examples_dir="${top_dir}/examples"
@ -191,7 +191,7 @@ if [[ $productName == "TDengine" ]]; then
mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json
fi fi
if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then if [ "$verMode" == "cluster" ]; then
# Copy connector # Copy connector
connector_dir="${code_dir}/connector" connector_dir="${code_dir}/connector"
mkdir -p ${install_dir}/connector mkdir -p ${install_dir}/connector

31
packaging/tools/makepkg.sh
View File

@ -69,25 +69,30 @@ if [ "$pagMode" == "lite" ]; then
bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark " bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark "
taostools_bin_files="" taostools_bin_files=""
else else
if [ "$verMode" == "cloud" ]; then
wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ taostools_bin_files=" ${build_dir}/bin/taosdump \
${build_dir}/bin/taosBenchmark"
else
wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \
&& echo "TDinsight.sh downloaded!" \ && echo "TDinsight.sh downloaded!" \
|| echo "failed to download TDinsight.sh" || echo "failed to download TDinsight.sh"
# download TDinsight caches # download TDinsight caches
orig_pwd=$(pwd) orig_pwd=$(pwd)
tdinsight_caches="" tdinsight_caches=""
cd ${build_dir}/bin/ && \ cd ${build_dir}/bin/ && \
chmod +x TDinsight.sh chmod +x TDinsight.sh
./TDinsight.sh --download-only ||: ./TDinsight.sh --download-only ||:
# tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ") # tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ")
cd $orig_pwd cd $orig_pwd
echo "TDinsight caches: $tdinsight_caches" echo "TDinsight caches: $tdinsight_caches"
taostools_bin_files=" ${build_dir}/bin/taosdump \ taostools_bin_files=" ${build_dir}/bin/taosdump \
${build_dir}/bin/taosBenchmark \ ${build_dir}/bin/taosBenchmark \
${build_dir}/bin/TDinsight.sh \ ${build_dir}/bin/TDinsight.sh \
${build_dir}/bin/tdengine-datasource.zip \ ${build_dir}/bin/tdengine-datasource.zip \
${build_dir}/bin/tdengine-datasource.zip.md5sum" ${build_dir}/bin/tdengine-datasource.zip.md5sum"
fi
[ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx" [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx"
explorer_bin_files=$(find ${build_dir}/bin/ -name '*-explorer') explorer_bin_files=$(find ${build_dir}/bin/ -name '*-explorer')
@ -334,7 +339,7 @@ mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo
[ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || :
# Copy connector # Copy connector
if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then if [ "$verMode" == "cluster" ]; then
connector_dir="${code_dir}/connector" connector_dir="${code_dir}/connector"
mkdir -p ${install_dir}/connector mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then

33
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -211,10 +211,18 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) {
taosMemoryFree(pTsdb->rCache.pTSchema); taosMemoryFree(pTsdb->rCache.pTSchema);
} }
static void rocksMayWrite(STsdb *pTsdb, bool force, bool read) { static void rocksMayWrite(STsdb *pTsdb, bool force, bool read, bool lock) {
rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; rocksdb_writebatch_t *wb = NULL;
if (read) { if (read) {
if (lock) {
taosThreadMutexLock(&pTsdb->lruMutex);
}
wb = pTsdb->rCache.rwritebatch; wb = pTsdb->rCache.rwritebatch;
} else {
if (lock) {
taosThreadMutexLock(&pTsdb->rCache.rMutex);
}
wb = pTsdb->rCache.writebatch;
} }
int count = rocksdb_writebatch_count(wb); int count = rocksdb_writebatch_count(wb);
if ((force && count > 0) || count >= 1024) { if ((force && count > 0) || count >= 1024) {
@ -228,14 +236,19 @@ static void rocksMayWrite(STsdb *pTsdb, bool force, bool read) {
rocksdb_writebatch_clear(wb); rocksdb_writebatch_clear(wb);
} }
if (read) {
if (lock) taosThreadMutexUnlock(&pTsdb->lruMutex);
} else {
if (lock) taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
}
} }
int32_t tsdbCacheCommit(STsdb *pTsdb) { int32_t tsdbCacheCommit(STsdb *pTsdb) {
int32_t code = 0; int32_t code = 0;
char *err = NULL; char *err = NULL;
rocksMayWrite(pTsdb, true, false); rocksMayWrite(pTsdb, true, false, true);
rocksMayWrite(pTsdb, true, true); rocksMayWrite(pTsdb, true, true, true);
rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err);
if (NULL != err) { if (NULL != err) {
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err);
@ -461,7 +474,7 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
taosMemoryFree(values_list); taosMemoryFree(values_list);
taosMemoryFree(values_list_sizes); taosMemoryFree(values_list_sizes);
rocksMayWrite(pTsdb, false, false); rocksMayWrite(pTsdb, false, false, false);
taosThreadMutexUnlock(&pTsdb->rCache.rMutex); taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
_exit: _exit:
@ -560,7 +573,7 @@ int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheR
} }
if (wb) { if (wb) {
rocksMayWrite(pTsdb, false, true); rocksMayWrite(pTsdb, false, true, false);
} }
taosThreadMutexUnlock(&pTsdb->rCache.rMutex); taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
@ -628,7 +641,7 @@ static SLastCol *tsdbCacheLoadCol(STsdb *pTsdb, SCacheRowsReader *pr, int16_t sl
} }
if (wb) { if (wb) {
rocksMayWrite(pTsdb, false, true); rocksMayWrite(pTsdb, false, true, false);
} }
taosThreadMutexUnlock(&pTsdb->rCache.rMutex); taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
@ -714,7 +727,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
} }
if (wb) { if (wb) {
rocksMayWrite(pTsdb, false, true); rocksMayWrite(pTsdb, false, true, false);
} }
taosArrayDestroy(pTmpColArray); taosArrayDestroy(pTmpColArray);
@ -944,7 +957,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
size_t *values_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); size_t *values_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t));
char **errs = taosMemoryCalloc(num_keys * 2, sizeof(char *)); char **errs = taosMemoryCalloc(num_keys * 2, sizeof(char *));
taosThreadMutexLock(&pTsdb->rCache.rMutex); taosThreadMutexLock(&pTsdb->rCache.rMutex);
rocksMayWrite(pTsdb, true, false); rocksMayWrite(pTsdb, true, false, false);
rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list, rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list,
keys_list_sizes, values_list, values_list_sizes, errs); keys_list_sizes, values_list, values_list_sizes, errs);
for (int i = 0; i < num_keys; ++i) { for (int i = 0; i < num_keys; ++i) {
@ -983,7 +996,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
taosMemoryFree(values_list); taosMemoryFree(values_list);
taosMemoryFree(values_list_sizes); taosMemoryFree(values_list_sizes);
rocksMayWrite(pTsdb, true, false); rocksMayWrite(pTsdb, true, false, false);
taosThreadMutexUnlock(&pTsdb->rCache.rMutex); taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
_exit: _exit:

3
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -63,9 +63,6 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
SColVal* pVal = &pColVal->colVal; SColVal* pVal = &pColVal->colVal;
if (COL_VAL_IS_NONE(&pColVal->colVal)) {
continue;
}
allNullRow = false; allNullRow = false;
if (IS_VAR_DATA_TYPE(pColVal->colVal.type)) { if (IS_VAR_DATA_TYPE(pColVal->colVal.type)) {
if (!COL_VAL_IS_VALUE(&pColVal->colVal)) { if (!COL_VAL_IS_VALUE(&pColVal->colVal)) {

6
source/libs/catalog/CMakeLists.txt
View File

@ -11,6 +11,6 @@ target_link_libraries(
PRIVATE os util transport qcom nodes PRIVATE os util transport qcom nodes
) )
if(${BUILD_TEST}) # if(${BUILD_TEST})
ADD_SUBDIRECTORY(test) # ADD_SUBDIRECTORY(test)
endif(${BUILD_TEST}) # endif(${BUILD_TEST})

2
source/libs/planner/src/planOptimizer.c
View File

@ -2233,7 +2233,7 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
SNode* pPar = nodesListGetNode(pAggFunc->pParameterList, 0); SNode* pPar = nodesListGetNode(pAggFunc->pParameterList, 0);
if (QUERY_NODE_COLUMN == nodeType(pPar)) { if (QUERY_NODE_COLUMN == nodeType(pPar)) {
SColumnNode* pCol = (SColumnNode*)pPar; SColumnNode* pCol = (SColumnNode*)pPar;
if (pCol->colType == COLUMN_TYPE_TAG) { if (pCol->colType != COLUMN_TYPE_COLUMN) {
return false; return false;
} }
} }

2
tests/parallel_test/cases.task
View File

@ -513,7 +513,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py #,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart.py