xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

add test

This commit is contained in:
yihaoDeng 2024-03-21 11:06:46 +00:00
parent 672d189d6f
commit 2903d568ea
5 changed files with 90 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cmake/zstd_CMakeLists.txt.in
View File

@ -1,14 +1,14 @@
# zstb # zstb
ExternalProject_Add(zstd #ExternalProject_Add(zstd
GIT_REPOSITORY https://github.com/facebook/zstd.git #GIT_REPOSITORY https://github.com/facebook/zstd.git
GIT_TAG v1.5.5 #GIT_TAG v1.5.5
SOURCE_DIR "${TD_CONTRIB_DIR}/zstd" #SOURCE_DIR "${TD_CONTRIB_DIR}/zstd"
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" #DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
BINARY_DIR "" #BINARY_DIR ""
CMAKE_COMMAND #CMAKE_COMMAND
CMAKE_ARGS ${TD_CONTRIB_DIR}/zstd/build/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${CMAKE_BINARY_DIR}/build/lib #CMAKE_ARGS ${TD_CONTRIB_DIR}/zstd/build/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${CMAKE_BINARY_DIR}/build/lib
BUILD_COMMAND make -j4 #BUILD_COMMAND make -j4
INSTALL_COMMAND "" #INSTALL_COMMAND ""
TEST_COMMAND "" #TEST_COMMAND ""
) #)

3
contrib/CMakeLists.txt
View File

@ -129,7 +129,8 @@ else()
endif() endif()
endif() endif()
cat("${TD_SUPPORT_DIR}/zstd_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/zstd_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
#libuv #libuv
if(${BUILD_WITH_UV}) if(${BUILD_WITH_UV})
cat("${TD_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})

2
source/dnode/mgmt/CMakeLists.txt
View File

@ -25,6 +25,6 @@ IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
ADD_DEPENDENCIES(taosd jemalloc) ADD_DEPENDENCIES(taosd jemalloc)
target_link_libraries(taosd dnode ${LINK_JEMALLOC}) target_link_libraries(taosd dnode ${LINK_JEMALLOC})
ELSE () ELSE ()
target_link_libraries(taosd dnode zstd) target_link_libraries(taosd dnode)
ENDIF () ENDIF ()

2
source/dnode/vnode/CMakeLists.txt
View File

@ -165,7 +165,7 @@ target_link_libraries(
# PUBLIC bdb # PUBLIC bdb
# PUBLIC scalar # PUBLIC scalar
PUBLIC zstd #PUBLIC zstd
PUBLIC rocksdb PUBLIC rocksdb
PUBLIC transport PUBLIC transport
PUBLIC stream PUBLIC stream

99
source/util/test/decompressTest.cpp
View File

@ -9,47 +9,50 @@ TEST(utilTest, decompress_test) {
int64_t tsList[10] = {1700000000, 1700000100, 1700000200, 1700000300, 1700000400, int64_t tsList[10] = {1700000000, 1700000100, 1700000200, 1700000300, 1700000400,
1700000500, 1700000600, 1700000700, 1700000800, 1700000900}; 1700000500, 1700000600, 1700000700, 1700000800, 1700000900};
char* pOutput[10 * sizeof(int64_t)] = {0}; char* pOutput[10 * sizeof(int64_t)] = {0};
int32_t len = tsCompressTimestamp(tsList, sizeof(tsList), sizeof(tsList) / sizeof(tsList[0]), pOutput, 10, ONE_STAGE_COMP, NULL, 0); int32_t len = tsCompressTimestamp(tsList, sizeof(tsList), sizeof(tsList) / sizeof(tsList[0]), pOutput, 10,
ONE_STAGE_COMP, NULL, 0);
char* decompOutput[10 * 8] = {0}; char* decompOutput[10 * 8] = {0};
tsDecompressTimestamp(pOutput, len, 10, decompOutput, sizeof(int64_t)*10, ONE_STAGE_COMP, NULL, 0); tsDecompressTimestamp(pOutput, len, 10, decompOutput, sizeof(int64_t) * 10, ONE_STAGE_COMP, NULL, 0);
for(int32_t i = 0; i < 10; ++i) { for (int32_t i = 0; i < 10; ++i) {
std::cout<< ((int64_t*)decompOutput)[i] << std::endl; std::cout << ((int64_t*)decompOutput)[i] << std::endl;
} }
memset(decompOutput, 0, 10*8); memset(decompOutput, 0, 10 * 8);
tsDecompressTimestampAvx512(reinterpret_cast<const char* const>(pOutput), 10, tsDecompressTimestampAvx512(reinterpret_cast<const char* const>(pOutput), 10,
reinterpret_cast<char* const>(decompOutput), false); reinterpret_cast<char* const>(decompOutput), false);
for(int32_t i = 0; i < 10; ++i) { for (int32_t i = 0; i < 10; ++i) {
std::cout<<((int64_t*)decompOutput)[i] << std::endl; std::cout << ((int64_t*)decompOutput)[i] << std::endl;
} }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
int64_t tsList1[7] = {1700000000, 1700000000, 1700000000, 1700000000, 1700000000, 1700000000, 1700000900}; int64_t tsList1[7] = {1700000000, 1700000000, 1700000000, 1700000000, 1700000000, 1700000000, 1700000900};
int32_t len1 = tsCompressTimestamp(tsList1, sizeof(tsList1), sizeof(tsList1) / sizeof(tsList1[0]), pOutput, 7, ONE_STAGE_COMP, NULL, 0); int32_t len1 = tsCompressTimestamp(tsList1, sizeof(tsList1), sizeof(tsList1) / sizeof(tsList1[0]), pOutput, 7,
ONE_STAGE_COMP, NULL, 0);
memset(decompOutput, 0, 10*8); memset(decompOutput, 0, 10 * 8);
tsDecompressTimestampAvx512(reinterpret_cast<const char* const>(pOutput), 7, tsDecompressTimestampAvx512(reinterpret_cast<const char* const>(pOutput), 7,
reinterpret_cast<char* const>(decompOutput), false); reinterpret_cast<char* const>(decompOutput), false);
for(int32_t i = 0; i < 7; ++i) { for (int32_t i = 0; i < 7; ++i) {
std::cout<<((int64_t*)decompOutput)[i] << std::endl; std::cout << ((int64_t*)decompOutput)[i] << std::endl;
} }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
int64_t tsList2[1] = {1700000000}; int64_t tsList2[1] = {1700000000};
int32_t len2 = tsCompressTimestamp(tsList2, sizeof(tsList2), sizeof(tsList2) / sizeof(tsList2[0]), pOutput, 1, ONE_STAGE_COMP, NULL, 0); int32_t len2 = tsCompressTimestamp(tsList2, sizeof(tsList2), sizeof(tsList2) / sizeof(tsList2[0]), pOutput, 1,
ONE_STAGE_COMP, NULL, 0);
memset(decompOutput, 0, 10*8); memset(decompOutput, 0, 10 * 8);
tsDecompressTimestampAvx512(reinterpret_cast<const char* const>(pOutput), 1, tsDecompressTimestampAvx512(reinterpret_cast<const char* const>(pOutput), 1,
reinterpret_cast<char* const>(decompOutput), false); reinterpret_cast<char* const>(decompOutput), false);
for(int32_t i = 0; i < 1; ++i) { for (int32_t i = 0; i < 1; ++i) {
std::cout<<((int64_t*)decompOutput)[i] << std::endl; std::cout << ((int64_t*)decompOutput)[i] << std::endl;
} }
} }
@ -57,22 +60,22 @@ TEST(utilTest, decompress_perf_test) {
int32_t num = 10000; int32_t num = 10000;
int64_t* pList = static_cast<int64_t*>(taosMemoryCalloc(num, sizeof(int64_t))); int64_t* pList = static_cast<int64_t*>(taosMemoryCalloc(num, sizeof(int64_t)));
int64_t iniVal = 1700000000; int64_t iniVal = 1700000000;
uint32_t v = 100; uint32_t v = 100;
for(int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
iniVal += taosRandR(&v)%10; iniVal += taosRandR(&v) % 10;
pList[i] = iniVal; pList[i] = iniVal;
} }
char* px = static_cast<char*>(taosMemoryMalloc(num * sizeof(int64_t))); char* px = static_cast<char*>(taosMemoryMalloc(num * sizeof(int64_t)));
int32_t len = tsCompressTimestamp(pList, num * sizeof(int64_t), num, px, num, ONE_STAGE_COMP, NULL, 0); int32_t len = tsCompressTimestamp(pList, num * sizeof(int64_t), num, px, num, ONE_STAGE_COMP, NULL, 0);
char* pOutput = static_cast<char*>(taosMemoryMalloc(num * sizeof(int64_t))); char* pOutput = static_cast<char*>(taosMemoryMalloc(num * sizeof(int64_t)));
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
for(int32_t k = 0; k < 10000; ++k) { for (int32_t k = 0; k < 10000; ++k) {
tsDecompressTimestamp(px, len, num, pOutput, sizeof(int64_t) * num, ONE_STAGE_COMP, NULL, 0); tsDecompressTimestamp(px, len, num, pOutput, sizeof(int64_t) * num, ONE_STAGE_COMP, NULL, 0);
} }
@ -81,7 +84,7 @@ TEST(utilTest, decompress_perf_test) {
memset(pOutput, 0, num * sizeof(int64_t)); memset(pOutput, 0, num * sizeof(int64_t));
st = taosGetTimestampUs(); st = taosGetTimestampUs();
for(int32_t k = 0; k < 10000; ++k) { for (int32_t k = 0; k < 10000; ++k) {
tsDecompressTimestampAvx512(px, num, pOutput, false); tsDecompressTimestampAvx512(px, num, pOutput, false);
} }
@ -93,3 +96,49 @@ TEST(utilTest, decompress_perf_test) {
taosMemoryFree(px); taosMemoryFree(px);
} }
void setColEncode(uint32_t* compress, uint8_t l1) {
*compress &= 0x00FFFFFF;
*compress |= (l1 << 24);
return;
}
void setColCompress(uint32_t* compress, uint16_t l2) {
*compress &= 0xFF0000FF;
*compress |= (l2 << 8);
return;
}
void setColLevel(uint32_t* compress, uint8_t level) {
*compress &= 0xFFFFFF00;
*compress |= level;
return;
}
TEST(utilTest, zstdtest) {
int32_t num = 10000;
int64_t* pList = static_cast<int64_t*>(taosMemoryCalloc(num, sizeof(int64_t)));
int64_t iniVal = 1700000000;
uint32_t v = 100;
for (int32_t i = 0; i < num; ++i) {
iniVal += num;
pList[i] = iniVal;
}
uint32_t cmprAlg = 0;
setColCompress(&cmprAlg, 1);
setColEncode(&cmprAlg, 1);
char* pOutput = static_cast<char*>(taosMemoryMalloc(num * sizeof(int64_t)));
int32_t bufSize = num * sizeof(int64_t) + 64;
char* pBuf = static_cast<char*>(taosMemoryMalloc(bufSize));
int32_t sz =
tsCompressTimestamp2(pList, num * sizeof(int64_t), num, pOutput, num * sizeof(int64_t), cmprAlg, pBuf, bufSize);
printf("compress size: %d", sz);
tsDecompressTimestamp(pOutput, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf)
}