diff --git a/cmake/zstd_CMakeLists.txt.in b/cmake/zstd_CMakeLists.txt.in index 4c9ee5b194..1a5d6bb13e 100644 --- a/cmake/zstd_CMakeLists.txt.in +++ b/cmake/zstd_CMakeLists.txt.in @@ -1,14 +1,14 @@ # zstb -ExternalProject_Add(zstd - GIT_REPOSITORY https://github.com/facebook/zstd.git - GIT_TAG v1.5.5 - SOURCE_DIR "${TD_CONTRIB_DIR}/zstd" - DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" - BINARY_DIR "" - CMAKE_COMMAND - CMAKE_ARGS ${TD_CONTRIB_DIR}/zstd/build/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${CMAKE_BINARY_DIR}/build/lib - BUILD_COMMAND make -j4 - INSTALL_COMMAND "" - TEST_COMMAND "" - ) \ No newline at end of file +#ExternalProject_Add(zstd + #GIT_REPOSITORY https://github.com/facebook/zstd.git + #GIT_TAG v1.5.5 + #SOURCE_DIR "${TD_CONTRIB_DIR}/zstd" + #DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" + #BINARY_DIR "" + #CMAKE_COMMAND + #CMAKE_ARGS ${TD_CONTRIB_DIR}/zstd/build/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${CMAKE_BINARY_DIR}/build/lib + #BUILD_COMMAND make -j4 + #INSTALL_COMMAND "" + #TEST_COMMAND "" + #) \ No newline at end of file diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e98d3d73a9..19900d2363 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -129,7 +129,8 @@ else() endif() endif() -cat("${TD_SUPPORT_DIR}/zstd_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +#cat("${TD_SUPPORT_DIR}/zstd_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #libuv if(${BUILD_WITH_UV}) cat("${TD_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 61bd4067c1..762b8fd529 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -25,6 +25,6 @@ IF (TD_LINUX_64 AND JEMALLOC_ENABLED) ADD_DEPENDENCIES(taosd jemalloc) target_link_libraries(taosd dnode ${LINK_JEMALLOC}) ELSE () - target_link_libraries(taosd dnode zstd) + target_link_libraries(taosd dnode) ENDIF () diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 68c5be13d9..ded4f5dee2 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -165,7 +165,7 @@ target_link_libraries( # PUBLIC bdb # PUBLIC scalar - PUBLIC zstd + #PUBLIC zstd PUBLIC rocksdb PUBLIC transport PUBLIC stream diff --git a/source/util/test/decompressTest.cpp b/source/util/test/decompressTest.cpp index 2241be67fb..1a614be36e 100644 --- a/source/util/test/decompressTest.cpp +++ b/source/util/test/decompressTest.cpp @@ -9,47 +9,50 @@ TEST(utilTest, decompress_test) { int64_t tsList[10] = {1700000000, 1700000100, 1700000200, 1700000300, 1700000400, 1700000500, 1700000600, 1700000700, 1700000800, 1700000900}; - char* pOutput[10 * sizeof(int64_t)] = {0}; - int32_t len = tsCompressTimestamp(tsList, sizeof(tsList), sizeof(tsList) / sizeof(tsList[0]), pOutput, 10, ONE_STAGE_COMP, NULL, 0); + char* pOutput[10 * sizeof(int64_t)] = {0}; + int32_t len = tsCompressTimestamp(tsList, sizeof(tsList), sizeof(tsList) / sizeof(tsList[0]), pOutput, 10, + ONE_STAGE_COMP, NULL, 0); char* decompOutput[10 * 8] = {0}; - tsDecompressTimestamp(pOutput, len, 10, decompOutput, sizeof(int64_t)*10, ONE_STAGE_COMP, NULL, 0); + tsDecompressTimestamp(pOutput, len, 10, decompOutput, sizeof(int64_t) * 10, ONE_STAGE_COMP, NULL, 0); - for(int32_t i = 0; i < 10; ++i) { - std::cout<< ((int64_t*)decompOutput)[i] << std::endl; + for (int32_t i = 0; i < 10; ++i) { + std::cout << ((int64_t*)decompOutput)[i] << std::endl; } - memset(decompOutput, 0, 10*8); + memset(decompOutput, 0, 10 * 8); tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 10, - reinterpret_cast(decompOutput), false); + reinterpret_cast(decompOutput), false); - for(int32_t i = 0; i < 10; ++i) { - std::cout<<((int64_t*)decompOutput)[i] << std::endl; + for (int32_t i = 0; i < 10; ++i) { + std::cout << ((int64_t*)decompOutput)[i] << std::endl; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// int64_t tsList1[7] = {1700000000, 1700000000, 1700000000, 1700000000, 1700000000, 1700000000, 1700000900}; - int32_t len1 = tsCompressTimestamp(tsList1, sizeof(tsList1), sizeof(tsList1) / sizeof(tsList1[0]), pOutput, 7, ONE_STAGE_COMP, NULL, 0); + int32_t len1 = tsCompressTimestamp(tsList1, sizeof(tsList1), sizeof(tsList1) / sizeof(tsList1[0]), pOutput, 7, + ONE_STAGE_COMP, NULL, 0); - memset(decompOutput, 0, 10*8); + memset(decompOutput, 0, 10 * 8); tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 7, - reinterpret_cast(decompOutput), false); + reinterpret_cast(decompOutput), false); - for(int32_t i = 0; i < 7; ++i) { - std::cout<<((int64_t*)decompOutput)[i] << std::endl; + for (int32_t i = 0; i < 7; ++i) { + std::cout << ((int64_t*)decompOutput)[i] << std::endl; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// int64_t tsList2[1] = {1700000000}; - int32_t len2 = tsCompressTimestamp(tsList2, sizeof(tsList2), sizeof(tsList2) / sizeof(tsList2[0]), pOutput, 1, ONE_STAGE_COMP, NULL, 0); + int32_t len2 = tsCompressTimestamp(tsList2, sizeof(tsList2), sizeof(tsList2) / sizeof(tsList2[0]), pOutput, 1, + ONE_STAGE_COMP, NULL, 0); - memset(decompOutput, 0, 10*8); + memset(decompOutput, 0, 10 * 8); tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 1, - reinterpret_cast(decompOutput), false); + reinterpret_cast(decompOutput), false); - for(int32_t i = 0; i < 1; ++i) { - std::cout<<((int64_t*)decompOutput)[i] << std::endl; + for (int32_t i = 0; i < 1; ++i) { + std::cout << ((int64_t*)decompOutput)[i] << std::endl; } } @@ -57,22 +60,22 @@ TEST(utilTest, decompress_perf_test) { int32_t num = 10000; int64_t* pList = static_cast(taosMemoryCalloc(num, sizeof(int64_t))); - int64_t iniVal = 1700000000; + int64_t iniVal = 1700000000; uint32_t v = 100; - for(int32_t i = 0; i < num; ++i) { - iniVal += taosRandR(&v)%10; + for (int32_t i = 0; i < num; ++i) { + iniVal += taosRandR(&v) % 10; pList[i] = iniVal; } - char* px = static_cast(taosMemoryMalloc(num * sizeof(int64_t))); + char* px = static_cast(taosMemoryMalloc(num * sizeof(int64_t))); int32_t len = tsCompressTimestamp(pList, num * sizeof(int64_t), num, px, num, ONE_STAGE_COMP, NULL, 0); char* pOutput = static_cast(taosMemoryMalloc(num * sizeof(int64_t))); int64_t st = taosGetTimestampUs(); - for(int32_t k = 0; k < 10000; ++k) { + for (int32_t k = 0; k < 10000; ++k) { tsDecompressTimestamp(px, len, num, pOutput, sizeof(int64_t) * num, ONE_STAGE_COMP, NULL, 0); } @@ -81,7 +84,7 @@ TEST(utilTest, decompress_perf_test) { memset(pOutput, 0, num * sizeof(int64_t)); st = taosGetTimestampUs(); - for(int32_t k = 0; k < 10000; ++k) { + for (int32_t k = 0; k < 10000; ++k) { tsDecompressTimestampAvx512(px, num, pOutput, false); } @@ -93,3 +96,49 @@ TEST(utilTest, decompress_perf_test) { taosMemoryFree(px); } +void setColEncode(uint32_t* compress, uint8_t l1) { + *compress &= 0x00FFFFFF; + *compress |= (l1 << 24); + return; +} +void setColCompress(uint32_t* compress, uint16_t l2) { + *compress &= 0xFF0000FF; + *compress |= (l2 << 8); + return; +} +void setColLevel(uint32_t* compress, uint8_t level) { + *compress &= 0xFFFFFF00; + *compress |= level; + return; +} +TEST(utilTest, zstdtest) { + int32_t num = 10000; + + int64_t* pList = static_cast(taosMemoryCalloc(num, sizeof(int64_t))); + int64_t iniVal = 1700000000; + + uint32_t v = 100; + + for (int32_t i = 0; i < num; ++i) { + iniVal += num; + pList[i] = iniVal; + } + uint32_t cmprAlg = 0; + setColCompress(&cmprAlg, 1); + setColEncode(&cmprAlg, 1); + + char* pOutput = static_cast(taosMemoryMalloc(num * sizeof(int64_t))); + + int32_t bufSize = num * sizeof(int64_t) + 64; + char* pBuf = static_cast(taosMemoryMalloc(bufSize)); + + int32_t sz = + tsCompressTimestamp2(pList, num * sizeof(int64_t), num, pOutput, num * sizeof(int64_t), cmprAlg, pBuf, bufSize); + printf("compress size: %d", sz); + + tsDecompressTimestamp(pOutput, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) + + + + +}