From 8aa89525c8349778320a74eb48e49f370ece30c4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 29 May 2020 10:56:45 +0000 Subject: [PATCH] add signed value codig functions --- src/tsdb/src/tsdbFile.c | 2 +- src/tsdb/src/tsdbRWHelper.c | 54 ++++---- src/util/inc/tcoding.h | 232 +++++++++++++++++++++++---------- src/util/tests/codingTests.cpp | 24 ++-- 4 files changed, 203 insertions(+), 109 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index bd5c20bd7a..e885d9a919 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -94,7 +94,7 @@ static int tsdbInitFile(char *dataDir, int fid, const char *suffix, SFile *pFile if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) return -1; void *pBuf = buf; - pBuf = taosDecodeFixed32(pBuf, &version); + pBuf = taosDecodeFixedU32(pBuf, &version); pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); tsdbCloseFile(pFile); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index eebe0b6b46..38e6b8f2f6 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -443,7 +443,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { for (uint32_t i = 0; i < pHelper->config.maxTables; i++) { SCompIdx *pCompIdx = pHelper->pCompIdx + i; if (pCompIdx->offset > 0) { - buf = taosEncodeVariant32(buf, i); + buf = taosEncodeVariantU32(buf, i); buf = tsdbEncodeSCompIdx(buf, pCompIdx); } } @@ -480,7 +480,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { void *ptr = pHelper->pBuffer; while (((char *)ptr - (char *)pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) { uint32_t tid = 0; - if ((ptr = taosDecodeVariant32(ptr, &tid)) == NULL) return -1; + if ((ptr = taosDecodeVariantU32(ptr, &tid)) == NULL) return -1; ASSERT(tid > 0 && tid < pHelper->config.maxTables); if ((ptr = tsdbDecodeSCompIdx(ptr, pHelper->pCompIdx + tid)) == NULL) return -1; @@ -1242,12 +1242,12 @@ static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey) } void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx) { - buf = taosEncodeVariant32(buf, pIdx->len); - buf = taosEncodeVariant32(buf, pIdx->offset); - buf = taosEncodeFixed8(buf, pIdx->hasLast); - buf = taosEncodeVariant32(buf, pIdx->numOfBlocks); - buf = taosEncodeFixed64(buf, pIdx->uid); - buf = taosEncodeFixed64(buf, pIdx->maxKey); + buf = taosEncodeVariantU32(buf, pIdx->len); + buf = taosEncodeVariantU32(buf, pIdx->offset); + buf = taosEncodeFixedU8(buf, pIdx->hasLast); + buf = taosEncodeVariantU32(buf, pIdx->numOfBlocks); + buf = taosEncodeFixedU64(buf, pIdx->uid); + buf = taosEncodeFixedU64(buf, pIdx->maxKey); return buf; } @@ -1257,15 +1257,15 @@ void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { uint32_t numOfBlocks = 0; uint64_t value = 0; - if ((buf = taosDecodeVariant32(buf, &(pIdx->len))) == NULL) return NULL; - if ((buf = taosDecodeVariant32(buf, &(pIdx->offset))) == NULL) return NULL; - if ((buf = taosDecodeFixed8(buf, &(hasLast))) == NULL) return NULL; + if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; + if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; + if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; pIdx->hasLast = hasLast; - if ((buf = taosDecodeVariant32(buf, &(numOfBlocks))) == NULL) return NULL; + if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL; pIdx->numOfBlocks = numOfBlocks; - if ((buf = taosDecodeFixed64(buf, &value)) == NULL) return NULL; + if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; pIdx->uid = (int64_t)value; - if ((buf = taosDecodeFixed64(buf, &value)) == NULL) return NULL; + if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; pIdx->maxKey = (TSKEY)value; return buf; @@ -1275,7 +1275,7 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { char buf[TSDB_FILE_HEAD_SIZE] = "\0"; void *pBuf = (void *)buf; - pBuf = taosEncodeFixed32(pBuf, version); + pBuf = taosEncodeFixedU32(pBuf, version); pBuf = tsdbEncodeSFileInfo(pBuf, &(pFile->info)); taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); @@ -1289,23 +1289,23 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo) { - buf = taosEncodeFixed32(buf, pInfo->offset); - buf = taosEncodeFixed32(buf, pInfo->len); - buf = taosEncodeFixed64(buf, pInfo->size); - buf = taosEncodeFixed64(buf, pInfo->tombSize); - buf = taosEncodeFixed32(buf, pInfo->totalBlocks); - buf = taosEncodeFixed32(buf, pInfo->totalSubBlocks); + buf = taosEncodeFixedU32(buf, pInfo->offset); + buf = taosEncodeFixedU32(buf, pInfo->len); + buf = taosEncodeFixedU64(buf, pInfo->size); + buf = taosEncodeFixedU64(buf, pInfo->tombSize); + buf = taosEncodeFixedU32(buf, pInfo->totalBlocks); + buf = taosEncodeFixedU32(buf, pInfo->totalSubBlocks); return buf; } void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { - buf = taosDecodeFixed32(buf, &(pInfo->offset)); - buf = taosDecodeFixed32(buf, &(pInfo->len)); - buf = taosDecodeFixed64(buf, &(pInfo->size)); - buf = taosDecodeFixed64(buf, &(pInfo->tombSize)); - buf = taosDecodeFixed32(buf, &(pInfo->totalBlocks)); - buf = taosDecodeFixed32(buf, &(pInfo->totalSubBlocks)); + buf = taosDecodeFixedU32(buf, &(pInfo->offset)); + buf = taosDecodeFixedU32(buf, &(pInfo->len)); + buf = taosDecodeFixedU64(buf, &(pInfo->size)); + buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); + buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); + buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); return buf; } \ No newline at end of file diff --git a/src/util/inc/tcoding.h b/src/util/inc/tcoding.h index cc9caf71d0..e22c959a56 100644 --- a/src/util/inc/tcoding.h +++ b/src/util/inc/tcoding.h @@ -29,12 +29,33 @@ extern "C" { static const int32_t TNUMBER = 1; #define IS_LITTLE_ENDIAN() (*(uint8_t *)(&TNUMBER) != 0) -static FORCE_INLINE void *taosEncodeFixed8(void *buf, uint8_t value) { +#define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode +#define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode + +// ---- Fixed U8 +static FORCE_INLINE void *taosEncodeFixedU8(void *buf, uint8_t value) { ((uint8_t *)buf)[0] = value; return POINTER_SHIFT(buf, sizeof(value)); } -static FORCE_INLINE void *taosEncodeFixed16(void *buf, uint16_t value) { +static FORCE_INLINE void *taosDecodeFixedU8(void *buf, uint8_t *value) { + *value = ((uint8_t *)buf)[0]; + return POINTER_SHIFT(buf, sizeof(*value)); +} + +// ---- Fixed I8 +static FORCE_INLINE void *taosEncodeFixedI8(void *buf, int8_t value) { + ((int8_t *)buf)[0] = value; + return POINTER_SHIFT(buf, sizeof(value)); +} + +static FORCE_INLINE void *taosDecodeFixedI8(void *buf, int8_t *value) { + *value = ((int8_t *)buf)[0]; + return POINTER_SHIFT(buf, sizeof(*value)); +} + +// ---- Fixed U16 +static FORCE_INLINE void *taosEncodeFixedU16(void *buf, uint16_t value) { if (IS_LITTLE_ENDIAN()) { memcpy(buf, &value, sizeof(value)); } else { @@ -45,7 +66,31 @@ static FORCE_INLINE void *taosEncodeFixed16(void *buf, uint16_t value) { return POINTER_SHIFT(buf, sizeof(value)); } -static FORCE_INLINE void *taosEncodeFixed32(void *buf, uint32_t value) { +static FORCE_INLINE void *taosDecodeFixedU16(void *buf, uint16_t *value) { + if (IS_LITTLE_ENDIAN()) { + memcpy(value, buf, sizeof(*value)); + } else { + ((uint8_t *)value)[1] = ((uint8_t *)buf)[0]; + ((uint8_t *)value)[0] = ((uint8_t *)buf)[1]; + } + + return POINTER_SHIFT(buf, sizeof(*value)); +} + +// ---- Fixed I16 +static FORCE_INLINE void *taosEncodeFixedI16(void *buf, int16_t value) { + return taosEncodeFixedU16(buf, ZIGZAGE(int16_t, value)); +} + +static FORCE_INLINE void *taosDecodeFixedI16(void *buf, int16_t *value) { + uint16_t tvalue = 0; + void * ret = taosDecodeFixedU16(buf, &tvalue); + *value = ZIGZAGD(int16_t, tvalue); + return ret; +} + +// ---- Fixed U32 +static FORCE_INLINE void *taosEncodeFixedU32(void *buf, uint32_t value) { if (IS_LITTLE_ENDIAN()) { memcpy(buf, &value, sizeof(value)); } else { @@ -58,7 +103,33 @@ static FORCE_INLINE void *taosEncodeFixed32(void *buf, uint32_t value) { return POINTER_SHIFT(buf, sizeof(value)); } -static FORCE_INLINE void *taosEncodeFixed64(void *buf, uint64_t value) { +static FORCE_INLINE void *taosDecodeFixedU32(void *buf, uint32_t *value) { + if (IS_LITTLE_ENDIAN()) { + memcpy(value, buf, sizeof(*value)); + } else { + ((uint8_t *)value)[3] = ((uint8_t *)buf)[0]; + ((uint8_t *)value)[2] = ((uint8_t *)buf)[1]; + ((uint8_t *)value)[1] = ((uint8_t *)buf)[2]; + ((uint8_t *)value)[0] = ((uint8_t *)buf)[3]; + } + + return POINTER_SHIFT(buf, sizeof(*value)); +} + +// ---- Fixed I32 +static FORCE_INLINE void *taosEncodeFixedI32(void *buf, int32_t value) { + return taosEncodeFixedU32(buf, ZIGZAGE(int32_t, value)); +} + +static FORCE_INLINE void *taosDecodeFixedI32(void *buf, int32_t *value) { + uint32_t tvalue = 0; + void * ret = taosDecodeFixedU32(buf, &tvalue); + *value = ZIGZAGD(int32_t, tvalue); + return ret; +} + +// ---- Fixed U64 +static FORCE_INLINE void *taosEncodeFixedU64(void *buf, uint64_t value) { if (IS_LITTLE_ENDIAN()) { memcpy(buf, &value, sizeof(value)); } else { @@ -75,36 +146,7 @@ static FORCE_INLINE void *taosEncodeFixed64(void *buf, uint64_t value) { return POINTER_SHIFT(buf, sizeof(value)); } -static FORCE_INLINE void *taosDecodeFixed8(void *buf, uint8_t *value) { - *value = ((uint8_t *)buf)[0]; - return POINTER_SHIFT(buf, sizeof(*value)); -} - -static FORCE_INLINE void *taosDecodeFixed16(void *buf, uint16_t *value) { - if (IS_LITTLE_ENDIAN()) { - memcpy(value, buf, sizeof(*value)); - } else { - ((uint8_t *)value)[1] = ((uint8_t *)buf)[0]; - ((uint8_t *)value)[0] = ((uint8_t *)buf)[1]; - } - - return POINTER_SHIFT(buf, sizeof(*value)); -} - -static FORCE_INLINE void *taosDecodeFixed32(void *buf, uint32_t *value) { - if (IS_LITTLE_ENDIAN()) { - memcpy(value, buf, sizeof(*value)); - } else { - ((uint8_t *)value)[3] = ((uint8_t *)buf)[0]; - ((uint8_t *)value)[2] = ((uint8_t *)buf)[1]; - ((uint8_t *)value)[1] = ((uint8_t *)buf)[2]; - ((uint8_t *)value)[0] = ((uint8_t *)buf)[3]; - } - - return POINTER_SHIFT(buf, sizeof(*value)); -} - -static FORCE_INLINE void *taosDecodeFixed64(void *buf, uint64_t *value) { +static FORCE_INLINE void *taosDecodeFixedU64(void *buf, uint64_t *value) { if (IS_LITTLE_ENDIAN()) { memcpy(value, buf, sizeof(*value)); } else { @@ -121,7 +163,20 @@ static FORCE_INLINE void *taosDecodeFixed64(void *buf, uint64_t *value) { return POINTER_SHIFT(buf, sizeof(*value)); } -static FORCE_INLINE void *taosEncodeVariant16(void *buf, uint16_t value) { +// ---- Fixed I64 +static FORCE_INLINE void *taosEncodeFixedI64(void *buf, int64_t value) { + return taosEncodeFixedU64(buf, ZIGZAGE(int64_t, value)); +} + +static FORCE_INLINE void *taosDecodeFixedI64(void *buf, int64_t *value) { + uint64_t tvalue = 0; + void * ret = taosDecodeFixedU64(buf, &tvalue); + *value = ZIGZAGD(int64_t, tvalue); + return ret; +} + +// ---- Variant U16 +static FORCE_INLINE void *taosEncodeVariantU16(void *buf, uint16_t value) { int i = 0; while (value >= ENCODE_LIMIT) { ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); @@ -132,39 +187,11 @@ static FORCE_INLINE void *taosEncodeVariant16(void *buf, uint16_t value) { ((uint8_t *)buf)[i] = value; - return POINTER_SHIFT(buf, i+1); -} - -static FORCE_INLINE void *taosEncodeVariant32(void *buf, uint32_t value) { - int i = 0; - while (value >= ENCODE_LIMIT) { - ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); - value >>= 7; - i++; - ASSERT(i < 5); - } - - ((uint8_t *)buf)[i] = value; - return POINTER_SHIFT(buf, i + 1); } -static FORCE_INLINE void *taosEncodeVariant64(void *buf, uint64_t value) { - int i = 0; - while (value >= ENCODE_LIMIT) { - ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); - value >>= 7; - i++; - ASSERT(i < 10); - } - - ((uint8_t *)buf)[i] = value; - - return POINTER_SHIFT(buf, i + 1); -} - -static FORCE_INLINE void *taosDecodeVariant16(void *buf, uint16_t *value) { - int i = 0; +static FORCE_INLINE void *taosDecodeVariantU16(void *buf, uint16_t *value) { + int i = 0; uint16_t tval = 0; *value = 0; while (i < 3) { @@ -181,8 +208,35 @@ static FORCE_INLINE void *taosDecodeVariant16(void *buf, uint16_t *value) { return NULL; // error happened } -static FORCE_INLINE void *taosDecodeVariant32(void *buf, uint32_t *value) { +// ---- Variant I16 +static FORCE_INLINE void *taosEncodeVariantI16(void *buf, int16_t value) { + return taosEncodeVariantU16(buf, ZIGZAGE(int16_t, value)); +} + +static FORCE_INLINE void *taosDecodeVariantI16(void *buf, int16_t *value) { + uint16_t tvalue = 0; + void * ret = taosDecodeVariantU16(buf, &tvalue); + *value = ZIGZAGD(int16_t, tvalue); + return ret; +} + +// ---- Variant U32 +static FORCE_INLINE void *taosEncodeVariantU32(void *buf, uint32_t value) { int i = 0; + while (value >= ENCODE_LIMIT) { + ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); + value >>= 7; + i++; + ASSERT(i < 5); + } + + ((uint8_t *)buf)[i] = value; + + return POINTER_SHIFT(buf, i + 1); +} + +static FORCE_INLINE void *taosDecodeVariantU32(void *buf, uint32_t *value) { + int i = 0; uint32_t tval = 0; *value = 0; while (i < 5) { @@ -199,8 +253,35 @@ static FORCE_INLINE void *taosDecodeVariant32(void *buf, uint32_t *value) { return NULL; // error happened } -static FORCE_INLINE void *taosDecodeVariant64(void *buf, uint64_t *value) { +// ---- Variant I32 +static FORCE_INLINE void *taosEncodeVariantI32(void *buf, int32_t value) { + return taosEncodeVariantU32(buf, ZIGZAGE(int32_t, value)); +} + +static FORCE_INLINE void *taosDecodeVariantI32(void *buf, int32_t *value) { + uint32_t tvalue = 0; + void * ret = taosDecodeVariantU32(buf, &tvalue); + *value = ZIGZAGD(int32_t, tvalue); + return ret; +} + +// ---- Variant U64 +static FORCE_INLINE void *taosEncodeVariantU64(void *buf, uint64_t value) { int i = 0; + while (value >= ENCODE_LIMIT) { + ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); + value >>= 7; + i++; + ASSERT(i < 10); + } + + ((uint8_t *)buf)[i] = value; + + return POINTER_SHIFT(buf, i + 1); +} + +static FORCE_INLINE void *taosDecodeVariantU64(void *buf, uint64_t *value) { + int i = 0; uint64_t tval = 0; *value = 0; while (i < 10) { @@ -217,10 +298,23 @@ static FORCE_INLINE void *taosDecodeVariant64(void *buf, uint64_t *value) { return NULL; // error happened } +// ---- Variant I64 +static FORCE_INLINE void *taosEncodeVariantI64(void *buf, int64_t value) { + return taosEncodeVariantU64(buf, ZIGZAGE(int64_t, value)); +} + +static FORCE_INLINE void *taosDecodeVariantI64(void *buf, int64_t *value) { + uint64_t tvalue = 0; + void * ret = taosDecodeVariantU64(buf, &tvalue); + *value = ZIGZAGD(int64_t, tvalue); + return ret; +} + +// ---- string static FORCE_INLINE void *taosEncodeString(void *buf, char *value) { size_t size = strlen(value); - buf = taosEncodeVariant64(buf, size); + buf = taosEncodeVariantU64(buf, size); memcpy(buf, value, size); return POINTER_SHIFT(buf, size); @@ -229,7 +323,7 @@ static FORCE_INLINE void *taosEncodeString(void *buf, char *value) { static FORCE_INLINE void *taosDecodeString(void *buf, char **value) { uint64_t size = 0; - buf = taosDecodeVariant64(buf, &size); + buf = taosDecodeVariantU64(buf, &size); *value = (char *)malloc(size + 1); if (*value == NULL) return NULL; memcpy(*value, buf, size); diff --git a/src/util/tests/codingTests.cpp b/src/util/tests/codingTests.cpp index a72c7ef291..036ed0bf83 100644 --- a/src/util/tests/codingTests.cpp +++ b/src/util/tests/codingTests.cpp @@ -9,8 +9,8 @@ static bool test_fixed_uint16(uint16_t value) { char buf[20] = "\0"; uint16_t value_check = 0; - void *ptr1 = taosEncodeFixed16(static_cast(buf), value); - void *ptr2 = taosDecodeFixed16(static_cast(buf), &value_check); + void *ptr1 = taosEncodeFixedU16(static_cast(buf), value); + void *ptr2 = taosDecodeFixedU16(static_cast(buf), &value_check); return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); } @@ -19,8 +19,8 @@ static bool test_fixed_uint32(uint32_t value) { char buf[20] = "\0"; uint32_t value_check = 0; - void *ptr1 = taosEncodeFixed32(static_cast(buf), value); - void *ptr2 = taosDecodeFixed32(static_cast(buf), &value_check); + void *ptr1 = taosEncodeFixedU32(static_cast(buf), value); + void *ptr2 = taosDecodeFixedU32(static_cast(buf), &value_check); return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); } @@ -29,8 +29,8 @@ static bool test_fixed_uint64(uint64_t value) { char buf[20] = "\0"; uint64_t value_check = 0; - void *ptr1 = taosEncodeFixed64(static_cast(buf), value); - void *ptr2 = taosDecodeFixed64(static_cast(buf), &value_check); + void *ptr1 = taosEncodeFixedU64(static_cast(buf), value); + void *ptr2 = taosDecodeFixedU64(static_cast(buf), &value_check); return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); } @@ -39,8 +39,8 @@ static bool test_variant_uint16(uint16_t value) { char buf[20] = "\0"; uint16_t value_check = 0; - void *ptr1 = taosEncodeVariant16(static_cast(buf), value); - void *ptr2 = taosDecodeVariant16(static_cast(buf), &value_check); + void *ptr1 = taosEncodeVariantU16(static_cast(buf), value); + void *ptr2 = taosDecodeVariantU16(static_cast(buf), &value_check); return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); } @@ -49,8 +49,8 @@ static bool test_variant_uint32(uint32_t value) { char buf[20] = "\0"; uint32_t value_check = 0; - void *ptr1 = taosEncodeVariant32(static_cast(buf), value); - void *ptr2 = taosDecodeVariant32(static_cast(buf), &value_check); + void *ptr1 = taosEncodeVariantU32(static_cast(buf), value); + void *ptr2 = taosDecodeVariantU32(static_cast(buf), &value_check); return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); } @@ -59,8 +59,8 @@ static bool test_variant_uint64(uint64_t value) { char buf[20] = "\0"; uint64_t value_check = 0; - void *ptr1 = taosEncodeVariant64(static_cast(buf), value); - void *ptr2 = taosDecodeVariant64(static_cast(buf), &value_check); + void *ptr1 = taosEncodeVariantU64(static_cast(buf), value); + void *ptr2 = taosDecodeVariantU64(static_cast(buf), &value_check); return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); }