xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

more code

This commit is contained in:
Hongze Cheng 2022-09-19 14:39:21 +08:00
parent f2cd319df2
commit 61496324e7
1 changed files with 80 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
source/util/src/tcompression.c
View File

@ -815,24 +815,24 @@ int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, cha
uint32_t predicted = prev_value; uint32_t predicted = prev_value;
uint32_t diff = curr.bits ^ predicted; uint32_t diff = curr.bits ^ predicted;
int32_t leading_zeros = FLOAT_BYTES * BITS_PER_BYTE; int32_t clz = FLOAT_BYTES * BITS_PER_BYTE;
int32_t trailing_zeros = leading_zeros; int32_t ctz = clz;
if (diff) { if (diff) {
trailing_zeros = BUILDIN_CTZ(diff); ctz = BUILDIN_CTZ(diff);
leading_zeros = BUILDIN_CLZ(diff); clz = BUILDIN_CLZ(diff);
} }
uint8_t nbytes = 0; uint8_t nbytes = 0;
uint8_t flag; uint8_t flag;
if (trailing_zeros > leading_zeros) { if (ctz > clz) {
nbytes = (uint8_t)(FLOAT_BYTES - trailing_zeros / BITS_PER_BYTE); nbytes = (uint8_t)(FLOAT_BYTES - ctz / BITS_PER_BYTE);
if (nbytes > 0) nbytes--; if (nbytes > 0) nbytes--;
flag = ((uint8_t)1 << 3) | nbytes; flag = ((uint8_t)1 << 3) | nbytes;
} else { } else {
nbytes = (uint8_t)(FLOAT_BYTES - leading_zeros / BITS_PER_BYTE); nbytes = (uint8_t)(FLOAT_BYTES - clz / BITS_PER_BYTE);
if (nbytes > 0) nbytes--; if (nbytes > 0) nbytes--;
flag = nbytes; flag = nbytes;
} }
@ -1041,7 +1041,7 @@ typedef struct {
}; };
// Binary ---- // Binary ----
struct { struct {
int32_t b_n; int32_t binary_n;
}; };
}; };
} SCompressor; } SCompressor;
@ -1100,7 +1100,7 @@ static int32_t tCompSetCopyMode(SCompressor *pCmprsor) {
memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->aBuf[1], pCmprsor->nBuf[1]); memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->aBuf[1], pCmprsor->nBuf[1]);
pCmprsor->nBuf[0] = 1 + pCmprsor->nBuf[1]; pCmprsor->nBuf[0] = 1 + pCmprsor->nBuf[1];
} }
pCmprsor->aBuf[0][0] = MODE_NOCOMPRESS; pCmprsor->aBuf[0][0] = 0;
return code; return code;
} }
@ -1109,7 +1109,7 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, int64_t ts) {
ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP);
if (pCmprsor->aBuf[0][0] == MODE_COMPRESS) { if (pCmprsor->aBuf[0][0] == 1) {
if (pCmprsor->ts_n == 0) { if (pCmprsor->ts_n == 0) {
pCmprsor->ts_prev_val = ts; pCmprsor->ts_prev_val = ts;
pCmprsor->ts_prev_delta = -ts; pCmprsor->ts_prev_delta = -ts;
@ -1259,43 +1259,47 @@ static int32_t tCompFloat(SCompressor *pCmprsor, float f) {
clz = BUILDIN_CLZ(diff); clz = BUILDIN_CLZ(diff);
ctz = BUILDIN_CTZ(diff); ctz = BUILDIN_CTZ(diff);
} else { } else {
clz = sizeof(uint32_t); clz = 32;
ctz = sizeof(uint32_t); ctz = 32;
} }
if (pCmprsor->f_n & 0x1 == 0) { uint8_t nBytes;
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9 /* sizeof(float) * 2 + 1 */); if (clz < ctz) {
if (code) return code; nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
if (nBytes) diff >>= (32 - nBytes * BITS_PER_BYTE);
} else {
nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
}
if (nBytes == 0) nBytes++;
if ((pCmprsor->f_n & 0x1) == 0) {
if (pCmprsor->autoAlloc) {
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9);
if (code) return code;
}
pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
pCmprsor->nBuf[0]++; pCmprsor->nBuf[0]++;
if (clz < ctz) { if (clz < ctz) {
uint8_t nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1)); pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1));
diff >>= (32 - nBytes * BITS_PER_BYTE);
} else { } else {
uint8_t nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
pCmprsor->f_flag_p[0] = nBytes - 1; pCmprsor->f_flag_p[0] = nBytes - 1;
} }
} else { } else {
if (clz < ctz) { if (clz < ctz) {
uint8_t nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
pCmprsor->f_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4); pCmprsor->f_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4);
diff >>= (32 - nBytes * BITS_PER_BYTE);
} else { } else {
uint8_t nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
pCmprsor->f_flag_p[0] |= ((nBytes - 1) << 4); pCmprsor->f_flag_p[0] |= ((nBytes - 1) << 4);
} }
} }
for (; nBytes; nBytes--) {
while (diff) {
pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
pCmprsor->nBuf[0]++; pCmprsor->nBuf[0]++;
diff >>= BITS_PER_BYTE; diff >>= BITS_PER_BYTE;
} }
pCmprsor->f_n++; pCmprsor->f_n++;
return code; return code;
} }
@ -1316,43 +1320,47 @@ static int32_t tCompDouble(SCompressor *pCmprsor, double d) {
clz = BUILDIN_CLZL(diff); clz = BUILDIN_CLZL(diff);
ctz = BUILDIN_CTZL(diff); ctz = BUILDIN_CTZL(diff);
} else { } else {
clz = sizeof(uint64_t); clz = 64;
ctz = sizeof(uint64_t); ctz = 64;
} }
if (pCmprsor->d_n & 0x1 == 0) { uint8_t nBytes;
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17 /* sizeof(double) * 2 + 1 */); if (clz < ctz) {
if (code) return code; nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
if (nBytes) diff >>= (64 - nBytes * BITS_PER_BYTE);
} else {
nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
}
if (nBytes == 0) nBytes++;
if ((pCmprsor->d_n & 0x1) == 0) {
if (pCmprsor->autoAlloc) {
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17);
if (code) return code;
}
pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
pCmprsor->nBuf[0]++; pCmprsor->nBuf[0]++;
if (clz < ctz) { if (clz < ctz) {
uint8_t nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1)); pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1));
diff >>= (64 - nBytes * BITS_PER_BYTE);
} else { } else {
uint8_t nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
pCmprsor->d_flag_p[0] = nBytes - 1; pCmprsor->d_flag_p[0] = nBytes - 1;
} }
} else { } else {
if (clz < ctz) { if (clz < ctz) {
uint8_t nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
pCmprsor->d_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4); pCmprsor->d_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4);
diff >>= (64 - nBytes * BITS_PER_BYTE);
} else { } else {
uint8_t nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
pCmprsor->d_flag_p[0] |= ((nBytes - 1) << 4); pCmprsor->d_flag_p[0] |= ((nBytes - 1) << 4);
} }
} }
for (; nBytes; nBytes--) {
while (diff) {
pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
pCmprsor->nBuf[0]++; pCmprsor->nBuf[0]++;
diff >>= BITS_PER_BYTE; diff >>= BITS_PER_BYTE;
} }
pCmprsor->d_n++; pCmprsor->d_n++;
return code; return code;
} }
@ -1361,13 +1369,15 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const uint8_t *pData, int32_t
int32_t code = 0; int32_t code = 0;
if (nData) { if (nData) {
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData); if (pCmprsor->autoAlloc) {
if (code) return code; code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData);
if (code) return code;
}
memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData); memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData);
pCmprsor->nBuf[0] += nData; pCmprsor->nBuf[0] += nData;
} }
pCmprsor->b_n++; pCmprsor->binary_n++;
return code; return code;
} }
@ -1379,19 +1389,21 @@ static int32_t tCompBool(SCompressor *pCmprsor, bool vBool) {
int32_t code = 0; int32_t code = 0;
int32_t mod4 = pCmprsor->bool_n & 3; int32_t mod4 = pCmprsor->bool_n & 3;
if (mod4 == 0) {
pCmprsor->nBuf[0]++;
if (pCmprsor->autoAlloc) {
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
if (code) return code;
}
pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] = 0;
}
if (vBool) { if (vBool) {
pCmprsor->aBuf[0][pCmprsor->nBuf[0]] |= BOOL_CMPR_TABLE[mod4]; pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] |= BOOL_CMPR_TABLE[mod4];
} }
pCmprsor->bool_n++; pCmprsor->bool_n++;
if (mod4 == 3) {
pCmprsor->nBuf[0]++;
pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = 0;
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
if (code) goto _exit;
}
_exit:
return code; return code;
} }
@ -1444,16 +1456,30 @@ int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int
pCmprsor->ts_prev_val = 0; pCmprsor->ts_prev_val = 0;
pCmprsor->ts_prev_delta = 0; pCmprsor->ts_prev_delta = 0;
pCmprsor->ts_flag_p = NULL; pCmprsor->ts_flag_p = NULL;
pCmprsor->aBuf[0][0] = MODE_COMPRESS; pCmprsor->aBuf[0][0] = 1; // For timestamp, 1 means compressed, 0 otherwise
pCmprsor->nBuf[0] = 1; pCmprsor->nBuf[0] = 1;
pCmprsor->nBuf[1] = 0;
break; break;
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
pCmprsor->bool_n = 0; pCmprsor->bool_n = 0;
pCmprsor->aBuf[0][0] = 0; pCmprsor->nBuf[0] = 0;
break; break;
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
pCmprsor->b_n = 0; pCmprsor->binary_n = 0;
pCmprsor->nBuf[0] = 0;
break;
case TSDB_DATA_TYPE_FLOAT:
pCmprsor->f_n = 0;
pCmprsor->f_prev = 0;
pCmprsor->f_flag_p = NULL;
pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility)
pCmprsor->nBuf[0] = 1;
break;
case TSDB_DATA_TYPE_DOUBLE:
pCmprsor->d_n = 0;
pCmprsor->d_prev = 0;
pCmprsor->d_flag_p = NULL;
pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility)
pCmprsor->nBuf[0] = 1;
break; break;
default: default:
break; break;