xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(tsdb/cache): new case for column changing

This commit is contained in:
Minglei Jin 2024-02-29 18:56:14 +08:00
parent 64f2dc607a
commit 3c80f171ef
4 changed files with 291 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -789,25 +789,6 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, int8_
return code; return code;
} }
static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) {
SLastCol *pLastCol = NULL;
char *err = NULL;
size_t vlen = 0;
SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid};
size_t klen = ROCKS_KEY_LEN;
char *value = NULL;
value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err);
if (NULL != err) {
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err);
rocksdb_free(err);
}
pLastCol = tsdbCacheDeserialize(value);
return pLastCol;
}
typedef struct { typedef struct {
int idx; int idx;
SLastKey key; SLastKey key;
@ -1052,6 +1033,25 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols,
int nCols, int16_t *slotIds); int nCols, int16_t *slotIds);
#ifdef BUILD_NO_CALL #ifdef BUILD_NO_CALL
static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) {
SLastCol *pLastCol = NULL;
char *err = NULL;
size_t vlen = 0;
SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid};
size_t klen = ROCKS_KEY_LEN;
char *value = NULL;
value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err);
if (NULL != err) {
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err);
rocksdb_free(err);
}
pLastCol = tsdbCacheDeserialize(value);
return pLastCol;
}
int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) {
rocksdb_writebatch_t *wb = NULL; rocksdb_writebatch_t *wb = NULL;
int32_t code = 0; int32_t code = 0;
@ -1233,10 +1233,10 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
int16_t *lastSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); int16_t *lastSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t));
int16_t *lastrowColIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); int16_t *lastrowColIds = taosMemoryMalloc(num_keys * sizeof(int16_t));
int16_t *lastrowSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); int16_t *lastrowSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t));
SArray* lastTmpColArray = NULL; SArray *lastTmpColArray = NULL;
SArray* lastTmpIndexArray = NULL; SArray *lastTmpIndexArray = NULL;
SArray* lastrowTmpColArray = NULL; SArray *lastrowTmpColArray = NULL;
SArray* lastrowTmpIndexArray = NULL; SArray *lastrowTmpIndexArray = NULL;
int lastIndex = 0; int lastIndex = 0;
int lastrowIndex = 0; int lastrowIndex = 0;
@ -1245,7 +1245,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
SIdxKey *idxKey = taosArrayGet(remainCols, i); SIdxKey *idxKey = taosArrayGet(remainCols, i);
slotIds[i] = pr->pSlotIds[idxKey->idx]; slotIds[i] = pr->pSlotIds[idxKey->idx];
if (idxKey->key.ltype == CACHESCAN_RETRIEVE_LAST >> 3) { if (idxKey->key.ltype == CACHESCAN_RETRIEVE_LAST >> 3) {
if(NULL == lastTmpIndexArray) { if (NULL == lastTmpIndexArray) {
lastTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); lastTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t));
} }
taosArrayPush(lastTmpIndexArray, &(i)); taosArrayPush(lastTmpIndexArray, &(i));
@ -1253,7 +1253,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
lastSlotIds[lastIndex] = pr->pSlotIds[idxKey->idx]; lastSlotIds[lastIndex] = pr->pSlotIds[idxKey->idx];
lastIndex++; lastIndex++;
} else { } else {
if(NULL == lastrowTmpIndexArray) { if (NULL == lastrowTmpIndexArray) {
lastrowTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); lastrowTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t));
} }
taosArrayPush(lastrowTmpIndexArray, &(i)); taosArrayPush(lastrowTmpIndexArray, &(i));
@ -1265,17 +1265,18 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
pTmpColArray = taosArrayInit(lastIndex + lastrowIndex, sizeof(SLastCol)); pTmpColArray = taosArrayInit(lastIndex + lastrowIndex, sizeof(SLastCol));
if(lastTmpIndexArray != NULL) { if (lastTmpIndexArray != NULL) {
mergeLastCid(uid, pTsdb, &lastTmpColArray, pr, lastColIds, lastIndex, lastSlotIds); mergeLastCid(uid, pTsdb, &lastTmpColArray, pr, lastColIds, lastIndex, lastSlotIds);
for(int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) { for (int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) {
taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i)); taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i));
} }
} }
if(lastrowTmpIndexArray != NULL) { if (lastrowTmpIndexArray != NULL) {
mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds); mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds);
for(int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) { for (int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) {
taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastrowTmpIndexArray, i), taosArrayGet(lastrowTmpColArray, i)); taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastrowTmpIndexArray, i),
taosArrayGet(lastrowTmpColArray, i));
} }
} }

1
tests/parallel_test/cases.task
View File

@ -1085,6 +1085,7 @@
,,y,script,./test.sh -f tsim/parser/join_multivnode.sim ,,y,script,./test.sh -f tsim/parser/join_multivnode.sim
,,y,script,./test.sh -f tsim/parser/join.sim ,,y,script,./test.sh -f tsim/parser/join.sim
,,y,script,./test.sh -f tsim/parser/last_cache.sim ,,y,script,./test.sh -f tsim/parser/last_cache.sim
,,y,script,./test.sh -f tsim/parser/last_both.sim
,,y,script,./test.sh -f tsim/parser/last_groupby.sim ,,y,script,./test.sh -f tsim/parser/last_groupby.sim
,,y,script,./test.sh -f tsim/parser/lastrow.sim ,,y,script,./test.sh -f tsim/parser/lastrow.sim
,,y,script,./test.sh -f tsim/parser/lastrow2.sim ,,y,script,./test.sh -f tsim/parser/lastrow2.sim

150
tests/script/tsim/parser/last_both.sim Normal file
View File

@ -0,0 +1,150 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
print ======================== dnode1 start
$db = testdb
sql drop database if exists $db
sql create database $db cachemodel 'both' minrows 10 stt_trigger 1
sql use $db
sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int)
sql create table tb1 using st2 tags (1);
sql create table tb2 using st2 tags (2);
sql create table tb3 using st2 tags (3);
sql create table tb4 using st2 tags (4);
sql create table tb5 using st2 tags (1);
sql create table tb6 using st2 tags (2);
sql create table tb7 using st2 tags (3);
sql create table tb8 using st2 tags (4);
sql create table tb9 using st2 tags (5);
sql create table tba using st2 tags (5);
sql create table tbb using st2 tags (5);
sql create table tbc using st2 tags (5);
sql create table tbd using st2 tags (5);
sql create table tbe using st2 tags (5);
sql create table tbf using st2 tags (5);
sql insert into tb9 values ("2021-05-09 10:12:26.000",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.001",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.002",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.003",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.004",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.005",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.006",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.007",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.008",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.009",28, 29, '30', -1005)
sql delete from tb9 where ts = "2021-05-09 10:12:26.000"
sql flush database $db
sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000)
sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000)
sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000)
sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3', -1001)
sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001)
sql insert into tb2 values ("2021-05-11 10:11:15",-6, -7, '-8', -3001)
sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002)
sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002)
sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002)
sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003)
sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003)
sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003)
sql insert into tb5 values ("2021-05-09 10:12:22",19, 20, '21', -1004)
sql insert into tb6 values ("2021-05-11 10:12:23",22, 23, NULL, -2004)
sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004)
sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004)
sql insert into tba values ("2021-05-10 10:12:27",31, 32, NULL, -2005)
sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005)
sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005)
sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL )
sql drop table tbf;
sql alter table st2 add column c1 int;
sql alter table st2 drop column c1;
run tsim/parser/last_cache_query.sim
sql flush database $db
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
run tsim/parser/last_cache_query.sim
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
sql drop database if exists $db
sql create database $db minrows 10 stt_trigger 1
sql use $db
sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int)
sql create table tb1 using st2 tags (1);
sql create table tb2 using st2 tags (2);
sql create table tb3 using st2 tags (3);
sql create table tb4 using st2 tags (4);
sql create table tb5 using st2 tags (1);
sql create table tb6 using st2 tags (2);
sql create table tb7 using st2 tags (3);
sql create table tb8 using st2 tags (4);
sql create table tb9 using st2 tags (5);
sql create table tba using st2 tags (5);
sql create table tbb using st2 tags (5);
sql create table tbc using st2 tags (5);
sql create table tbd using st2 tags (5);
sql create table tbe using st2 tags (5);
sql create table tbf using st2 tags (5);
sql insert into tb9 values ("2021-05-09 10:12:26.000",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.001",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.002",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.003",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.004",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.005",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.006",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.007",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.008",28, 29, '30', -1005)
sql insert into tb9 values ("2021-05-09 10:12:26.009",28, 29, '30', -1005)
sql delete from tb9 where ts = "2021-05-09 10:12:26.000"
sql flush database $db
sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000)
sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000)
sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000)
sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3', -1001)
sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001)
sql insert into tb2 values ("2021-05-11 10:11:15",-6, -7, '-8', -3001)
sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002)
sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002)
sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002)
sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003)
sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003)
sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003)
sql insert into tb5 values ("2021-05-09 10:12:22",19, 20, '21', -1004)
sql insert into tb6 values ("2021-05-11 10:12:23",22, 23, NULL, -2004)
sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004)
sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004)
sql insert into tba values ("2021-05-10 10:12:27",31, 32, NULL, -2005)
sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005)
sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005)
sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL )
sql drop table tbf
sql alter database $db cachemodel 'both'
sql alter database $db cachesize 2
sleep 11000
run tsim/parser/last_cache_query.sim
system sh/exec.sh -n dnode1 -s stop -x SIGINT

108
tests/script/tsim/parser/last_cache_query.sim
View File

@ -357,6 +357,112 @@ if $data45 != 5 then
return -1 return -1
endi endi
sql select last_row(*), id from st2 group by id order by id
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49
if $rows != 5 then
return -1
endi
if $data00 != @21-05-12 10:10:12.000@ then
return -1
endi
if $data01 != 6 then
return -1
endi
if $data02 != NULL then
print $data02
return -1
endi
if $data03 != NULL then
return -1
endi
if $data04 != @70-01-01 07:59:57.000@ then
return -1
endi
if $data05 != 1 then
return -1
endi
if $data10 != @21-05-11 10:12:23.000@ then
return -1
endi
if $data11 != 22 then
return -1
endi
if $data12 != 23.000000000 then
print $data02
return -1
endi
if $data13 != NULL then
return -1
endi
if $data14 != @70-01-01 07:59:58.-04@ then
return -1
endi
if $data15 != 2 then
return -1
endi
if $data20 != @21-05-10 10:12:24.000@ then
return -1
endi
if $data21 != 24 then
return -1
endi
if $data22 != NULL then
print expect NULL actual: $data22
return -1
endi
if $data23 != 25 then
return -1
endi
if $data24 != @70-01-01 07:59:57.-04@ then =
return -1
endi
if $data25 != 3 then
return -1
endi
if $data30 != @21-05-11 10:12:25.000@ then
return -1
endi
if $data31 != 26 then
return -1
endi
if $data32 != NULL then
print $data02
return -1
endi
if $data33 != 27 then
return -1
endi
if $data34 != @70-01-01 07:59:56.-04@ then
return -1
endi
if $data35 != 4 then
return -1
endi
if $data40 != @21-05-11 10:12:29.000@ then
return -1
endi
if $data41 != 36 then
return -1
endi
if $data42 != 37.000000000 then
print $data02
return -1
endi
if $data43 != NULL then
return -1
endi
if $data44 != @70-01-01 07:59:56.-05@ then
return -1
endi
if $data45 != 5 then
return -1
endi
print "test tbn" print "test tbn"
sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp)
sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000)
@ -386,3 +492,5 @@ if $data04 != @70-01-01 07:59:57.000@ then
return -1 return -1
endi endi
sql alter table tbn add column c1 int;
sql alter table tbn drop column c1;