From 1f2d5f9ee8412e864cab5733592a18f1aca5845c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 8 Sep 2020 00:08:18 +0800 Subject: [PATCH 1/2] [td-225] fix bug while the content in binary column equals to the maximum allowed length --- src/client/src/TSDBJNIConnector.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index b25f620508..bd980b75a3 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -437,13 +437,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaData * @return */ jstring jniFromNCharToByteArray(JNIEnv *env, char *nchar, int32_t maxBytes) { - int len = (int)strlen(nchar); - if (len > maxBytes) { // no terminated symbol exists '\0' - len = maxBytes; - } - - jbyteArray bytes = (*env)->NewByteArray(env, len); - (*env)->SetByteArrayRegion(env, bytes, 0, len, (jbyte *)nchar); + jbyteArray bytes = (*env)->NewByteArray(env, maxBytes); + (*env)->SetByteArrayRegion(env, bytes, 0, maxBytes, (jbyte *)nchar); return bytes; } @@ -481,6 +476,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn } } + int32_t* length = taos_fetch_lengths(result); + char tmp[TSDB_MAX_BYTES_PER_ROW] = {0}; for (int i = 0; i < num_fields; i++) { @@ -515,15 +512,15 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv); } break; case TSDB_DATA_TYPE_BINARY: { - strncpy(tmp, row[i], (size_t)fields[i].bytes); // handle the case that terminated does not exist + memcpy(tmp, row[i], length[i]); // handle the case that terminated does not exist (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp)); - memset(tmp, 0, (size_t)fields[i].bytes); + memset(tmp, 0, length[i]); break; } case TSDB_DATA_TYPE_NCHAR: { (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteArrayFp, i, - jniFromNCharToByteArray(env, (char *)row[i], fields[i].bytes)); + jniFromNCharToByteArray(env, (char *)row[i], length[i])); break; } case TSDB_DATA_TYPE_TIMESTAMP: From f7d2e7d905f5a6d9770844a25db77737a1efaa6d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 8 Sep 2020 10:00:08 +0800 Subject: [PATCH 2/2] fix TD-1362 --- src/tsdb/src/tsdbFile.c | 47 ++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 29e46a88af..75b742e81a 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -26,11 +26,13 @@ const char *tsdbFileSuffix[] = {".head", ".data", ".last", ".stat", ".h", ".d", ".l", ".s"}; -static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); -static void tsdbDestroyFile(SFile *pFile); -static int compFGroup(const void *arg1, const void *arg2); -static int keyFGroupCompFunc(const void *key, const void *fgroup); -static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo); +static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); +static void tsdbDestroyFile(SFile *pFile); +static int compFGroup(const void *arg1, const void *arg2); +static int keyFGroupCompFunc(const void *key, const void *fgroup); +static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo); +static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep); +static int tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days); // ---------------- INTERNAL FUNCTIONS ---------------- STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) { @@ -79,9 +81,11 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { int vid = 0; regex_t regex1, regex2; int code = 0; + char fname[TSDB_FILENAME_LEN] = "\0"; SFileGroup fileGroup = {0}; STsdbFileH *pFileH = pRepo->tsdbFileH; + STsdbCfg * pCfg = &(pRepo->config); tDataDir = tsdbGetDataDirName(pRepo->rootDir); if (tDataDir == NULL) { @@ -108,6 +112,8 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { goto _err; } + int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile); + struct dirent *dp = NULL; while ((dp = readdir(dir)) != NULL) { if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; @@ -120,6 +126,14 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { continue; } + if (fid < mfid) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { + tsdbGetDataFileName(pRepo, fid, type, fname); + (void)remove(fname); + } + continue; + } + if (tsdbSearchFGroup(pFileH, fid, TD_EQ) != NULL) continue; memset((void *)(&fileGroup), 0, sizeof(SFileGroup)); fileGroup.fileId = fid; @@ -179,8 +193,18 @@ void tsdbCloseFileH(STsdbRepo *pRepo) { SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) { STsdbFileH *pFileH = pRepo->tsdbFileH; + STsdbCfg * pCfg = &(pRepo->config); - if (pFileH->nFGroups >= pFileH->maxFGroups) return NULL; + if (pFileH->nFGroups >= pFileH->maxFGroups) { + int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile); + if (pFileH->pFGroup[0].fileId < mfid) { + pthread_rwlock_wrlock(&pFileH->fhlock); + tsdbRemoveFileGroup(pRepo, &(pFileH->pFGroup[0])); + pthread_rwlock_unlock(&pFileH->fhlock); + } + } + + ASSERT(pFileH->nFGroups < pFileH->maxFGroups); SFileGroup fGroup; SFileGroup *pFGroup = &fGroup; @@ -342,8 +366,7 @@ void tsdbFitRetention(STsdbRepo *pRepo) { STsdbFileH *pFileH = pRepo->tsdbFileH; SFileGroup *pGroup = pFileH->pFGroup; - int mfid = (int)(TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) - - TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile)); + int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile); pthread_rwlock_wrlock(&(pFileH->fhlock)); @@ -533,3 +556,11 @@ static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo) { } } } + +static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep) { + return (TSKEY)(taosGetTimestamp(precision) - keep * tsMsPerDay[precision]); +} + +static int tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days) { + return TSDB_KEY_FILEID(tsdbGetCurrMinKey(precision, keep), days, precision); +} \ No newline at end of file