xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of github.com:taosdata/TDengine into 3.0

This commit is contained in:
plum-lihui 2022-07-22 20:02:55 +08:00
parent b98da6dd04 ebc80183e7
commit 1568fabc5c
4 changed files with 136 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
source/common/src/tglobal.c
View File

@ -226,8 +226,17 @@ static int32_t taosSetTfsCfg(SConfig *pCfg) {
}
if (tsDataDir[0] == 0) {
uError("datadir not set");
return -1;
if (pItem->str != NULL) {
taosAddDataDir(0, pItem->str, 0, 1);
tstrncpy(tsDataDir, pItem->str, PATH_MAX);
if (taosMulMkDir(tsDataDir) != 0) {
uError("failed to create dataDir:%s since %s", tsDataDir, terrstr());
return -1;
}
} else {
uError("datadir not set");
return -1;
}
}
return 0;

57
source/libs/function/src/tudf.c
View File

@ -875,7 +875,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf);
void udfcUvHandleRsp(SClientUvConn *conn);
void udfcUvHandleError(SClientUvConn *conn);
void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf);
void onUdfcPipetWrite(uv_write_t *write, int status);
void onUdfcPipeWrite(uv_write_t *write, int status);
void onUdfcPipeConnect(uv_connect_t *connect, int status);
int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask);
int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask);
@ -1226,7 +1226,7 @@ int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *
}
void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
SClientUvConn *conn = handle->data;
SClientUvConn *conn = handle->data;
SClientConnBuf *connBuf = &conn->readBuf;
int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t);
@ -1244,6 +1244,9 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf
buf->base = NULL;
buf->len = 0;
}
} else if (connBuf->total == -1 && connBuf->len < msgHeadSize) {
buf->base = connBuf->buf + connBuf->len;
buf->len = msgHeadSize - connBuf->len;
} else {
connBuf->cap = connBuf->total > connBuf->cap ? connBuf->total : connBuf->cap;
void *resultBuf = taosMemoryRealloc(connBuf->buf, connBuf->cap);
@ -1258,8 +1261,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf
}
}
fnTrace("conn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total);
fnDebug("udfc uv alloc buffer: cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total);
}
bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) {
@ -1267,7 +1269,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) {
connBuf->total = *(int32_t *) (connBuf->buf);
}
if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) {
fnTrace("udfc complete message is received, now handle it");
fnDebug("udfc complete message is received, now handle it");
return true;
}
return false;
@ -1278,7 +1280,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen then seqnum
if (QUEUE_EMPTY(&conn->taskQueue)) {
fnError("udfc no task waiting for response on connection");
fnError("udfc no task waiting on connection. response seqnum:%"PRId64, seqNum);
return;
}
bool found = false;
@ -1287,6 +1289,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue);
while (h != &conn->taskQueue) {
fnDebug("udfc handle response iterate through queue. uvTask:%d-%p", task->seqNum, task);
if (task->seqNum == seqNum) {
if (found == false) {
found = true;
@ -1315,6 +1318,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
}
void udfcUvHandleError(SClientUvConn *conn) {
fnDebug("handle error on conn: %p, pipe: %p", conn, conn->pipe);
while (!QUEUE_EMPTY(&conn->taskQueue)) {
QUEUE* h = QUEUE_HEAD(&conn->taskQueue);
SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue);
@ -1328,7 +1332,7 @@ void udfcUvHandleError(SClientUvConn *conn) {
}
void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread);
fnDebug("udfc client %p, client read from pipe. nread: %zd", client, nread);
if (nread == 0) return;
SClientUvConn *conn = client->data;
@ -1338,31 +1342,25 @@ void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
if (isUdfcUvMsgComplete(connBuf)) {
udfcUvHandleRsp(conn);
}
}
if (nread < 0) {
fnError("udfc client pipe %p read error: %zd, %s.", client, nread, uv_strerror(nread));
fnError("udfc client pipe %p read error: %zd(%s).", client, nread, uv_strerror(nread));
if (nread == UV_EOF) {
fnError("\tudfc client pipe %p closed", client);
}
udfcUvHandleError(conn);
}
}
void onUdfcPipetWrite(uv_write_t *write, int status) {
SClientUvTaskNode *uvTask = write->data;
uv_pipe_t *pipe = uvTask->pipe;
fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len);
SClientUvConn *conn = pipe->data;
if (status == 0) {
QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue);
} else {
fnError("udfc client %p write error.", pipe);
void onUdfcPipeWrite(uv_write_t *write, int status) {
SClientUvConn *conn = write->data;
if (status < 0) {
fnError("udfc client connection %p write failed. status: %d(%s)", conn, status, uv_strerror(status));
udfcUvHandleError(conn);
} else {
fnDebug("udfc client connection %p write succeed", conn);
}
taosMemoryFree(write);
taosMemoryFree(uvTask->reqBuf.base);
}
void onUdfcPipeConnect(uv_connect_t *connect, int status) {
@ -1419,7 +1417,7 @@ int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN
}
int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) {
fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask);
fnDebug("queue uv task to event loop, uvTask: %d-%p", uvTask->type, uvTask);
SUdfcProxy *udfc = uvTask->udfc;
uv_mutex_lock(&udfc->taskQueueMutex);
QUEUE_INSERT_TAIL(&udfc->taskQueue, &uvTask->recvTaskQueue);
@ -1427,14 +1425,14 @@ int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) {
uv_async_send(&udfc->loopTaskAync);
uv_sem_wait(&uvTask->taskSem);
fnInfo("udfc uv task finished. task: %d, %p", uvTask->type, uvTask);
fnInfo("udfc uvTask finished. uvTask:%"PRId64"-%d-%p", uvTask->seqNum, uvTask->type, uvTask);
uv_sem_destroy(&uvTask->taskSem);
return 0;
}
int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask);
fnDebug("event loop start uv task. uvTask: %"PRId64"-%d-%p", uvTask->seqNum, uvTask->type, uvTask);
int32_t code = 0;
switch (uvTask->type) {
@ -1465,10 +1463,12 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
code = TSDB_CODE_UDF_PIPE_NO_PIPE;
} else {
uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t));
write->data = uvTask;
int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipetWrite);
write->data = pipe->data;
QUEUE* connTaskQueue = &((SClientUvConn*)pipe->data)->taskQueue;
QUEUE_INSERT_TAIL(connTaskQueue, &uvTask->connTaskQueue);
int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipeWrite);
if (err != 0) {
fnError("udfc event loop start req/rsp task uv_write failed. code: %s", uv_strerror(err));
fnError("udfc event loop start req_rsp task uv_write failed. uvtask: %p, code: %s", uvTask, uv_strerror(err));
}
code = err;
}
@ -1618,6 +1618,7 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) {
SClientUvTaskNode *uvTask = NULL;
udfcCreateUvTask(task, uvTaskType, &uvTask);
fnDebug("udfc client task: %p created uvTask: %p. pipe: %p", task, uvTask, task->session->udfUvPipe);
udfcQueueUvTask(uvTask);
udfcGetUdfTaskResultFromUvTask(task, uvTask);
if (uvTaskType == UV_TASK_CONNECT) {
@ -1625,6 +1626,8 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) {
SClientUvConn *conn = uvTask->pipe->data;
conn->session = task->session;
}
taosMemoryFree(uvTask->reqBuf.base);
uvTask->reqBuf.base = NULL;
taosMemoryFree(uvTask);
uvTask = NULL;
return task->errCode;
@ -1670,7 +1673,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
SSDataBlock* output, SUdfInterBuf *newState) {
fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle);
fnDebug("udfc call udf. callType: %d, funcHandle: %p", callType, handle);
SUdfcUvSession *session = (SUdfcUvSession *) handle;
if (session->udfUvPipe == NULL) {
fnError("No pipe to udfd");

3
source/libs/function/src/udfd.c
View File

@ -671,6 +671,9 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
fnError("udfd can not allocate enough memory") buf->base = NULL;
buf->len = 0;
}
} else if (ctx->inputTotal == -1 && ctx->inputLen < msgHeadSize) {
buf->base = ctx->inputBuf + ctx->inputLen;
buf->len = msgHeadSize - ctx->inputLen;
} else {
ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap;
void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap);

102
source/util/src/tconfig.c
View File

@ -626,6 +626,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
char *line = NULL, *name, *value, *value2, *value3;
int32_t olen, vlen, vlen2, vlen3;
int32_t code = 0;
ssize_t _bytes = 0;
TdCmdPtr pCmd = taosOpenCmd("set");
if (pCmd == NULL) {
@ -658,9 +659,12 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
if (vlen3 != 0) value3[vlen3] = 0;
}
cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR);
if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_VAR);
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_VAR);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
} else {
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
}
}
@ -674,6 +678,7 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) {
char buf[1024], *name, *value, *value2, *value3;
int32_t olen, vlen, vlen2, vlen3;
int32_t code = 0;
int32_t index = 0;
if (envCmd == NULL) return 0;
while (envCmd[index]!=NULL) {
@ -700,9 +705,12 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) {
if (vlen3 != 0) value3[vlen3] = 0;
}
cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD);
if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD);
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
} else {
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
}
}
@ -713,6 +721,7 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) {
int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) {
char *line = NULL, *name, *value, *value2, *value3;
int32_t olen, vlen, vlen2, vlen3;
int32_t code = 0;
ssize_t _bytes = 0;
const char *filepath = ".env";
@ -761,9 +770,12 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) {
if (vlen3 != 0) value3[vlen3] = 0;
}
cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE);
if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_FILE);
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
} else {
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
}
}
@ -819,11 +831,12 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
if (vlen3 != 0) value3[vlen3] = 0;
}
code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
} else {
code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
}
}
@ -839,9 +852,75 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
}
}
// int32_t cfgLoadFromCfgText(SConfig *pConfig, const char *configText) {
// char *line = NULL, *name, *value, *value2, *value3;
// int32_t olen, vlen, vlen2, vlen3;
// ssize_t _bytes = 0;
// int32_t code = 0;
// TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM);
// if (pFile == NULL) {
// // success when the file does not exist
// if (errno == ENOENT) {
// terrno = TAOS_SYSTEM_ERROR(errno);
// uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, terrstr());
// return 0;
// } else {
// uError("failed to load from cfg file %s since %s", filepath, terrstr());
// return -1;
// }
// }
// while (!taosEOFFile(pFile)) {
// name = value = value2 = value3 = NULL;
// olen = vlen = vlen2 = vlen3 = 0;
// _bytes = taosGetLineFile(pFile, &line);
// if (_bytes <= 0) {
// break;
// }
// if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0;
// paGetToken(line, &name, &olen);
// if (olen == 0) continue;
// name[olen] = 0;
// paGetToken(name + olen + 1, &value, &vlen);
// if (vlen == 0) continue;
// value[vlen] = 0;
// paGetToken(value + vlen + 1, &value2, &vlen2);
// if (vlen2 != 0) {
// value2[vlen2] = 0;
// paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
// if (vlen3 != 0) value3[vlen3] = 0;
// }
// code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
// if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
// if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
// code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE);
// if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
// }
// }
// taosCloseFile(&pFile);
// if (line != NULL) taosMemoryFreeClear(line);
// if (code == 0 || (code != 0 && terrno == TSDB_CODE_CFG_NOT_FOUND)) {
// uInfo("load from cfg file %s success", filepath);
// return 0;
// } else {
// uError("failed to load from cfg file %s since %s", filepath, terrstr());
// return -1;
// }
// }
int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
char *cfgLineBuf = NULL, *name, *value, *value2, *value3;
int32_t olen, vlen, vlen2, vlen3;
int32_t code = 0;
if (url == NULL || strlen(url) == 0) {
uInfo("fail to load apoll url");
return 0;
@ -916,9 +995,12 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0;
}
cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL);
if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_APOLLO_URL);
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_APOLLO_URL);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
} else {
code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
}
}
}