Merge pull request #18002 from taosdata/feat/TS-2007

enh: support wider range for future timestamp

include/util/tdef.h

@@ -26,7 +26,7 @@ extern "C" {
 
 #define TSKEY             int64_t
 #define TSKEY_MIN         INT64_MIN
-#define TSKEY_MAX         (INT64_MAX - 1)
+#define TSKEY_MAX         INT64_MAX
 #define TSKEY_INITIAL_VAL TSKEY_MIN
 
 #define TD_VER_MAX UINT64_MAX  // TODO: use the real max version from query handle

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -495,6 +495,10 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
   pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision);
   tsdbFidKeyRange(pCommitter->commitFid, pCommitter->minutes, pCommitter->precision, &pCommitter->minKey,
                   &pCommitter->maxKey);
+#if 0
+  ASSERT(pCommitter->minKey <= pCommitter->nextKey && pCommitter->maxKey >= pCommitter->nextKey);
+#endif
+  
   pCommitter->nextKey = TSKEY_MAX;
 
   // Reader

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -505,8 +505,8 @@ int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision) {
 }
 
 void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey) {
-  *minKey = fid * minutes * tsTickPerMin[precision];
+  *minKey = tsTickPerMin[precision] * fid * minutes;
-  *maxKey = *minKey + minutes * tsTickPerMin[precision] - 1;
+  *maxKey = *minKey + tsTickPerMin[precision] * minutes - 1;
 }
 
 int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now) {

source/dnode/vnode/src/tsdb/tsdbWrite.c

@@ -15,6 +15,15 @@
 
 #include "tsdb.h"
 
+/**
+ * @brief max key by precision
+ *  approximately calculation:
+ *  ms: 3600*1000*8765*1000         // 1970 + 1000 years
+ *  us: 3600*1000000*8765*1000      // 1970 + 1000 years
+ *  ns: 3600*1000000000*8765*292    // 1970 + 292 years
+ */
+static int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 9214646400000000000L};
+
 // static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg);
 
 int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
@@ -97,7 +106,7 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) {
   STsdbKeepCfg  *pCfg = &pTsdb->keepCfg;
   TSKEY          now = taosGetTimestamp(pCfg->precision);
   TSKEY          minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2;
-  TSKEY          maxKey = now + tsTickPerMin[pCfg->precision] * pCfg->days;
+  TSKEY          maxKey = tsMaxKeyByPrecision[pCfg->precision];
 
   terrno = TSDB_CODE_SUCCESS;
   // pMsg->length = htonl(pMsg->length);

tests/script/tsim/db/commit.sim

@@ -89,8 +89,8 @@ if $data01 != 40 then
 endi
 
 print ======== step4 import new data
-sql_error import into tb values (now + 30d , 30 )
+#sql_error import into tb values (now + 30d , 30 )
-sql_error import into tb values (now + 31d , 31 )
+#sql_error import into tb values (now + 31d , 31 )
 
 sql select * from tb order by ts desc
 print ===> rows $rows

tests/script/tsim/table/date.sim

@@ -9,7 +9,7 @@ $tbPrefix = lm_da_tb
 $db = $dbPrefix . $i
 $tb = $tbPrefix . $i
 
-print =============== step1
+print =============== step1 ms db
 sql create database $db
 sql use $db
 
@@ -23,7 +23,7 @@ if $data00 != @17-01-01 08:00:00.001@ then
   return -1
 endi
 
-print =============== step2
+print =============== step2 ms db
 sql_error insert into $tb values ('2017-08-28 00:23:46.429+ 1a', 2) 
 sql_error insert into $tb values ('2017-08-28 00:23:46cd .429', 2) 
 sql select ts from $tb
@@ -31,7 +31,7 @@ if $rows != 1 then
   return -1
 endi
 
-print =============== step3
+print =============== step3 ms db
 sql_error insert into $tb values ('1970-01-01 08:00:00.000', 3)
 sql_error insert into $tb values ('1970-01-01 08:00:00.000', 3)
 sql select ts from $tb
@@ -39,39 +39,48 @@ if $rows != 1 then
   return -1
 endi
 
-print =============== step4
+print =============== step4 ms db
 sql insert into $tb values(now, 4);
 sql insert into $tb values(now+1a, 5);
 sql insert into $tb values(now+1s, 6);
 sql insert into $tb values(now+1m, 7);
 sql insert into $tb values(now+1h, 8);
 sql insert into $tb values(now+1d, 9);
-sql_error insert into $tb values(now+3w, 10);
+sql insert into $tb values(now+3w, 10);
-sql_error insert into $tb values(now+1n, 11);
+sql insert into $tb values(31556995200000, 11);
-sql_error insert into $tb values(now+1y, 12);
+sql insert into $tb values('2970-01-01 00:00:00.000', 12);
 
-print =============== step5
+sql_error insert into $tb values(now+1n, 20);
-sql_error insert into $tb values ('9999-12-31 213:59:59.999', 13)
+sql_error insert into $tb values(now+1y, 21);
+sql_error insert into $tb values(31556995200001, 22);
+sql_error insert into $tb values('2970-01-02 00:00:00.000', 23);
+sql_error insert into $tb values(9223372036854775807, 24);
+sql_error insert into $tb values(9223372036854775808, 25);
+sql_error insert into $tb values(92233720368547758088, 26);
+
+
+print =============== step5 ms db
+sql_error insert into $tb values ('9999-12-31 213:59:59.999', 27)
 sql select ts from $tb
 print $rows
-if $rows != 7 then
+if $rows != 10 then
   return -1
 endi
 
-print =============== step6
+print =============== step6 ms db
-sql_error insert into $tb values ('9999-12-99 23:59:59.999', 13)
+sql_error insert into $tb values ('9999-12-99 23:59:59.999', 28)
 
 sql select ts from $tb
-if $rows != 7 then
+if $rows != 10 then
   return -1
 endi
 
-print =============== step7
+print =============== step7 ms db
 $i = 1
 $tb = $tbPrefix . $i
 sql create table $tb (ts timestamp, ts2 timestamp)
 
-print =============== step8
+print =============== step8 ms db
 sql insert into $tb values (now, now)
 sql select * from $tb 
 if $rows != 1 then 
@@ -84,4 +93,131 @@ if $rows != 2 then
   return -1
 endi
 
+print =============== step20 us db
+sql create database $db precision 'us' keep 365000d;
+
+sql create table $tb (ts timestamp, speed int)
+sql insert into $tb values ('2017-01-01 08:00:00.001', 1)
+sql select ts from $tb
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != @17-01-01 08:00:00.001000@ then
+  print data00 = $data00
+  return -1
+endi
+
+print =============== step21 us db
+sql_error insert into $tb values ('2017-08-28 00:23:46.429+ 1a', 2) 
+sql_error insert into $tb values ('2017-08-28 00:23:46cd .429', 2) 
+sql select ts from $tb
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== step22 us db
+sql_error insert into $tb values ('970-01-01 08:00:00.000', 3)
+sql_error insert into $tb values ('970-01-01 08:00:00.000', 3)
+sql select ts from $tb
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== step23 us db
+sql insert into $tb values(now, 4);
+sql insert into $tb values(now+1a, 5);
+sql insert into $tb values(now+1s, 6);
+sql insert into $tb values(now+1m, 7);
+sql insert into $tb values(now+1h, 8);
+sql insert into $tb values(now+1d, 9);
+sql insert into $tb values(now+3w, 10);
+sql insert into $tb values(31556995200000000, 11);
+sql insert into $tb values('2970-01-01 00:00:00.000000', 12);
+
+sql_error insert into $tb values(now+1n, 20);
+sql_error insert into $tb values(now+1y, 21);
+sql_error insert into $tb values(31556995200000001, 22);
+sql_error insert into $tb values('2970-01-02 00:00:00.000000', 23);
+sql_error insert into $tb values(9223372036854775807, 24);
+sql_error insert into $tb values(9223372036854775808, 25);
+sql_error insert into $tb values(92233720368547758088, 26);
+sql_error insert into $tb values ('9999-12-31 213:59:59.999', 27)
+
+print =============== step24 us db
+sql select ts from $tb
+print $rows
+if $rows != 10 then
+  return -1
+endi
+
+sql drop database $db
+sql select * from information_schema.ins_databases
+if $rows != 2 then 
+  return -1
+endi
+
+print =============== step30 ns db
+sql create database $db precision 'ns' keep 36500d;
+
+sql create table $tb (ts timestamp, speed int)
+sql insert into $tb values ('2017-01-01 08:00:00.001', 1)
+sql select ts from $tb
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != @17-01-01 08:00:00.001000000@ then
+  print data00 = $data00
+  return -1
+endi
+
+print =============== step31 ns db
+sql_error insert into $tb values ('2017-08-28 00:23:46.429+ 1a', 2) 
+sql_error insert into $tb values ('2017-08-28 00:23:46cd .429', 2) 
+sql select ts from $tb
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== step32 ns db
+#sql_error insert into $tb values ('970-01-01 08:00:00.000000000', 3)
+#sql_error insert into $tb values ('970-01-01 08:00:00.000000000', 3)
+sql select ts from $tb
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== step33 ns db
+sql insert into $tb values(now, 4);
+sql insert into $tb values(now+1a, 5);
+sql insert into $tb values(now+1s, 6);
+sql insert into $tb values(now+1m, 7);
+sql insert into $tb values(now+1h, 8);
+sql insert into $tb values(now+1d, 9);
+sql insert into $tb values(now+3w, 10);
+sql insert into $tb values(9214646400000000000, 11);
+sql insert into $tb values('2262-01-01 00:00:00.000000000', 12);
+
+sql_error insert into $tb values(now+1n, 20);
+sql_error insert into $tb values(now+1y, 21);
+sql_error insert into $tb values(9214646400000000001, 22);
+sql_error insert into $tb values('2262-01-02 00:00:00.000000000', 23);
+sql_error insert into $tb values(9223372036854775807, 24);
+sql_error insert into $tb values(9223372036854775808, 25);
+sql_error insert into $tb values(92233720368547758088, 26);
+sql_error insert into $tb values ('9999-12-31 213:59:59.999', 27)
+
+print =============== step34 ns db
+sql select ts from $tb
+print $rows
+if $rows != 10 then
+  return -1
+endi
+
+sql drop database $db
+sql select * from information_schema.ins_databases
+if $rows != 2 then 
+  return -1
+endi
+
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT