From ca2fb49112b5b4380415c6cb26842dc93dfab389 Mon Sep 17 00:00:00 2001
From: maxmon <541182180@qq.com>
Date: Fri, 14 Apr 2023 00:50:09 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E4=B8=8A=E4=BC=A0jso?=
 =?UTF-8?q?nl=E6=96=87=E4=BB=B6=EF=BC=8C=E5=B9=B6=E6=94=AF=E6=8C=81?=
 =?UTF-8?q?=E5=B0=86=E5=80=BC=E6=8C=89=E7=89=B9=E5=AE=9A=E6=A0=87=E8=AE=B0?=
 =?UTF-8?q?=E5=88=86=E5=89=B2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 be/app/api/v1/project.py         | 35 ++++++++++++++++++++++++++++++++
 be/app/libs/tools.py             |  2 ++
 fe/src/components/NLP/rlhf.vue   |  2 +-
 fe/src/components/home/index.vue | 13 +++++++++---
 4 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/be/app/api/v1/project.py b/be/app/api/v1/project.py
index ac4636b..b84e53e 100644
--- a/be/app/api/v1/project.py
+++ b/be/app/api/v1/project.py
@@ -67,6 +67,41 @@ def get_zipped_data():
 
     return json.dumps(ret_info, default=lambda o: o.__dict__)
 
+@api.route('/get_jsonl_data', methods=['POST'])
+def get_jsonl_data():
+    ret_info = ReturnInfo()
+    try:
+        project_name = request.form.get('projectName')
+
+        upload_file = request.files['file']
+        file_path = os.path.join(PROJECT_PATH.format(project_name), upload_file.filename)
+        file_name = upload_file.filename.replace('.jsonl', '')
+        target_path = PROJECT_PATH.format(project_name) + '/data'
+        upload_file.save(file_path)
+        
+        with open(file_path, 'r', encoding='utf-8') as fh:
+            l = fh.readline().strip()
+            idx = 0
+            while l:
+                idx += 1
+                j = json.loads(l)
+                txts = []
+                for key in j:
+                    txts.append(j[key])
+                _idx = ('000000'+str(idx))[-6:]
+                if len(str(idx)) > 6:
+                    _idx = str(idx)
+                open(f'{target_path}/{file_name}_{_idx}', 'w', encoding='utf-8').write('<whale_n>'.join(txts))
+                l = fh.readline().strip()
+        os.remove(file_path)
+
+    except Exception as e:
+        print(e)
+        ret_info.errCode = 404
+        ret_info.errMsg = str(e)
+
+    return json.dumps(ret_info, default=lambda o: o.__dict__)
+
 
 @api.route('/create', methods=['POST'])
 def create_project():
diff --git a/be/app/libs/tools.py b/be/app/libs/tools.py
index 142a068..5bf3c13 100644
--- a/be/app/libs/tools.py
+++ b/be/app/libs/tools.py
@@ -41,6 +41,8 @@ def read_file(file_path):
         return read_img_file(file_path)
     elif ext in ['txt', 'json']:
         return read_txt_file(file_path)
+    elif '.' not in file_path:
+        return read_txt_file(file_path)
     return ''
 
 # 读取img文件
diff --git a/fe/src/components/NLP/rlhf.vue b/fe/src/components/NLP/rlhf.vue
index 220d8d7..f50fe31 100644
--- a/fe/src/components/NLP/rlhf.vue
+++ b/fe/src/components/NLP/rlhf.vue
@@ -1,6 +1,6 @@
 <template>
   <div class="rlhf-box">
-    <div v-for="(line, idx) in fileContent.split('\n')" :key="idx" class="line" @click="clickLine(idx)">
+    <div v-for="(line, idx) in fileContent.split('<whale_n>')" :key="idx" class="line" @click="clickLine(idx)">
       <template v-for="(word, idx) in line">
         <span class="word" v-if="word !== '\n'" :key="idx">{{ word }}</span>
         <br v-if="word === '\n'" :key="idx"/>
diff --git a/fe/src/components/home/index.vue b/fe/src/components/home/index.vue
index 9013f90..bb23d80 100644
--- a/fe/src/components/home/index.vue
+++ b/fe/src/components/home/index.vue
@@ -53,7 +53,7 @@
         </div>
         <p>上传文本：</p>
         <p style="font-size:10px">（请选择包含文本文件的zip、tar文件）</p>
-        <input type="file" id="file-input" accept=".zip,.tar"/>
+        <input type="file" id="file-input" accept=".zip,.tar,.jsonl"/>
         <p class="edit-box-btn-area">
           <button class="button danger" @click="del" v-if="page==='edit'">删除</button>
           <button class="button" @click="submit">提交</button>
@@ -169,9 +169,16 @@ export default {
         const fileInputElement = document.getElementById('file-input')
         if (fileInputElement.files[0]) {
           let formData = new FormData()
+          let file = fileInputElement.files[0]
           formData.append('projectName', projectName)
-          formData.append('file', fileInputElement.files[0])
-          form('/v1/project/get_zipped_data', formData)
+          formData.append('file', file)
+          if (file.name.endsWith('.zip')) {
+            form('/v1/project/get_zipped_data', formData)
+          } else if (file.name.endsWith('.tar')) {
+            form('/v1/project/get_zipped_data', formData)
+          } else if (file.name.endsWith('.jsonl')) {
+            form('/v1/project/get_jsonl_data', formData)
+          }
         }
       })
     },