Fix some issues in API (#2982)

### What problem does this PR solve?

Fix some issues in API

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

---------

Co-authored-by: liuhua <10215101452@stu.ecun.edu.cn>

api/apps/sdk/chat.py

@@ -18,20 +18,21 @@
 from api.db import StatusEnum
 from api.db.services.dialog_service import DialogService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.llm_service import TenantLLMService
+from api.db.services.llm_service import  TenantLLMService
 from api.db.services.user_service import TenantService
 from api.utils import get_uuid
 from api.utils.api_utils import get_error_data_result, token_required
 from api.utils.api_utils import get_result
 
 
+
 @manager.route('/chat', methods=['POST'])
 @token_required
 def create(tenant_id):
     req=request.json
-    ids= req.get("knowledgebases")
+    ids= req.get("datasets")
     if not ids:
-        return get_error_data_result(retmsg="`knowledgebases` is required")
+        return get_error_data_result(retmsg="`datasets` is required")
     for kb_id in ids:
         kbs = KnowledgebaseService.query(id=kb_id,tenant_id=tenant_id)
         if not kbs:
@@ -45,6 +46,8 @@ def create(tenant_id):
     if llm:
         if "model_name" in llm:
             req["llm_id"] = llm.pop("model_name")
+            if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req["llm_id"],model_type="chat"):
+                return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
         req["llm_setting"] = req.pop("llm")
     e, tenant = TenantService.get_by_id(tenant_id)
     if not e:
@@ -73,10 +76,10 @@ def create(tenant_id):
     req["top_n"] = req.get("top_n", 6)
     req["top_k"] = req.get("top_k", 1024)
     req["rerank_id"] = req.get("rerank_id", "")
-    if req.get("llm_id"):
-        if not TenantLLMService.query(llm_name=req["llm_id"]):
-            return get_error_data_result(retmsg="the model_name does not exist.")
-    else:
+    if req.get("rerank_id"):
+        if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req.get("rerank_id"),model_type="rerank"):
+            return get_error_data_result(f"`rerank_model` {req.get('rerank_id')} doesn't exist")
+    if not req.get("llm_id"):
         req["llm_id"] = tenant.llm_id
     if not req.get("name"):
         return get_error_data_result(retmsg="`name` is required.")
@@ -135,7 +138,7 @@ def create(tenant_id):
     res["llm"] = res.pop("llm_setting")
     res["llm"]["model_name"] = res.pop("llm_id")
     del res["kb_ids"]
-    res["knowledgebases"] = req["knowledgebases"]
+    res["datasets"] = req["datasets"]
     res["avatar"] = res.pop("icon")
     return get_result(data=res)
 
@@ -145,27 +148,32 @@ def update(tenant_id,chat_id):
     if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
         return get_error_data_result(retmsg='You do not own the chat')
     req =request.json
-    if "knowledgebases" in req:
-        if not req.get("knowledgebases"):
-            return  get_error_data_result(retmsg="`knowledgebases` can't be empty value")
-        kb_list = []
-        for kb in req.get("knowledgebases"):
-            if not kb["id"]:
-                return get_error_data_result(retmsg="knowledgebase needs id")
-            if not KnowledgebaseService.query(id=kb["id"], tenant_id=tenant_id):
-                return get_error_data_result(retmsg="you do not own the knowledgebase")
-            # if not DocumentService.query(kb_id=kb["id"]):
-            #  return get_error_data_result(retmsg="There is a invalid knowledgebase")
-            kb_list.append(kb["id"])
-        req["kb_ids"] = kb_list
+    ids = req.get("datasets")
+    if "datasets" in req:
+        if not ids:
+            return get_error_data_result("`datasets` can't be empty")
+        if ids:
+            for kb_id in ids:
+                kbs = KnowledgebaseService.query(id=kb_id, tenant_id=tenant_id)
+                if not kbs:
+                    return get_error_data_result(f"You don't own the dataset {kb_id}")
+                kb = kbs[0]
+                if kb.chunk_num == 0:
+                    return get_error_data_result(f"The dataset {kb_id} doesn't own parsed file")
+            req["kb_ids"] = ids
     llm = req.get("llm")
     if llm:
         if "model_name" in llm:
             req["llm_id"] = llm.pop("model_name")
+            if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req["llm_id"],model_type="chat"):
+                return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
         req["llm_setting"] = req.pop("llm")
     e, tenant = TenantService.get_by_id(tenant_id)
     if not e:
         return get_error_data_result(retmsg="Tenant not found!")
+    if req.get("rerank_model"):
+        if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req.get("rerank_model"),model_type="rerank"):
+            return get_error_data_result(f"`rerank_model` {req.get('rerank_model')} doesn't exist")
     # prompt
     prompt = req.get("prompt")
     key_mapping = {"parameters": "variables",
@@ -185,9 +193,6 @@ def update(tenant_id,chat_id):
         req["prompt_config"] = req.pop("prompt")
     e, res = DialogService.get_by_id(chat_id)
     res = res.to_json()
-    if "llm_id" in req:
-        if not TenantLLMService.query(llm_name=req["llm_id"]):
-            return get_error_data_result(retmsg="The `model_name` does not exist.")
     if "name" in req:
         if not req.get("name"):
             return get_error_data_result(retmsg="`name` is not empty.")
@@ -209,8 +214,8 @@ def update(tenant_id,chat_id):
     # avatar
     if "avatar" in req:
         req["icon"] = req.pop("avatar")
-    if "knowledgebases" in req:
-        req.pop("knowledgebases")
+    if "datasets" in req:
+        req.pop("datasets")
     if not DialogService.update_by_id(chat_id, req):
         return get_error_data_result(retmsg="Chat not found!")
     return get_result()
@@ -279,7 +284,7 @@ def list_chat(tenant_id):
                 return get_error_data_result(retmsg=f"Don't exist the kb {kb_id}")
             kb_list.append(kb[0].to_json())
         del res["kb_ids"]
-        res["knowledgebases"] = kb_list
+        res["datasets"] = kb_list
         res["avatar"] = res.pop("icon")
         list_assts.append(res)
     return get_result(data=list_assts)

api/apps/sdk/dataset.py

@@ -15,17 +15,17 @@
 #
 
 from flask import request
-
 from api.db import StatusEnum, FileSource
 from api.db.db_models import File
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.llm_service import TenantLLMService
 from api.db.services.user_service import TenantService
 from api.settings import RetCode
 from api.utils import get_uuid
-from api.utils.api_utils import get_result, token_required, get_error_data_result, valid
+from api.utils.api_utils import get_result, token_required, get_error_data_result, valid,get_parser_config
 
 
 @manager.route('/dataset', methods=['POST'])
@@ -36,15 +36,17 @@ def create(tenant_id):
     permission = req.get("permission")
     language = req.get("language")
     chunk_method = req.get("chunk_method")
-    valid_permission = ("me", "team")
-    valid_language =("Chinese", "English")
-    valid_chunk_method = ("naive","manual","qa","table","paper","book","laws","presentation","picture","one","knowledge_graph","email")
+    parser_config = req.get("parser_config")
+    valid_permission = {"me", "team"}
+    valid_language ={"Chinese", "English"}
+    valid_chunk_method = {"naive","manual","qa","table","paper","book","laws","presentation","picture","one","knowledge_graph","email"}
     check_validation=valid(permission,valid_permission,language,valid_language,chunk_method,valid_chunk_method)
     if check_validation:
         return check_validation
-    if "tenant_id" in req or "embedding_model" in req:
+    req["parser_config"]=get_parser_config(chunk_method,parser_config)
+    if "tenant_id" in req:
         return get_error_data_result(
-            retmsg="`tenant_id` or `embedding_model` must not be provided")
+            retmsg="`tenant_id` must not be provided")
     chunk_count=req.get("chunk_count")
     document_count=req.get("document_count")
     if chunk_count or document_count:
@@ -59,9 +61,13 @@ def create(tenant_id):
             retmsg="`name` is not empty string!")
     if KnowledgebaseService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value):
         return get_error_data_result(
-            retmsg="Duplicated knowledgebase name in creating dataset.")
+            retmsg="Duplicated dataset name in creating dataset.")
     req["tenant_id"] = req['created_by'] = tenant_id
-    req['embedding_model'] = t.embd_id
+    if not req.get("embedding_model"):
+        req['embedding_model'] = t.embd_id
+    else:
+        if not TenantLLMService.query(tenant_id=tenant_id,model_type="embedding", llm_name=req.get("embedding_model")):
+            return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
     key_mapping = {
         "chunk_num": "chunk_count",
         "doc_num": "document_count",
@@ -116,10 +122,12 @@ def update(tenant_id,dataset_id):
     permission = req.get("permission")
     language = req.get("language")
     chunk_method = req.get("chunk_method")
-    valid_permission = ("me", "team")
-    valid_language =("Chinese", "English")
-    valid_chunk_method = ("naive","manual","qa","table","paper","book","laws","presentation","picture","one","knowledge_graph","email")
-    check_validation=valid(permission,valid_permission,language,valid_language,chunk_method,valid_chunk_method)
+    parser_config = req.get("parser_config")
+    valid_permission = {"me", "team"}
+    valid_language = {"Chinese", "English"}
+    valid_chunk_method = {"naive", "manual", "qa", "table", "paper", "book", "laws", "presentation", "picture", "one",
+                          "knowledge_graph", "email"}
+    check_validation = valid(permission, valid_permission, language, valid_language, chunk_method, valid_chunk_method)
     if check_validation:
         return check_validation
     if "tenant_id" in req:
@@ -142,18 +150,24 @@ def update(tenant_id,dataset_id):
             return get_error_data_result(
                 retmsg="If `chunk_count` is not 0, `chunk_method` is not changeable.")
         req['parser_id'] = req.pop('chunk_method')
+        if req['parser_id'] != kb.parser_id:
+            req["parser_config"] = get_parser_config(chunk_method, parser_config)
     if "embedding_model" in req:
         if kb.chunk_num != 0 and req['embedding_model'] != kb.embd_id:
             return get_error_data_result(
                 retmsg="If `chunk_count` is not 0, `embedding_method` is not changeable.")
+        if not req.get("embedding_model"):
+            return get_error_data_result("`embedding_model` can't be empty")
+        if not TenantLLMService.query(tenant_id=tenant_id,model_type="embedding", llm_name=req.get("embedding_model")):
+            return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
         req['embd_id'] = req.pop('embedding_model')
     if "name" in req:
         req["name"] = req["name"].strip()
         if req["name"].lower() != kb.name.lower() \
                 and len(KnowledgebaseService.query(name=req["name"], tenant_id=tenant_id,
                                                    status=StatusEnum.VALID.value)) > 0:
             return get_error_data_result(
-                retmsg="Duplicated knowledgebase name in updating dataset.")
+                retmsg="Duplicated dataset name in updating dataset.")
     if not KnowledgebaseService.update_by_id(kb.id, req):
         return get_error_data_result(retmsg="Update dataset error.(Database error)")
     return get_result(retcode=RetCode.SUCCESS)