add API for tenant function (#2866)

### What problem does this PR solve?

feat: API access key management
https://github.com/infiniflow/ragflow/issues/2846
feat: Render markdown file with remark-loader
https://github.com/infiniflow/ragflow/issues/2846

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

api/apps/canvas_app.py

@@ -18,8 +18,6 @@ from functools import partial
 from flask import request, Response
 from flask_login import login_required, current_user
 from api.db.services.canvas_service import CanvasTemplateService, UserCanvasService
-from api.db.services.dialog_service import full_question
-from api.db.services.user_service import TenantService
 from api.settings import RetCode
 from api.utils import get_uuid
 from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result
@@ -111,8 +109,9 @@ def run():
         if "message" in req:
             canvas.messages.append({"role": "user", "content": req["message"], "id": message_id})
             if len([m for m in canvas.messages if m["role"] == "user"]) > 1:
-                ten = TenantService.get_by_user_id(current_user.id)[0]
+                #ten = TenantService.get_info_by(current_user.id)[0]
                 #req["message"] = full_question(ten["tenant_id"], ten["llm_id"], canvas.messages)
+                pass
             canvas.add_user_input(req["message"])
         answer = canvas.run(stream=stream)
         print(canvas)

api/apps/conversation_app.py

@@ -218,7 +218,7 @@ def tts():
     req = request.json
     text = req["text"]
 
-    tenants = TenantService.get_by_user_id(current_user.id)
+    tenants = TenantService.get_info_by(current_user.id)
     if not tenants:
         return get_data_error_result(retmsg="Tenant not found!")
 

api/apps/system_app.py

@@ -133,15 +133,9 @@ def token_list():
         return server_error_response(e)
 
 
-@manager.route('/rm', methods=['POST'])
-@validate_request("tokens", "tenant_id")
+@manager.route('/token/<token>', methods=['DELETE'])
 @login_required
-def rm():
-    req = request.json
-    try:
-        for token in req["tokens"]:
-            APITokenService.filter_delete(
-                [APIToken.tenant_id == req["tenant_id"], APIToken.token == token])
-        return get_json_result(data=True)
-    except Exception as e:
-        return server_error_response(e)
+def rm(token):
+    APITokenService.filter_delete(
+                [APIToken.tenant_id == current_user.id, APIToken.token == token])
+    return get_json_result(data=True)

api/apps/tenant_app.py

@@ -15,25 +15,14 @@
 #
 
 from flask import request
-from flask_login import current_user, login_required
+from flask_login import login_required, current_user
 
 from api.db import UserTenantRole, StatusEnum
 from api.db.db_models import UserTenant
-from api.db.services.user_service import TenantService, UserTenantService
-from api.settings import RetCode
+from api.db.services.user_service import UserTenantService, UserService
 
-from api.utils import get_uuid
-from api.utils.api_utils import get_json_result, validate_request, server_error_response
-
-
-@manager.route("/list", methods=["GET"])
-@login_required
-def tenant_list():
-    try:
-        tenants = TenantService.get_by_user_id(current_user.id)
-        return get_json_result(data=tenants)
-    except Exception as e:
-        return server_error_response(e)
+from api.utils import get_uuid, delta_seconds
+from api.utils.api_utils import get_json_result, validate_request, server_error_response, get_data_error_result
 
 
 @manager.route("/<tenant_id>/user/list", methods=["GET"])
@@ -41,6 +30,8 @@ def tenant_list():
 def user_list(tenant_id):
     try:
         users = UserTenantService.get_by_tenant_id(tenant_id)
+        for u in users:
+            u["delta_seconds"] = delta_seconds(u["update_date"])
         return get_json_result(data=users)
     except Exception as e:
         return server_error_response(e)
@@ -48,30 +39,31 @@ def user_list(tenant_id):
 
 @manager.route('/<tenant_id>/user', methods=['POST'])
 @login_required
-@validate_request("user_id")
+@validate_request("email")
 def create(tenant_id):
-    user_id = request.json.get("user_id")
-    if not user_id:
-        return get_json_result(
-            data=False, retmsg='Lack of "USER ID"', retcode=RetCode.ARGUMENT_ERROR)
+    req = request.json
+    usrs = UserService.query(email=req["email"])
+    if not usrs:
+        return get_data_error_result(retmsg="User not found.")
 
-    try:
-        user_tenants = UserTenantService.query(user_id=user_id, tenant_id=tenant_id)
-        if user_tenants:
-            uuid = user_tenants[0].id
-            return get_json_result(data={"id": uuid})
-
-        uuid = get_uuid()
-        UserTenantService.save(
-            id = uuid,
-            user_id = user_id,
-            tenant_id = tenant_id,
-            role = UserTenantRole.NORMAL.value,
-            status = StatusEnum.VALID.value)
-
-        return get_json_result(data={"id": uuid})
-    except Exception as e:
-        return server_error_response(e)
+    user_id = usrs[0].id
+    user_tenants = UserTenantService.query(user_id=user_id, tenant_id=tenant_id)
+    if user_tenants:
+        if user_tenants[0].status == UserTenantRole.NORMAL.value:
+            return get_data_error_result(retmsg="This user is in the team already.")
+        return get_data_error_result(retmsg="Invitation notification is sent.")
+
+    UserTenantService.save(
+        id=get_uuid(),
+        user_id=user_id,
+        tenant_id=tenant_id,
+        role=UserTenantRole.INVITE,
+        status=StatusEnum.VALID.value)
+
+    usr = list(usrs.dicts())[0]
+    usr = {k: v for k, v in usr.items() if k in ["id", "avatar", "email", "nickname"]}
+
+    return get_json_result(data=usr)
 
 
 @manager.route('/<tenant_id>/user/<user_id>', methods=['DELETE'])
@@ -82,4 +74,25 @@ def rm(tenant_id, user_id):
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
-    
+
+
+@manager.route("/list", methods=["GET"])
+@login_required
+def tenant_list():
+    try:
+        users = UserTenantService.get_tenants_by_user_id(current_user.id)
+        for u in users:
+            u["delta_seconds"] = delta_seconds(u["update_date"])
+        return get_json_result(data=users)
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/agree/<tenant_id>", methods=["GET"])
+@login_required
+def agree(tenant_id):
+    try:
+        UserTenantService.filter_update([UserTenant.tenant_id == tenant_id, UserTenant.user_id == current_user.id], {"role": UserTenantRole.NORMAL})
+        return get_json_result(data=True)
+    except Exception as e:
+        return server_error_response(e)

api/apps/user_app.py

@@ -23,7 +23,7 @@ from flask_login import login_required, current_user, login_user, logout_user
 
 from api.db.db_models import TenantLLM
 from api.db.services.llm_service import TenantLLMService, LLMService
-from api.utils.api_utils import server_error_response, validate_request
+from api.utils.api_utils import server_error_response, validate_request, get_data_error_result
 from api.utils import get_uuid, get_format_time, decrypt, download_img, current_timestamp, datetime_format
 from api.db import UserTenantRole, LLMType, FileType
 from api.settings import RetCode, GITHUB_OAUTH, FEISHU_OAUTH, CHAT_MDL, EMBEDDING_MDL, ASR_MDL, IMAGE2TEXT_MDL, PARSERS, \
@@ -402,8 +402,10 @@ def user_add():
 @login_required
 def tenant_info():
     try:
-        tenants = TenantService.get_by_user_id(current_user.id)[0]
-        return get_json_result(data=tenants)
+        tenants = TenantService.get_info_by(current_user.id)
+        if not tenants:
+            return get_data_error_result(retmsg="Tenant not found!")
+        return get_json_result(data=tenants[0])
     except Exception as e:
         return server_error_response(e)
 

api/db/__init__.py

@@ -27,6 +27,7 @@ class UserTenantRole(StrEnum):
     OWNER = 'owner'
     ADMIN = 'admin'
     NORMAL = 'normal'
+    INVITE = 'invite'
 
 
 class TenantPermission(StrEnum):

api/db/services/document_service.py

@@ -362,7 +362,7 @@ class DocumentService(CommonService):
                 elif finished:
                     if d["parser_config"].get("raptor", {}).get("use_raptor") and d["progress_msg"].lower().find(" raptor")<0:
                         queue_raptor_tasks(d)
-                        prg *= 0.98
+                        prg = 0.98 * len(tsks)/(len(tsks)+1)
                         msg.append("------ RAPTOR -------")
                     else:
                         status = TaskStatus.DONE.value
@@ -379,7 +379,8 @@ class DocumentService(CommonService):
                     info["progress_msg"] = msg
                 cls.update_by_id(d["id"], info)
             except Exception as e:
-                stat_logger.error("fetch task exception:" + str(e))
+                if str(e).find("'0'") < 0:
+                    stat_logger.error("fetch task exception:" + str(e))
 
     @classmethod
     @DB.connection_context()

api/db/services/user_service.py

@@ -87,7 +87,7 @@ class TenantService(CommonService):
 
     @classmethod
     @DB.connection_context()
-    def get_by_user_id(cls, user_id):
+    def get_info_by(cls, user_id):
         fields = [
             cls.model.id.alias("tenant_id"),
             cls.model.name,
@@ -100,7 +100,7 @@ class TenantService(CommonService):
             cls.model.parser_ids,
             UserTenant.role]
         return list(cls.model.select(*fields)
-                    .join(UserTenant, on=((cls.model.id == UserTenant.tenant_id) & (UserTenant.user_id == user_id) & (UserTenant.status == StatusEnum.VALID.value)))
+                    .join(UserTenant, on=((cls.model.id == UserTenant.tenant_id) & (UserTenant.user_id == user_id) & (UserTenant.status == StatusEnum.VALID.value) & (UserTenant.role == UserTenantRole.OWNER)))
                     .where(cls.model.status == StatusEnum.VALID.value).dicts())
 
     @classmethod
@@ -115,7 +115,7 @@ class TenantService(CommonService):
             cls.model.img2txt_id,
             UserTenant.role]
         return list(cls.model.select(*fields)
-                    .join(UserTenant, on=((cls.model.id == UserTenant.tenant_id) & (UserTenant.user_id == user_id) & (UserTenant.status == StatusEnum.VALID.value) & (UserTenant.role == UserTenantRole.NORMAL.value)))
+                    .join(UserTenant, on=((cls.model.id == UserTenant.tenant_id) & (UserTenant.user_id == user_id) & (UserTenant.status == StatusEnum.VALID.value) & (UserTenant.role == UserTenantRole.NORMAL)))
                     .where(cls.model.status == StatusEnum.VALID.value).dicts())
 
     @classmethod
@@ -143,9 +143,8 @@ class UserTenantService(CommonService):
     def get_by_tenant_id(cls, tenant_id):
         fields = [
             cls.model.user_id,
-            cls.model.tenant_id,
-            cls.model.role,
             cls.model.status,
+            cls.model.role,
             User.nickname,
             User.email,
             User.avatar,
@@ -153,8 +152,24 @@ class UserTenantService(CommonService):
             User.is_active,
             User.is_anonymous,
             User.status,
+            User.update_date,
             User.is_superuser]
         return list(cls.model.select(*fields)
-                    .join(User, on=((cls.model.user_id == User.id) & (cls.model.status == StatusEnum.VALID.value)))
+                    .join(User, on=((cls.model.user_id == User.id) & (cls.model.status == StatusEnum.VALID.value) & (cls.model.role != UserTenantRole.OWNER)))
                     .where(cls.model.tenant_id == tenant_id)
-                    .dicts())
+                    .dicts())
+
+    @classmethod
+    @DB.connection_context()
+    def get_tenants_by_user_id(cls, user_id):
+        fields = [
+            cls.model.tenant_id,
+            cls.model.role,
+            User.nickname,
+            User.email,
+            User.avatar,
+            User.update_date
+        ]
+        return list(cls.model.select(*fields)
+                    .join(User, on=((cls.model.tenant_id == User.id) & (UserTenant.user_id == user_id) & (UserTenant.status == StatusEnum.VALID.value)))
+                    .where(cls.model.status == StatusEnum.VALID.value).dicts())

api/ragflow_server.py

@@ -38,7 +38,7 @@ from api.versions import get_versions
 
 def update_progress():
     while True:
-        time.sleep(1)
+        time.sleep(3)
         try:
             DocumentService.update_progress()
         except Exception as e:

api/utils/__init__.py

@@ -344,3 +344,8 @@ def download_img(url):
     return "data:" + \
            response.headers.get('Content-Type', 'image/jpg') + ";" + \
            "base64," + base64.b64encode(response.content).decode("utf-8")
+
+
+def delta_seconds(date_string: str):
+    dt = datetime.datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S")
+    return (datetime.datetime.now() - dt).total_seconds()

rag/raptor.py

@@ -49,6 +49,7 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:
         layers = [(0, len(chunks))]
         start, end = 0, len(chunks)
         if len(chunks) <= 1: return
+        chunks = [(s, a) for s, a in chunks if len(a) > 0]
 
         def summarize(ck_idx, lock):
             nonlocal chunks
@@ -64,6 +65,7 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:
                 print("SUM:", cnt)
                 embds, _ = self._embd_model.encode([cnt])
                 with lock:
+                    if not len(embds[0]): return
                     chunks.append((cnt, embds[0]))
             except Exception as e:
                 print(e, flush=True)