From cacd21bde78bd441f6852ce148acc57ed9f4fe35 Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Fri, 23 Feb 2024 17:20:57 +0800 Subject: [PATCH 1/7] =?UTF-8?q?refactor:=20=E7=A7=BB=E5=8A=A8=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E5=99=A8=E5=88=B0pipeline=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/core/app.py | 4 ++-- pkg/core/boot.py | 2 +- pkg/{core => pipeline}/controller.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename pkg/{core => pipeline}/controller.py (98%) diff --git a/pkg/core/app.py b/pkg/core/app.py index a31d2f77..df1566ef 100644 --- a/pkg/core/app.py +++ b/pkg/core/app.py @@ -13,8 +13,8 @@ from ..audit.center import v2 as center_mgr from ..command import cmdmgr from ..plugin import manager as plugin_mgr -from . import pool, controller -from ..pipeline import stagemgr +from . import pool +from ..pipeline import controller, stagemgr from ..utils import version as version_mgr, proxy as proxy_mgr diff --git a/pkg/core/boot.py b/pkg/core/boot.py index 3ecfa8ae..33cc83c0 100644 --- a/pkg/core/boot.py +++ b/pkg/core/boot.py @@ -10,7 +10,7 @@ from . import app from . import pool -from . import controller +from ..pipeline import controller from ..pipeline import stagemgr from ..audit import identifier from ..provider.session import sessionmgr as llm_session_mgr diff --git a/pkg/core/controller.py b/pkg/pipeline/controller.py similarity index 98% rename from pkg/core/controller.py rename to pkg/pipeline/controller.py index 42ef435c..a48c664d 100644 --- a/pkg/core/controller.py +++ b/pkg/pipeline/controller.py @@ -4,8 +4,8 @@ import typing import traceback -from . import app, entities -from ..pipeline import entities as pipeline_entities +from ..core import app, entities +from . import entities as pipeline_entities from ..plugin import events From 1f07a8a9e3efe8c992eeebd7fce20b2af4533cab Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Fri, 23 Feb 2024 17:46:22 +0800 Subject: [PATCH 2/7] =?UTF-8?q?refactor:=20=E7=A7=BB=E5=8A=A8pool=E5=88=B0?= =?UTF-8?q?pipeline=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/core/app.py | 2 +- pkg/core/boot.py | 2 +- pkg/{core => pipeline}/pool.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename pkg/{core => pipeline}/pool.py (98%) diff --git a/pkg/core/app.py b/pkg/core/app.py index df1566ef..80f6cc3c 100644 --- a/pkg/core/app.py +++ b/pkg/core/app.py @@ -13,7 +13,7 @@ from ..audit.center import v2 as center_mgr from ..command import cmdmgr from ..plugin import manager as plugin_mgr -from . import pool +from ..pipeline import pool from ..pipeline import controller, stagemgr from ..utils import version as version_mgr, proxy as proxy_mgr diff --git a/pkg/core/boot.py b/pkg/core/boot.py index 33cc83c0..7a5d965d 100644 --- a/pkg/core/boot.py +++ b/pkg/core/boot.py @@ -9,7 +9,7 @@ from .bootutils import config from . import app -from . import pool +from ..pipeline import pool from ..pipeline import controller from ..pipeline import stagemgr from ..audit import identifier diff --git a/pkg/core/pool.py b/pkg/pipeline/pool.py similarity index 98% rename from pkg/core/pool.py rename to pkg/pipeline/pool.py index 5c8000dd..15149296 100644 --- a/pkg/core/pool.py +++ b/pkg/pipeline/pool.py @@ -4,7 +4,7 @@ import mirai -from . import entities +from ..core import entities from ..platform import adapter as msadapter From 71f2a58acb45614e9344ee0c488af215d39ee4e2 Mon Sep 17 00:00:00 2001 From: Junyan Qin <1010553892@qq.com> Date: Thu, 29 Feb 2024 11:10:30 +0000 Subject: [PATCH 3/7] =?UTF-8?q?feat:=20=E4=BE=9D=E8=B5=96=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E7=A7=BB=E5=8A=A8=E5=88=B0main.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 29 +++++++++++++++++++++++++---- pkg/core/boot.py | 11 ----------- pkg/core/bootutils/log.py | 6 +++++- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/main.py b/main.py index 4f3373df..3d97b1e3 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,5 @@ -import asyncio - +# QChatGPT 终端启动入口 +# 在此层级解决依赖项检查。 asciiart = r""" ___ ___ _ _ ___ ___ _____ @@ -11,8 +11,29 @@ 📖文档地址: https://q.rkcn.top """ -if __name__ == '__main__': + +async def main_entry(): print(asciiart) + import sys + + from pkg.core.bootutils import deps + + missing_deps = await deps.check_deps() + + if missing_deps: + print("以下依赖包未安装,将自动安装,请完成后重启程序:") + for dep in missing_deps: + print("-", dep) + await deps.install_deps(missing_deps) + print("已自动安装缺失的依赖包,请重启程序。") + sys.exit(0) + from pkg.core import boot - asyncio.run(boot.main()) + await boot.main() + + +if __name__ == '__main__': + import asyncio + + asyncio.run(main_entry()) diff --git a/pkg/core/boot.py b/pkg/core/boot.py index 7a5d965d..7b4318ca 100644 --- a/pkg/core/boot.py +++ b/pkg/core/boot.py @@ -1,10 +1,8 @@ from __future__ import print_function -import os import sys from .bootutils import files -from .bootutils import deps from .bootutils import log from .bootutils import config @@ -38,15 +36,6 @@ async def make_app() -> app.Application: sys.exit(0) - missing_deps = await deps.check_deps() - - if missing_deps: - print("以下依赖包未安装,将自动安装,请完成后重启程序:") - for dep in missing_deps: - print("-", dep) - await deps.install_deps(missing_deps) - sys.exit(0) - qcg_logger = await log.init_logging() # 生成标识符 diff --git a/pkg/core/bootutils/log.py b/pkg/core/bootutils/log.py index 308ca8c4..d7b6da0a 100644 --- a/pkg/core/bootutils/log.py +++ b/pkg/core/bootutils/log.py @@ -16,6 +16,10 @@ async def init_logging() -> logging.Logger: + # 删除所有现有的logger + for handler in logging.root.handlers[:]: + logging.root.removeHandler(handler) + level = logging.INFO if "DEBUG" in os.environ and os.environ["DEBUG"] in ["true", "1"]: @@ -46,7 +50,7 @@ async def init_logging() -> logging.Logger: qcg_logger.debug("日志初始化完成,日志级别:%s" % level) logging.basicConfig( - level=logging.INFO, # 设置日志输出格式 + level=logging.CRITICAL, # 设置日志输出格式 format="[DEPR][%(asctime)s.%(msecs)03d] %(pathname)s (%(lineno)d) - [%(levelname)s] :\n%(message)s", # 日志输出的格式 # -8表示占位符,让输出左对齐,输出长度都为8位 From 92aa9c17117099fe79a6218bec5fb902a9520d5c Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Sat, 2 Mar 2024 14:57:55 +0800 Subject: [PATCH 4/7] =?UTF-8?q?perf:=20=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E7=94=9F=E6=88=90=E6=AD=A5=E9=AA=A4=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E5=88=B0main.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 15 +++++++++++++++ pkg/core/boot.py | 12 ------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/main.py b/main.py index 3d97b1e3..e4dbef02 100644 --- a/main.py +++ b/main.py @@ -17,6 +17,8 @@ async def main_entry(): import sys + # 检查依赖 + from pkg.core.bootutils import deps missing_deps = await deps.check_deps() @@ -29,6 +31,19 @@ async def main_entry(): print("已自动安装缺失的依赖包,请重启程序。") sys.exit(0) + # 检查配置文件 + + from pkg.core.bootutils import files + + generated_files = await files.generate_files() + + if generated_files: + print("以下文件不存在,已自动生成,请按需修改配置文件后重启:") + for file in generated_files: + print("-", file) + + sys.exit(0) + from pkg.core import boot await boot.main() diff --git a/pkg/core/boot.py b/pkg/core/boot.py index 7b4318ca..c04fa798 100644 --- a/pkg/core/boot.py +++ b/pkg/core/boot.py @@ -21,20 +21,8 @@ from ..audit.center import v2 as center_v2 from ..utils import version, proxy, announce -use_override = False - async def make_app() -> app.Application: - global use_override - - generated_files = await files.generate_files() - - if generated_files: - print("以下文件不存在,已自动生成,请按需修改配置文件后重启:") - for file in generated_files: - print("-", file) - - sys.exit(0) qcg_logger = await log.init_logging() From c89623967e9f2ffd5340ebcea947e04691226bf7 Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Sat, 2 Mar 2024 16:37:30 +0800 Subject: [PATCH 5/7] =?UTF-8?q?refactor:=20=E5=BA=94=E7=94=A8=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E5=8C=96=E6=B5=81=E7=A8=8B=E5=88=9D=E6=AD=A5=E5=88=86?= =?UTF-8?q?=E9=98=B6=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/core/boot.py | 100 +++++--------------------------- pkg/core/stage.py | 30 ++++++++++ pkg/core/stages/__init__.py | 0 pkg/core/stages/build_app.py | 95 ++++++++++++++++++++++++++++++ pkg/core/stages/load_config.py | 19 ++++++ pkg/core/stages/setup_logger.py | 15 +++++ 6 files changed, 173 insertions(+), 86 deletions(-) create mode 100644 pkg/core/stage.py create mode 100644 pkg/core/stages/__init__.py create mode 100644 pkg/core/stages/build_app.py create mode 100644 pkg/core/stages/load_config.py create mode 100644 pkg/core/stages/setup_logger.py diff --git a/pkg/core/boot.py b/pkg/core/boot.py index c04fa798..6a395c7f 100644 --- a/pkg/core/boot.py +++ b/pkg/core/boot.py @@ -21,100 +21,28 @@ from ..audit.center import v2 as center_v2 from ..utils import version, proxy, announce +from .stages import build_app, load_config, setup_logger +from . import stage -async def make_app() -> app.Application: - qcg_logger = await log.init_logging() +stage_order = [ + "LoadConfigStage", + "SetupLoggerStage", + "BuildAppStage" +] - # 生成标识符 - identifier.init() - # ========== 加载配置文件 ========== +async def make_app() -> app.Application: - command_cfg = await config.load_json_config("data/config/command.json", "templates/command.json") - pipeline_cfg = await config.load_json_config("data/config/pipeline.json", "templates/pipeline.json") - platform_cfg = await config.load_json_config("data/config/platform.json", "templates/platform.json") - provider_cfg = await config.load_json_config("data/config/provider.json", "templates/provider.json") - system_cfg = await config.load_json_config("data/config/system.json", "templates/system.json") + # 生成标识符 + identifier.init() - # ========== 构建应用实例 ========== ap = app.Application() - ap.logger = qcg_logger - - ap.command_cfg = command_cfg - ap.pipeline_cfg = pipeline_cfg - ap.platform_cfg = platform_cfg - ap.provider_cfg = provider_cfg - ap.system_cfg = system_cfg - - proxy_mgr = proxy.ProxyManager(ap) - await proxy_mgr.initialize() - ap.proxy_mgr = proxy_mgr - - ver_mgr = version.VersionManager(ap) - await ver_mgr.initialize() - ap.ver_mgr = ver_mgr - - center_v2_api = center_v2.V2CenterAPI( - ap, - basic_info={ - "host_id": identifier.identifier["host_id"], - "instance_id": identifier.identifier["instance_id"], - "semantic_version": ver_mgr.get_current_version(), - "platform": sys.platform, - }, - runtime_info={ - "admin_id": "{}".format(system_cfg.data["admin-sessions"]), - "msg_source": str([ - adapter_cfg['adapter'] if 'adapter' in adapter_cfg else 'unknown' - for adapter_cfg in platform_cfg.data['platform-adapters'] if adapter_cfg['enable'] - ]), - }, - ) - ap.ctr_mgr = center_v2_api - - # 发送公告 - ann_mgr = announce.AnnouncementManager(ap) - await ann_mgr.show_announcements() - - ap.query_pool = pool.QueryPool() - - await ap.ver_mgr.show_version_update() - - plugin_mgr_inst = plugin_mgr.PluginManager(ap) - await plugin_mgr_inst.initialize() - ap.plugin_mgr = plugin_mgr_inst - - cmd_mgr_inst = cmdmgr.CommandManager(ap) - await cmd_mgr_inst.initialize() - ap.cmd_mgr = cmd_mgr_inst - - llm_model_mgr_inst = llm_model_mgr.ModelManager(ap) - await llm_model_mgr_inst.initialize() - ap.model_mgr = llm_model_mgr_inst - - llm_session_mgr_inst = llm_session_mgr.SessionManager(ap) - await llm_session_mgr_inst.initialize() - ap.sess_mgr = llm_session_mgr_inst - - llm_prompt_mgr_inst = llm_prompt_mgr.PromptManager(ap) - await llm_prompt_mgr_inst.initialize() - ap.prompt_mgr = llm_prompt_mgr_inst - - llm_tool_mgr_inst = llm_tool_mgr.ToolManager(ap) - await llm_tool_mgr_inst.initialize() - ap.tool_mgr = llm_tool_mgr_inst - - im_mgr_inst = im_mgr.PlatformManager(ap=ap) - await im_mgr_inst.initialize() - ap.im_mgr = im_mgr_inst - - stage_mgr = stagemgr.StageManager(ap) - await stage_mgr.initialize() - ap.stage_mgr = stage_mgr - ctrl = controller.Controller(ap) - ap.ctrl = ctrl + for stage_name in stage_order: + stage_cls = stage.preregistered_stages[stage_name] + stage_inst = stage_cls() + await stage_inst.run(ap) await ap.initialize() diff --git a/pkg/core/stage.py b/pkg/core/stage.py new file mode 100644 index 00000000..cb2e4868 --- /dev/null +++ b/pkg/core/stage.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import abc +import typing + +from . import app + + +preregistered_stages: list[typing.Type[BootingStage]] = {} + +def stage_class( + name: str +): + def decorator(cls: typing.Type[BootingStage]) -> typing.Type[BootingStage]: + preregistered_stages[name] = cls + return cls + + return decorator + + +class BootingStage(abc.ABC): + """启动阶段 + """ + name: str = None + + @abc.abstractmethod + async def run(self, ap: app.Application): + """启动 + """ + pass diff --git a/pkg/core/stages/__init__.py b/pkg/core/stages/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pkg/core/stages/build_app.py b/pkg/core/stages/build_app.py new file mode 100644 index 00000000..7a840b86 --- /dev/null +++ b/pkg/core/stages/build_app.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import sys + +from .. import stage, app +from ...utils import version, proxy, announce +from ...audit.center import v2 as center_v2 +from ...audit import identifier +from ...pipeline import pool, controller, stagemgr +from ...plugin import manager as plugin_mgr +from ...command import cmdmgr +from ...provider.session import sessionmgr as llm_session_mgr +from ...provider.requester import modelmgr as llm_model_mgr +from ...provider.sysprompt import sysprompt as llm_prompt_mgr +from ...provider.tools import toolmgr as llm_tool_mgr +from ...platform import manager as im_mgr + + +@stage.stage_class("BuildAppStage") +class BuildAppStage(stage.BootingStage): + """构建应用阶段 + """ + + async def run(self, ap: app.Application): + """启动 + """ + + proxy_mgr = proxy.ProxyManager(ap) + await proxy_mgr.initialize() + ap.proxy_mgr = proxy_mgr + + ver_mgr = version.VersionManager(ap) + await ver_mgr.initialize() + ap.ver_mgr = ver_mgr + + center_v2_api = center_v2.V2CenterAPI( + ap, + basic_info={ + "host_id": identifier.identifier["host_id"], + "instance_id": identifier.identifier["instance_id"], + "semantic_version": ver_mgr.get_current_version(), + "platform": sys.platform, + }, + runtime_info={ + "admin_id": "{}".format(ap.system_cfg.data["admin-sessions"]), + "msg_source": str([ + adapter_cfg['adapter'] if 'adapter' in adapter_cfg else 'unknown' + for adapter_cfg in ap.platform_cfg.data['platform-adapters'] if adapter_cfg['enable'] + ]), + }, + ) + ap.ctr_mgr = center_v2_api + + # 发送公告 + ann_mgr = announce.AnnouncementManager(ap) + await ann_mgr.show_announcements() + + ap.query_pool = pool.QueryPool() + + await ap.ver_mgr.show_version_update() + + plugin_mgr_inst = plugin_mgr.PluginManager(ap) + await plugin_mgr_inst.initialize() + ap.plugin_mgr = plugin_mgr_inst + + cmd_mgr_inst = cmdmgr.CommandManager(ap) + await cmd_mgr_inst.initialize() + ap.cmd_mgr = cmd_mgr_inst + + llm_model_mgr_inst = llm_model_mgr.ModelManager(ap) + await llm_model_mgr_inst.initialize() + ap.model_mgr = llm_model_mgr_inst + + llm_session_mgr_inst = llm_session_mgr.SessionManager(ap) + await llm_session_mgr_inst.initialize() + ap.sess_mgr = llm_session_mgr_inst + + llm_prompt_mgr_inst = llm_prompt_mgr.PromptManager(ap) + await llm_prompt_mgr_inst.initialize() + ap.prompt_mgr = llm_prompt_mgr_inst + + llm_tool_mgr_inst = llm_tool_mgr.ToolManager(ap) + await llm_tool_mgr_inst.initialize() + ap.tool_mgr = llm_tool_mgr_inst + + im_mgr_inst = im_mgr.PlatformManager(ap=ap) + await im_mgr_inst.initialize() + ap.im_mgr = im_mgr_inst + + stage_mgr = stagemgr.StageManager(ap) + await stage_mgr.initialize() + ap.stage_mgr = stage_mgr + + ctrl = controller.Controller(ap) + ap.ctrl = ctrl diff --git a/pkg/core/stages/load_config.py b/pkg/core/stages/load_config.py new file mode 100644 index 00000000..0ee2c1e4 --- /dev/null +++ b/pkg/core/stages/load_config.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from .. import stage, app +from ..bootutils import config + + +@stage.stage_class("LoadConfigStage") +class LoadConfigStage(stage.BootingStage): + """加载配置文件阶段 + """ + + async def run(self, ap: app.Application): + """启动 + """ + ap.command_cfg = await config.load_json_config("data/config/command.json", "templates/command.json") + ap.pipeline_cfg = await config.load_json_config("data/config/pipeline.json", "templates/pipeline.json") + ap.platform_cfg = await config.load_json_config("data/config/platform.json", "templates/platform.json") + ap.provider_cfg = await config.load_json_config("data/config/provider.json", "templates/provider.json") + ap.system_cfg = await config.load_json_config("data/config/system.json", "templates/system.json") diff --git a/pkg/core/stages/setup_logger.py b/pkg/core/stages/setup_logger.py new file mode 100644 index 00000000..02446b85 --- /dev/null +++ b/pkg/core/stages/setup_logger.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from .. import stage, app +from ..bootutils import log + + +@stage.stage_class("SetupLoggerStage") +class SetupLoggerStage(stage.BootingStage): + """设置日志器阶段 + """ + + async def run(self, ap: app.Application): + """启动 + """ + ap.logger = await log.init_logging() From cac8a0a4147707095d5d72233dc51e85e6058ace Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Sat, 2 Mar 2024 16:39:29 +0800 Subject: [PATCH 6/7] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E5=AF=BC?= =?UTF-8?q?=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/core/boot.py | 20 -------------------- pkg/core/stage.py | 2 +- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/pkg/core/boot.py b/pkg/core/boot.py index 6a395c7f..2e5a2b6c 100644 --- a/pkg/core/boot.py +++ b/pkg/core/boot.py @@ -1,27 +1,7 @@ from __future__ import print_function -import sys - -from .bootutils import files -from .bootutils import log -from .bootutils import config - from . import app -from ..pipeline import pool -from ..pipeline import controller -from ..pipeline import stagemgr from ..audit import identifier -from ..provider.session import sessionmgr as llm_session_mgr -from ..provider.requester import modelmgr as llm_model_mgr -from ..provider.sysprompt import sysprompt as llm_prompt_mgr -from ..provider.tools import toolmgr as llm_tool_mgr -from ..platform import manager as im_mgr -from ..command import cmdmgr -from ..plugin import manager as plugin_mgr -from ..audit.center import v2 as center_v2 -from ..utils import version, proxy, announce - -from .stages import build_app, load_config, setup_logger from . import stage diff --git a/pkg/core/stage.py b/pkg/core/stage.py index cb2e4868..cb8e90eb 100644 --- a/pkg/core/stage.py +++ b/pkg/core/stage.py @@ -6,7 +6,7 @@ from . import app -preregistered_stages: list[typing.Type[BootingStage]] = {} +preregistered_stages: dict[str, typing.Type[BootingStage]] = {} def stage_class( name: str From 8d375a02dbbd87f4dc36972cedd001bdbbf54df7 Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Sat, 2 Mar 2024 20:05:23 +0800 Subject: [PATCH 7/7] =?UTF-8?q?fix:=20=E6=9C=AA=E5=AF=BC=E5=85=A5=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/core/boot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/core/boot.py b/pkg/core/boot.py index 2e5a2b6c..c5790306 100644 --- a/pkg/core/boot.py +++ b/pkg/core/boot.py @@ -3,6 +3,7 @@ from . import app from ..audit import identifier from . import stage +from .stages import load_config, setup_logger, build_app stage_order = [