From b6fa3f9a38119835874cea7377c10115094d755d Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 03:58:33 +0000 Subject: [PATCH 01/12] Update mkdocs --- pipelines/.readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/.readthedocs.yaml b/pipelines/.readthedocs.yaml index e7e5e27fc3cf..f7453c9a4eb9 100644 --- a/pipelines/.readthedocs.yaml +++ b/pipelines/.readthedocs.yaml @@ -17,4 +17,4 @@ mkdocs: # Optionally declare the Python requirements required to build your docs python: install: - - requirements: docs/requirements.txt \ No newline at end of file + - requirements: pipelines/docs/requirements.txt \ No newline at end of file From d14ea39cc6f531c99872b78e952c72dc1f477553 Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 06:57:31 +0000 Subject: [PATCH 02/12] Update mkdocs configuration --- pipelines/.readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/.readthedocs.yaml b/pipelines/.readthedocs.yaml index f7453c9a4eb9..a43de5826895 100644 --- a/pipelines/.readthedocs.yaml +++ b/pipelines/.readthedocs.yaml @@ -12,7 +12,7 @@ build: python: "3.12" mkdocs: - configuration: mkdocs.yml + configuration: pipelines/mkdocs.yml # Optionally declare the Python requirements required to build your docs python: From c38c312acf26ea89b52f022fa1407b09b453fd02 Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 07:01:24 +0000 Subject: [PATCH 03/12] migrate paths --- pipelines/mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/mkdocs.yml b/pipelines/mkdocs.yml index 8037d9775069..6aa7e381fd6c 100644 --- a/pipelines/mkdocs.yml +++ b/pipelines/mkdocs.yml @@ -85,7 +85,7 @@ plugins: show_bases: false show_root_heading: true docstring_style: sphinx - paths: [] + paths: [pipelines] selection: filters: - "^__init__$" # but always include __init__ modules and methods From 0316fa61902a2ba64cb060740d364d2b332d2fa6 Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 07:24:34 +0000 Subject: [PATCH 04/12] change root path --- pipelines/mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/mkdocs.yml b/pipelines/mkdocs.yml index 6aa7e381fd6c..df7e9af0a261 100644 --- a/pipelines/mkdocs.yml +++ b/pipelines/mkdocs.yml @@ -85,7 +85,7 @@ plugins: show_bases: false show_root_heading: true docstring_style: sphinx - paths: [pipelines] + paths: [pipelines/pipelines] selection: filters: - "^__init__$" # but always include __init__ modules and methods From e8e3e8efad6a3762e62d18a324e954045d360b2c Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 07:42:54 +0000 Subject: [PATCH 05/12] try to fix --- pipelines/mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/mkdocs.yml b/pipelines/mkdocs.yml index df7e9af0a261..df70799673ae 100644 --- a/pipelines/mkdocs.yml +++ b/pipelines/mkdocs.yml @@ -3,7 +3,7 @@ repo_url: https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines repo_name: PaddleNLP-Pipelines site_url: https://paddlenlp-pipelines.readthedocs.io/zh-cn/latest/ site_description: provides an easy-to-use paddlenlp application tool suite. - +docs_dir: pipelines/docs nav: - Pipelines module: - Pipeline: 'package/pipelines/standard_pipelines.md' From 6045cbda9e8f931890a6dfe2774a877f8cbc4a4f Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 07:45:47 +0000 Subject: [PATCH 06/12] revert --- pipelines/mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/mkdocs.yml b/pipelines/mkdocs.yml index df70799673ae..df7e9af0a261 100644 --- a/pipelines/mkdocs.yml +++ b/pipelines/mkdocs.yml @@ -3,7 +3,7 @@ repo_url: https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines repo_name: PaddleNLP-Pipelines site_url: https://paddlenlp-pipelines.readthedocs.io/zh-cn/latest/ site_description: provides an easy-to-use paddlenlp application tool suite. -docs_dir: pipelines/docs + nav: - Pipelines module: - Pipeline: 'package/pipelines/standard_pipelines.md' From 1c69a421902769ba9e83e4d9fdb48c26fd30a48a Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 07:49:31 +0000 Subject: [PATCH 07/12] Update memory.md --- pipelines/docs/package/agents/memory.md | 15 +++++++++++++++ pipelines/mkdocs.yml | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 pipelines/docs/package/agents/memory.md diff --git a/pipelines/docs/package/agents/memory.md b/pipelines/docs/package/agents/memory.md new file mode 100644 index 000000000000..0a4cf335e22b --- /dev/null +++ b/pipelines/docs/package/agents/memory.md @@ -0,0 +1,15 @@ +# Memory Module + +::: pipelines.agents.memory.conversation_memory + options: + summary: true + separate_signature: true + show_signature_annotations: true + line_length: 60 + +::: pipelines.agents.memory.no_memory + options: + summary: true + separate_signature: true + show_signature_annotations: true + line_length: 60 diff --git a/pipelines/mkdocs.yml b/pipelines/mkdocs.yml index df7e9af0a261..e540d05c806b 100644 --- a/pipelines/mkdocs.yml +++ b/pipelines/mkdocs.yml @@ -29,7 +29,7 @@ nav: - milvus2: 'package/docstore/milvus2.md' - Agents module: - - agent: 'package/agents/agents.md' + - agent: 'package/g/agents.md' - memory: 'package/agents/memory.md' - Server module: From d20e4a896350daec358602562becbe7faab1829a Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 08:05:43 +0000 Subject: [PATCH 08/12] restore agent.md path --- pipelines/mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/mkdocs.yml b/pipelines/mkdocs.yml index e540d05c806b..df7e9af0a261 100644 --- a/pipelines/mkdocs.yml +++ b/pipelines/mkdocs.yml @@ -29,7 +29,7 @@ nav: - milvus2: 'package/docstore/milvus2.md' - Agents module: - - agent: 'package/g/agents.md' + - agent: 'package/agents/agents.md' - memory: 'package/agents/memory.md' - Server module: From 0a462900439e2dbc2d701ac2d75770585836a805 Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 08:20:13 +0000 Subject: [PATCH 09/12] fix path error --- pipelines/docs/package/agents/agents.md | 4 ++-- pipelines/docs/package/agents/memory.md | 4 ++-- pipelines/docs/package/docstore/elasticsearch.md | 2 +- pipelines/docs/package/docstore/faiss.md | 2 +- pipelines/docs/package/docstore/milvus2.md | 2 +- pipelines/docs/package/nodes/answer_extractor.md | 8 ++++---- .../docs/package/nodes/document_intelligence.md | 4 ++-- pipelines/docs/package/nodes/file_converter.md | 10 +++++----- pipelines/docs/package/nodes/llm.md | 8 ++++---- pipelines/docs/package/nodes/other.md | 8 ++++---- pipelines/docs/package/nodes/preprocessor.md | 4 ++-- pipelines/docs/package/nodes/question_generator.md | 2 +- pipelines/docs/package/nodes/ranker.md | 2 +- pipelines/docs/package/nodes/reader.md | 2 +- pipelines/docs/package/nodes/retriever.md | 14 +++++++------- pipelines/docs/package/nodes/search_engine.md | 4 ++-- pipelines/docs/package/nodes/sentiment_analysis.md | 6 +++--- .../docs/package/pipelines/standard_pipelines.md | 2 +- pipelines/docs/package/rest_api/application.md | 2 +- pipelines/docs/package/rest_api/controller.md | 10 +++++----- 20 files changed, 50 insertions(+), 50 deletions(-) diff --git a/pipelines/docs/package/agents/agents.md b/pipelines/docs/package/agents/agents.md index 8ebc8c3c5f6d..9abca38b8d83 100644 --- a/pipelines/docs/package/agents/agents.md +++ b/pipelines/docs/package/agents/agents.md @@ -1,6 +1,6 @@ # Agent Module -::: pipelines.agents.base +::: pipelines.pipelines.agents.base options: summary: true separate_signature: true @@ -12,7 +12,7 @@ - Tool -::: pipelines.agents.agent_step +::: pipelines.pipelines.agents.agent_step options: summary: true separate_signature: true diff --git a/pipelines/docs/package/agents/memory.md b/pipelines/docs/package/agents/memory.md index 0a4cf335e22b..499467862122 100644 --- a/pipelines/docs/package/agents/memory.md +++ b/pipelines/docs/package/agents/memory.md @@ -1,13 +1,13 @@ # Memory Module -::: pipelines.agents.memory.conversation_memory +::: pipelines.pipelines.agents.memory.conversation_memory options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.agents.memory.no_memory +::: pipelines.pipelines.agents.memory.no_memory options: summary: true separate_signature: true diff --git a/pipelines/docs/package/docstore/elasticsearch.md b/pipelines/docs/package/docstore/elasticsearch.md index 67af4bad8052..780f1be1d901 100644 --- a/pipelines/docs/package/docstore/elasticsearch.md +++ b/pipelines/docs/package/docstore/elasticsearch.md @@ -1,6 +1,6 @@ # Elasticsearch Module -::: pipelines.document_stores.elasticsearch +::: pipelines.pipelines.document_stores.elasticsearch options: summary: true separate_signature: true diff --git a/pipelines/docs/package/docstore/faiss.md b/pipelines/docs/package/docstore/faiss.md index 5b70ea088f92..f740e1ce0205 100644 --- a/pipelines/docs/package/docstore/faiss.md +++ b/pipelines/docs/package/docstore/faiss.md @@ -1,6 +1,6 @@ # FAISS Module -::: pipelines.document_stores.faiss +::: pipelines.pipelines.document_stores.faiss options: summary: true separate_signature: true diff --git a/pipelines/docs/package/docstore/milvus2.md b/pipelines/docs/package/docstore/milvus2.md index 767c3fed1cda..b1c54ad0c650 100644 --- a/pipelines/docs/package/docstore/milvus2.md +++ b/pipelines/docs/package/docstore/milvus2.md @@ -1,6 +1,6 @@ # Milvus2 Module -::: pipelines.document_stores.milvus2 +::: pipelines.pipelines.document_stores.milvus2 options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/answer_extractor.md b/pipelines/docs/package/nodes/answer_extractor.md index 391501dda858..ea931aba0c3c 100644 --- a/pipelines/docs/package/nodes/answer_extractor.md +++ b/pipelines/docs/package/nodes/answer_extractor.md @@ -1,27 +1,27 @@ # Answer Extractor Module -::: pipelines.nodes.answer_extractor.answer_extractor_preprocessor +::: pipelines.pipelines.nodes.answer_extractor.answer_extractor_preprocessor options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.answer_extractor.answer_extractor +::: pipelines.pipelines.nodes.answer_extractor.answer_extractor options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.answer_extractor.qa_filter_postprocessor +::: pipelines.pipelines.nodes.answer_extractor.qa_filter_postprocessor options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.answer_extractor.qa_filter +::: pipelines.pipelines.nodes.answer_extractor.qa_filter options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/document_intelligence.md b/pipelines/docs/package/nodes/document_intelligence.md index 9c2003bc8322..cf4d4af37619 100644 --- a/pipelines/docs/package/nodes/document_intelligence.md +++ b/pipelines/docs/package/nodes/document_intelligence.md @@ -1,13 +1,13 @@ # Document Intelligence Module -::: pipelines.nodes.document.document_intelligence +::: pipelines.pipelines.nodes.document.document_intelligence options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.document.document_preprocessor +::: pipelines.pipelines.nodes.document.document_preprocessor options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/file_converter.md b/pipelines/docs/package/nodes/file_converter.md index 230c75ac4abd..7b58685b630b 100644 --- a/pipelines/docs/package/nodes/file_converter.md +++ b/pipelines/docs/package/nodes/file_converter.md @@ -1,27 +1,27 @@ # File Converter Module -::: pipelines.nodes.file_converter.docx +::: pipelines.pipelines.nodes.file_converter.docx options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.file_converter.image +::: pipelines.pipelines.nodes.file_converter.image options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.file_converter.markdown +::: pipelines.pipelines.nodes.file_converter.markdown options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.file_converter.pdf +::: pipelines.pipelines.nodes.file_converter.pdf options: summary: true separate_signature: true @@ -29,7 +29,7 @@ line_length: 60 -::: pipelines.nodes.file_converter.txt +::: pipelines.pipelines.nodes.file_converter.txt options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/llm.md b/pipelines/docs/package/nodes/llm.md index 739da3cd5873..cd00da89a3e9 100644 --- a/pipelines/docs/package/nodes/llm.md +++ b/pipelines/docs/package/nodes/llm.md @@ -1,6 +1,6 @@ # LLM Module -::: pipelines.nodes.llm.chatglm +::: pipelines.pipelines.nodes.llm.chatglm options: summary: true separate_signature: true @@ -8,7 +8,7 @@ line_length: 60 -::: pipelines.nodes.llm.ernie_bot +::: pipelines.pipelines.nodes.llm.ernie_bot options: summary: true separate_signature: true @@ -16,7 +16,7 @@ line_length: 60 -::: pipelines.nodes.llm.history +::: pipelines.pipelines.nodes.llm.history options: summary: true separate_signature: true @@ -24,7 +24,7 @@ line_length: 60 -::: pipelines.nodes.llm.prompt_template +::: pipelines.pipelines.nodes.llm.prompt_template options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/other.md b/pipelines/docs/package/nodes/other.md index 32e7163613bb..b304e6855ef1 100644 --- a/pipelines/docs/package/nodes/other.md +++ b/pipelines/docs/package/nodes/other.md @@ -1,20 +1,20 @@ # Other Module -::: pipelines.nodes.other.docs2answers +::: pipelines.pipelines.nodes.other.docs2answers options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.other.join_answers +::: pipelines.pipelines.nodes.other.join_answers options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.other.join_docs +::: pipelines.pipelines.nodes.other.join_docs options: summary: true separate_signature: true @@ -22,7 +22,7 @@ line_length: 60 -::: pipelines.nodes.other.route_documents +::: pipelines.pipelines.nodes.other.route_documents options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/preprocessor.md b/pipelines/docs/package/nodes/preprocessor.md index af19b708df19..6231b64797b8 100644 --- a/pipelines/docs/package/nodes/preprocessor.md +++ b/pipelines/docs/package/nodes/preprocessor.md @@ -1,13 +1,13 @@ # Preprocesssor Module -::: pipelines.nodes.preprocessor.preprocessor +::: pipelines.pipelines.nodes.preprocessor.preprocessor options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.preprocessor.text_splitter +::: pipelines.pipelines.nodes.preprocessor.text_splitter options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/question_generator.md b/pipelines/docs/package/nodes/question_generator.md index 823dc7a1025f..06c0370ee4dc 100644 --- a/pipelines/docs/package/nodes/question_generator.md +++ b/pipelines/docs/package/nodes/question_generator.md @@ -1,6 +1,6 @@ # Question Generator Module -::: pipelines.nodes.question_generator.question_generator +::: pipelines.pipelines.nodes.question_generator.question_generator options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/ranker.md b/pipelines/docs/package/nodes/ranker.md index 0221a53864c5..910b8d76e186 100644 --- a/pipelines/docs/package/nodes/ranker.md +++ b/pipelines/docs/package/nodes/ranker.md @@ -1,6 +1,6 @@ # Ranker Module -::: pipelines.nodes.ranker.ernie_ranker +::: pipelines.pipelines.nodes.ranker.ernie_ranker options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/reader.md b/pipelines/docs/package/nodes/reader.md index 0612c6922848..b2530557f5dc 100644 --- a/pipelines/docs/package/nodes/reader.md +++ b/pipelines/docs/package/nodes/reader.md @@ -1,6 +1,6 @@ # Reader Module -::: pipelines.nodes.reader.ernie_dureader +::: pipelines.pipelines.nodes.reader.ernie_dureader options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/retriever.md b/pipelines/docs/package/nodes/retriever.md index b514a746f0a0..a4252819e5c6 100644 --- a/pipelines/docs/package/nodes/retriever.md +++ b/pipelines/docs/package/nodes/retriever.md @@ -1,20 +1,20 @@ # Retriever Module -::: pipelines.nodes.retriever.dense +::: pipelines.pipelines.nodes.retriever.dense options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.retriever.embedder +::: pipelines.pipelines.nodes.retriever.embedder options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.retriever.ernie_encoder +::: pipelines.pipelines.nodes.retriever.ernie_encoder options: summary: true separate_signature: true @@ -22,14 +22,14 @@ line_length: 60 -::: pipelines.nodes.retriever.multimodal_retriever +::: pipelines.pipelines.nodes.retriever.multimodal_retriever options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.retriever.parallel_retriever +::: pipelines.pipelines.nodes.retriever.parallel_retriever options: summary: true separate_signature: true @@ -37,14 +37,14 @@ line_length: 60 -::: pipelines.nodes.retriever.sparse +::: pipelines.pipelines.nodes.retriever.sparse options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.retriever.web +::: pipelines.pipelines.nodes.retriever.web options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/search_engine.md b/pipelines/docs/package/nodes/search_engine.md index fb2144224e21..a9ba2e3f0c5c 100644 --- a/pipelines/docs/package/nodes/search_engine.md +++ b/pipelines/docs/package/nodes/search_engine.md @@ -1,13 +1,13 @@ # Search Engine Module -::: pipelines.nodes.search_engine.providers +::: pipelines.pipelines.nodes.search_engine.providers options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.search_engine.web +::: pipelines.pipelines.nodes.search_engine.web options: summary: true separate_signature: true diff --git a/pipelines/docs/package/nodes/sentiment_analysis.md b/pipelines/docs/package/nodes/sentiment_analysis.md index 8b598f923366..6b8953ef6d1c 100644 --- a/pipelines/docs/package/nodes/sentiment_analysis.md +++ b/pipelines/docs/package/nodes/sentiment_analysis.md @@ -1,6 +1,6 @@ # Sentiment Analysis Module -::: pipelines.nodes.sentiment_analysis.senta_preprocessor +::: pipelines.pipelines.nodes.sentiment_analysis.senta_preprocessor options: summary: true separate_signature: true @@ -8,14 +8,14 @@ line_length: 60 -::: pipelines.nodes.sentiment_analysis.senta_visualization +::: pipelines.pipelines.nodes.sentiment_analysis.senta_visualization options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: pipelines.nodes.sentiment_analysis.senta +::: pipelines.pipelines.nodes.sentiment_analysis.senta options: summary: true separate_signature: true diff --git a/pipelines/docs/package/pipelines/standard_pipelines.md b/pipelines/docs/package/pipelines/standard_pipelines.md index f96b01f434d3..9137721f1168 100644 --- a/pipelines/docs/package/pipelines/standard_pipelines.md +++ b/pipelines/docs/package/pipelines/standard_pipelines.md @@ -1,6 +1,6 @@ # Pipeline Module -::: pipelines.pipelines.standard_pipelines +::: pipelines.pipelines.pipelines.standard_pipelines options: summary: true separate_signature: true diff --git a/pipelines/docs/package/rest_api/application.md b/pipelines/docs/package/rest_api/application.md index 63a92a3a1897..f0fa6843b9e9 100644 --- a/pipelines/docs/package/rest_api/application.md +++ b/pipelines/docs/package/rest_api/application.md @@ -1,6 +1,6 @@ # Application Module -::: rest_api.application +::: pipelines.rest_api.application options: summary: true separate_signature: true diff --git a/pipelines/docs/package/rest_api/controller.md b/pipelines/docs/package/rest_api/controller.md index fed96c36ea58..832455bc9bd5 100644 --- a/pipelines/docs/package/rest_api/controller.md +++ b/pipelines/docs/package/rest_api/controller.md @@ -1,34 +1,34 @@ # Controller Module -::: rest_api.controller.document +::: pipelines.rest_api.controller.document options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: rest_api.controller.feedback +::: pipelines.rest_api.controller.feedback options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: rest_api.controller.file_upload +::: pipelines.rest_api.controller.file_upload options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: rest_api.controller.router +::: pipelines.rest_api.controller.router options: summary: true separate_signature: true show_signature_annotations: true line_length: 60 -::: rest_api.controller.search +::: pipelines.rest_api.controller.search options: summary: true separate_signature: true From 8e729fb408d62e71d20473d3985a2beee3de5b1f Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 08:29:40 +0000 Subject: [PATCH 10/12] Add index.md --- pipelines/docs/index.md | 229 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 pipelines/docs/index.md diff --git a/pipelines/docs/index.md b/pipelines/docs/index.md new file mode 100644 index 000000000000..d108c25a216a --- /dev/null +++ b/pipelines/docs/index.md @@ -0,0 +1,229 @@ +## PaddleNLP Pipelines:NLP流水线系统 + +PaddleNLP Pipelines 是一个端到端NLP流水线系统框架,面向 NLP **全场景**,帮助用户**低门槛**构建强大**产品级系统**。 + +
+ +
+ +更多效果展示Demo请参考 [效果展示](#效果展示) + +## NLP流水线系统特色 +* **全场景支持**:依托灵活的插拔式组件产线化设计,支持各类 NLP 场景任务,包括:信息抽取、情感倾向分析、阅读理解、检索系统、问答系统、文本分类、文本生成等。 + +* **低门槛开发**:依托丰富的预置组件,像搭积木一样快速构建产品级系统,预置组件覆盖文档解析、数据处理、模型组网、预测部署、Web 服务、UI 界面等全流程系统功能。 + +* **高精度预测**:基于前沿的预训练模型、成熟的系统方案,可构建效果领先的产品级系统,如[NLP流水线系统](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines#NLP流水线系统)中预置的语义检索系统、阅读理解式智能问答系统等。 + +* **灵活可定制**:除深度兼容 PaddleNLP 模型组件外,还可嵌入飞桨生态下任意模型、[AI 开放平台算子](https://ai.baidu.com/)、其它开源项目如 Elasticsearch 等作为基础组件,快速扩展,从而实现任意复杂系统的灵活定制开发。 + +## Benchmarks + +
+ +
+ +更多的Benchmarks的信息请参考文档[Benchmarks](./benchmarks/README.md) + +## NLP流水线系统 + +PaddleNLP Pipelines NLP流水线系统针对 NLP 部分高频场景开源了经过充分打磨的产品级系统,并会不断开放其它场景的产品级系统,用户可以基于NLP流水线系统提供的系统能力快速开发出适配业务数据的产品。 + +* 快速搭建产品级[**语义检索**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines/examples/semantic-search)系统:使用自然语言文本通过语义进行智能文档查询,而不是关键字匹配 +* 快速搭建产品级[**智能问答**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines/examples/question-answering)系统:用自然语言提问,即可获得精准答案片段 +* 快速搭建产品级 [**FAQ 问答**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines/examples/FAQ)系统:用自然语言提问,匹配相关的高频问题,并返回匹配到的高频问题的答案 +* 快速搭建产品级**多模态信息抽取**系统(即将开放,敬请期待) + +### 效果展示 + ++ 语义检索 + +
+ +
+ ++ 智能问答 + +
+ +
+ ++ FAQ智能问答 + +
+ +
+ +| | | +|-|-| +| :floppy_disk: [快速安装](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines#floppy_disk-安装) |安装 PaddleNLP Pipelines| +| :beginner: [快速体验](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines#beginner-快速体验) |基于 Pipelines 快速搭建语义检索/智能问答等产品系统| +| :man_office_worker: [用户案例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines#man_office_worker-用户案例) |各行业用户基于PaddleNLP Pipelinse 构建的产品案例| +| :mortar_board: [Tutorials](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines#mortar_board-tutorials) |像搭积木一样一步步构建 NLP 流水线系统教程| +| :bar_chart: [Benchmarks](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines/benchmarks) |针对各场景模型的性能、精度评测指标| +| :telescope: [Roadmap](https://github.com/PaddlePaddle/PaddleNLP) | PaddleNLP Pipelines 产品路线图| +| :newspaper: [技术博客](https://github.com/PaddlePaddle/PaddleNLP) | 阅读 PaddleNLP Pipelines 系列技术文章| +| :vulcan_salute: [社区交流](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines#vulcan_salute-社区交流) | [官方微信群](https://github.com/PaddlePaddle/PaddleNLP#社区交流), [GitHub Discussions](https://github.com/PaddlePaddle/PaddleNLP/discussions) | + +## :floppy_disk: 安装 +Note: 因为 pipelines 依赖较多, 安装耗时大概 10 分钟左右,安装过程中请请耐心等待。 +### 环境依赖 +- python >= 3.7.3 +- paddlenlp >= 2.2.1 +- paddlepaddle >=2.3 +- CUDA Version: 10.2 +- NVIDIA Driver Version: 440.64.00 +- Docker 18.03 以上 +### pip 安装 +``` +pip install --upgrade paddle-pipelines +``` + +### 源码安装 +``` +git clone https://github.com/PaddlePaddle/PaddleNLP.git +cd PaddleNLP/pipelines +python setup.py install +``` + +## :beginner: 快速体验 + +### 快速开发 + +您可以参考如下示例像搭积木一样快速构建语义检索流水线,通过命令行终端输出快速体验流水线系统效果 + +```python +from pipelines.document_stores import FAISSDocumentStore +from pipelines.nodes import DensePassageRetriever, ErnieRanker + +# Step1: Preparing the data +documents = [ + {'content': '金钱龟不分品种,只有生长地之分,在我国主要分布于广东、广西、福建、海南、香港、澳门等地,在国外主要分布于越南等亚热带国家和地区。', + 'meta': {'name': 'test1.txt'}}, + {'content': '衡量酒水的价格的因素很多的,酒水的血统(也就是那里产的,采用什么工艺等);存储的时间等等,酒水是一件很难标准化得商品,只要你敢要价,有买的那就值那个钱。', + 'meta': {'name': 'test2.txt'}} +] + +# Step2: Initialize a FaissDocumentStore to store texts of documents +document_store = FAISSDocumentStore(embedding_dim=768) +document_store.write_documents(documents) + +# Step3: Initialize a DenseRetriever and build ANN index +retriever = DensePassageRetriever(document_store=document_store, query_embedding_model="rocketqa-zh-base-query-encoder",embed_title=False) +document_store.update_embeddings(retriever) + +# Step4: Initialize a Ranker +ranker = ErnieRanker(model_name_or_path="rocketqa-base-cross-encoder") + +# Step5: Initialize a SemanticSearchPipeline and ask questions +from pipelines import SemanticSearchPipeline +pipeline = SemanticSearchPipeline(retriever, ranker) +prediction = pipeline.run(query="衡量酒水的价格的因素有哪些?") +``` +### 快速部署 + +您可以基于我们发布的 Docker 镜像一键部署智能文本流水线系统,通过 Web UI 快速体验。 + +#### 启动 elastic search + +``` +docker network create elastic +docker pull docker.elastic.co/elasticsearch/elasticsearch:8.3.3 +docker run \ + -d \ + --name es02 \ + --net elastic \ + -p 9200:9200 \ + -e discovery.type=single-node \ + -e ES_JAVA_OPTS="-Xms256m -Xmx256m"\ + -e xpack.security.enabled=false \ + -e cluster.routing.allocation.disk.threshold_enabled=false \ + -it \ + docker.elastic.co/elasticsearch/elasticsearch:8.3.3 +``` + +#### 部署 CPU 服务 + +对于Linux使用Docker的用户,使用下面的命令: +``` +docker pull registry.baidubce.com/paddlepaddle/paddlenlp:2.4.0 +docker run -d --name paddlenlp_pipelines --net host -ti registry.baidubce.com/paddlepaddle/paddlenlp:2.4.0 +``` +对于Windows&Macos上使用Docker的用户,用下面的命令: + +``` +docker pull registry.baidubce.com/paddlepaddle/paddlenlp:2.4.0.windows.darwin +docker run -d --name paddlenlp_pipelines -p 8891:8891 -p 8502:8502 -ti registry.baidubce.com/paddlepaddle/paddlenlp:2.4.0.windows.darwin +``` +CPU 镜像下载大概耗时 10 分钟左右,容器启动成功后,等待3分钟左右,通过浏览器访问 [http://127.0.0.1:8502](http://127.0.0.1:8502) 快速体验产品级语义检索服务。 + + +#### 部署 GPU 服务 +``` +docker pull registry.baidubce.com/paddlepaddle/paddlenlp:2.4.0-gpu-cuda10.2-cudnn7 +nvidia-docker run -d --name paddlenlp_pipelines_gpu --net host -ti registry.baidubce.com/paddlepaddle/paddlenlp:2.4.0-gpu-cuda10.2-cudnn7 +``` +GPU 镜像下载大概耗时 15 分钟左右,容器启动成功后,等待1分钟左右,通过浏览器访问 [http://127.0.0.1:8502](http://127.0.0.1:8502) 快速体验产品级语义检索服务。 + + +对于国内用户,因为网络问题下载docker比较慢时,可使用百度提供的镜像: + + +| 环境 | 镜像 Tag | 运行平台 | +| :--------------------------: | :-------------------------------: | :-------------: | +| CPU | registry.baidubce.com/paddlepaddle/paddlenlp:2.4.0 | Linux | +| CPU | registry.baidubce.com/paddlepaddle/paddlenlp:2.4.0.windows.darwin | Windows&Macos | +| CUDA10.2 + cuDNN 7 | registry.baidubce.com/paddlepaddle/paddlenlp:2.4.0-gpu-cuda10.2-cudnn7 | Linux | +| CUDA11.2 + cuDNN 8 | registry.baidubce.com/paddlepaddle/paddlenlp:2.4.0-gpu-cuda11.2-cudnn8 | Linux | + +如果您的机器不在中国大陆地区,我们推荐您使用DockerHub的镜像: + +| 环境 | 镜像 Tag | 运行平台 | +| :--------------------------: | :-------------------------------: | :-------------: | +| CPU | paddlepaddle/paddlenlp:2.4.0 | Linux | +| CPU | paddlepaddle/paddlenlp:2.4.0.windows.darwin | Windows&Macos | +| CUDA10.2 + cuDNN 7 | paddlepaddle/paddlenlp:2.4.0-gpu-cuda10.2-cudnn7 | Linux | +| CUDA11.2 + cuDNN 8 | paddlepaddle/paddlenlp:2.4.0-gpu-cuda11.2-cudnn8 | Linux | + +对于智能问答应用,请参考Docker文档[docker文档](./docker/README.md),只需做少量的修改,就可以完成智能问答应用的部署。 + +#### REST API + +Pipelines可以服务化,通过HTTP接口的形式供其他程序进行调用,Pipelines提供了Swagger API方便用户查询接口文档,从而把Pipelines的能力接入到自己的应用系统中,只需要在启动REST API后通过浏览器访问 [http://127.0.0.1:8891/docs](http://127.0.0.1:8891/docs) + +
+ +
+ + +## :man_office_worker: 用户案例 + +### 案例1: [寻规-工程规范搜索引擎](https://xungui365.com/) + +[寻规](https://xungui365.com/),是一款基于飞桨 PaddleNLP Pipelines 构建的建筑工程规范搜索引擎。大幅提升了工程设计人员工作效率。 + +#### 查询效率提升 36~60 倍 + +相比市面当前的工程规范查询系统/网站,平均查询到一条规范条款要 3\~5 分钟,而基于 PaddleNLP Pipelines 构建的[寻规](https://xungui365.com/)检索系统,平均查询到一条规范条款仅需 5 秒左右,搜索时间大幅缩短,仅规范查询效率方面就提升**36\~60** 倍! + +#### 查询精度大幅提升 + +市面现已有的工程规范查询系统解决方案一直延续着传统关键字词匹配的查询方式,依赖用户对查询结果进行自行排序、筛选、鉴别,有时甚至还要再次由工程设计人员耗费一定时间精力人工查阅工程规范文件后,才能最终确认是否为想要查询的规范条款。传统规范查询系统至少需要进行 3~5 次查询才能找到用户想要的规范条款,而寻规系统是基于强大预训练模型构建起来的语义检索系统,针对 80% 的规范查询需求仅 **1 次查询** 就能精确命中查询意图,并返回真正符合工程设计人员查询意图的结果! + +## :mortar_board: Tutorials +- Tutorial 1 - Pipelines [Windows视频安装教程](https://www.bilibili.com/video/BV1DY4y1M7HE/?zw) +- Tutorial 2 - 语义检索 Pipeline: [AIStudio notebook](https://aistudio.baidu.com/aistudio/projectdetail/4442670) | [Python](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/pipelines/examples/semantic-search/semantic_search_example.py) +- Tutorial 3 - 智能问答 Pipeline: [AIStudio notebook](https://aistudio.baidu.com/aistudio/projectdetail/4442857) | [Python](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/pipelines/examples/question-answering/dense_qa_example.py) +- Tutorial 4 - FAQ智能问答 Pipeline: [AIStudio notebook](https://aistudio.baidu.com/aistudio/projectdetail/4465498) | [Python](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/pipelines/examples/FAQ/dense_faq_example.py) +- Tutorial 5 - Pipelines 快速上手二次开发教程: [AIStudio notebook](https://aistudio.baidu.com/aistudio/projectdetail/5011119) +## :vulcan_salute: 社区交流 +微信扫描二维码并填写问卷之后,加入交流群与来自各行各业的小伙伴交流学习吧~ +
+ +
+ + +## :heart: Acknowledge +我们借鉴了 Deepset.ai [Haystack](https://github.com/deepset-ai/haystack) 优秀的框架设计,在此对[Haystack](https://github.com/deepset-ai/haystack)作者及其开源社区表示感谢。 + +We learn form the excellent framework design of Deepset.ai [Haystack](https://github.com/deepset-ai/haystack), and we would like to express our thanks to the authors of Haystack and their open source community. From 93a97daccff3d60ba80ededea2d09c2707b67517 Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 08:43:19 +0000 Subject: [PATCH 11/12] Update README.md --- pipelines/docs/index.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pipelines/docs/index.md b/pipelines/docs/index.md index d108c25a216a..ab3d8a1b562c 100644 --- a/pipelines/docs/index.md +++ b/pipelines/docs/index.md @@ -8,6 +8,7 @@ PaddleNLP Pipelines 是一个端到端NLP流水线系统框架,面向 NLP ** 更多效果展示Demo请参考 [效果展示](#效果展示) + ## NLP流水线系统特色 * **全场景支持**:依托灵活的插拔式组件产线化设计,支持各类 NLP 场景任务,包括:信息抽取、情感倾向分析、阅读理解、检索系统、问答系统、文本分类、文本生成等。 @@ -120,6 +121,8 @@ from pipelines import SemanticSearchPipeline pipeline = SemanticSearchPipeline(retriever, ranker) prediction = pipeline.run(query="衡量酒水的价格的因素有哪些?") ``` +更多详细信息请参考[开发文档](https://paddle-pipelines.readthedocs.io/en/latest/) + ### 快速部署 您可以基于我们发布的 Docker 镜像一键部署智能文本流水线系统,通过 Web UI 快速体验。 From b8456b13484af9ca0290c249570a5782f2f3e802 Mon Sep 17 00:00:00 2001 From: w5688414 Date: Tue, 23 Apr 2024 15:59:42 +0000 Subject: [PATCH 12/12] Update README.md --- pipelines/README.md | 3 ++- pipelines/docs/index.md | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pipelines/README.md b/pipelines/README.md index d108c25a216a..5778158dff41 100644 --- a/pipelines/README.md +++ b/pipelines/README.md @@ -32,7 +32,6 @@ PaddleNLP Pipelines NLP流水线系统针对 NLP 部分高频场景开源了经 * 快速搭建产品级[**语义检索**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines/examples/semantic-search)系统:使用自然语言文本通过语义进行智能文档查询,而不是关键字匹配 * 快速搭建产品级[**智能问答**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines/examples/question-answering)系统:用自然语言提问,即可获得精准答案片段 * 快速搭建产品级 [**FAQ 问答**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines/examples/FAQ)系统:用自然语言提问,匹配相关的高频问题,并返回匹配到的高频问题的答案 -* 快速搭建产品级**多模态信息抽取**系统(即将开放,敬请期待) ### 效果展示 @@ -120,6 +119,8 @@ from pipelines import SemanticSearchPipeline pipeline = SemanticSearchPipeline(retriever, ranker) prediction = pipeline.run(query="衡量酒水的价格的因素有哪些?") ``` +更多详细信息请参考[开发文档](https://paddle-pipelines.readthedocs.io/en/latest/) + ### 快速部署 您可以基于我们发布的 Docker 镜像一键部署智能文本流水线系统,通过 Web UI 快速体验。 diff --git a/pipelines/docs/index.md b/pipelines/docs/index.md index ab3d8a1b562c..b7da5ef42200 100644 --- a/pipelines/docs/index.md +++ b/pipelines/docs/index.md @@ -33,7 +33,6 @@ PaddleNLP Pipelines NLP流水线系统针对 NLP 部分高频场景开源了经 * 快速搭建产品级[**语义检索**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines/examples/semantic-search)系统:使用自然语言文本通过语义进行智能文档查询,而不是关键字匹配 * 快速搭建产品级[**智能问答**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines/examples/question-answering)系统:用自然语言提问,即可获得精准答案片段 * 快速搭建产品级 [**FAQ 问答**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines/examples/FAQ)系统:用自然语言提问,匹配相关的高频问题,并返回匹配到的高频问题的答案 -* 快速搭建产品级**多模态信息抽取**系统(即将开放,敬请期待) ### 效果展示 @@ -121,7 +120,6 @@ from pipelines import SemanticSearchPipeline pipeline = SemanticSearchPipeline(retriever, ranker) prediction = pipeline.run(query="衡量酒水的价格的因素有哪些?") ``` -更多详细信息请参考[开发文档](https://paddle-pipelines.readthedocs.io/en/latest/) ### 快速部署