From 5080ce2ccf2a3fa936e3fa904b3780a8b17c3bf0 Mon Sep 17 00:00:00 2001 From: Chase Pursley Date: Sat, 11 May 2024 10:35:15 -0400 Subject: [PATCH] Separators: Implement html chunking strategy. --- lib/text_chunker.ex | 17 +-- .../strategies/recursive_chunk/separators.ex | 76 +++++++------ test/recursive_chunk_test.exs | 107 +++++++++++++++++- .../fixtures/document_fixtures/test_file.html | 63 +++++++++++ 4 files changed, 218 insertions(+), 45 deletions(-) create mode 100644 test/support/fixtures/document_fixtures/test_file.html diff --git a/lib/text_chunker.ex b/lib/text_chunker.ex index c695cba..6a5d47b 100644 --- a/lib/text_chunker.ex +++ b/lib/text_chunker.ex @@ -21,20 +21,21 @@ defmodule TextChunker do @supported_formats [ :doc, :docx, + :elixir, :epub, + :html, + :javascript, :latex, + :markdown, :odt, :pdf, - :rtf, - :markdown, - :plaintext, - :elixir, - :ruby, :php, + :plaintext, :python, - :vue, - :javascript, - :typescript + :rtf, + :ruby, + :typescript, + :vue ] @opts_schema [ diff --git a/lib/text_chunker/strategies/recursive_chunk/separators.ex b/lib/text_chunker/strategies/recursive_chunk/separators.ex index 0692b1a..cf0850c 100644 --- a/lib/text_chunker/strategies/recursive_chunk/separators.ex +++ b/lib/text_chunker/strategies/recursive_chunk/separators.ex @@ -34,20 +34,12 @@ defmodule TextChunker.Strategies.RecursiveChunk.Separators do "```\n\n", "\n\n___\n\n", "\n\n---\n\n", - "\n\n***\n\n", - "\n\n", - "\n", - " " - ] + "\n\n***\n\n" + ] ++ empty_and_new_line_separators() end def get_separators(:plaintext) do - [ - "\n\n", - "\n", - " ", - "" - ] + empty_and_new_line_separators() ++ [""] end def get_separators(:elixir) do @@ -68,11 +60,8 @@ defmodule TextChunker.Strategies.RecursiveChunk.Separators do " with ", " cond ", " case ", - " if ", - "\n\n", - "\n", - " " - ] + " if " + ] ++ empty_and_new_line_separators() end def get_separators(:ruby) do @@ -90,11 +79,8 @@ defmodule TextChunker.Strategies.RecursiveChunk.Separators do " for ", " do ", " begin ", - " rescue ", - "\n\n", - "\n", - " " - ] + " rescue " + ] ++ empty_and_new_line_separators() end def get_separators(:php) do @@ -113,22 +99,16 @@ defmodule TextChunker.Strategies.RecursiveChunk.Separators do " while ", " do ", " switch ", - " case ", - "\n\n", - "\n", - " " - ] + " case " + ] ++ empty_and_new_line_separators() end def get_separators(:python) do [ "\nclass ", "\ndef ", - "\n\tdef ", - "\n\n", - "\n", - " " - ] + "\n\tdef " + ] ++ empty_and_new_line_separators() end def get_separators(:vue) do @@ -157,14 +137,38 @@ defmodule TextChunker.Strategies.RecursiveChunk.Separators do " while ", " switch ", " case ", - " default ", - "\n\n", - "\n", - " " - ] + " default " + ] ++ empty_and_new_line_separators() + end + + def get_separators(:html) do + [ + " TextChunker.split(opts) |> TestHelpers.extract_text_from_chunks() + + expected_result = [ + "

Elixir: A Powerful Language for Building Scalable Applications

\n", + "

Elixir is a dynamic, functional programming language designed for building scalable and", + " scalable and maintainable applications. It runs on the Erlang Virtual Machine (VM), which is known", + " which is known for its robust concurrency and fault-tolerance capabilities. In this article, we'll", + " this article, we'll explore the key features of Elixir and discuss different chunking approaches", + " chunking approaches for Retrieval Augmented Generation (RAG).

", + "\n\n", + "

Key Features of Elixir

\n", + "", + "\n\n", + "
\n ", + "

Chunking Approaches for Retrieval Augmented Generation

\n ", + "

Retrieval Augmented Generation (RAG) is a technique that combines information retrieval with", + " retrieval with language generation to generate high-quality and informative text. Chunking, the", + " text. Chunking, the process of breaking down text into smaller units, plays a crucial role in RAG.", + " role in RAG. Let's explore different chunking approaches commonly used in Elixir:

", + "\n ", + "
    \n ", + "
  1. Sentence-based Chunking: This approach splits the text into individual sentences using", + " sentences using punctuation markers such as periods, question marks, and exclamation points. Each", + " points. Each sentence becomes a separate chunk, allowing for fine-grained retrieval and", + " retrieval and generation.
    2. ", + "\n ", + "
  2. Paragraph-based Chunking: With this approach, the text is divided into paragraphs based on the", + " based on the presence of newline characters or specific paragraph delimiters. Paragraphs provide a", + " provide a coherent and self-contained unit of information suitable for RAG.
    3. ", + "\n ", + "
  3. Semantic Chunking: Semantic chunking involves analyzing the text and identifying meaningful", + " meaningful semantic units or phrases. This can be achieved using techniques like named entity", + " like named entity recognition, noun phrase extraction, or dependency parsing. Semantic chunks", + " Semantic chunks capture the core concepts and ideas within the text.
    4. ", + "\n ", + "
  4. Custom Chunking: Elixir provides the flexibility to define custom chunking rules based on", + " rules based on specific requirements. For example, you can chunk text based on a certain number of", + " a certain number of words, specific delimiters, or regular expressions that match particular", + " match particular patterns.
    5. ", + "\n
\n ", + "

The choice of chunking approach depends on the nature of the text and the desired granularity of", + " granularity of retrieval and generation. Elixir's powerful string manipulation and pattern matching", + " pattern matching capabilities make it easy to implement various chunking strategies", + " chunking strategies efficiently.

", + "\n
", + "\n\n", + "
\n ", + "

Benefits of Elixir/Erlang

\n ", + "

Elixir and Erlang offer several advantages over other language stacks when it comes to building", + " comes to building scalable and fault-tolerant systems. Let's take a look at some of the key", + " at some of the key benefits:

", + "\n ", + "\n \n \n \n \n ", + "\n \n \n \n \n ", + "\n ", + "\n \n \n ", + "\n \n ", + "\n \n \n ", + "\n \n ", + "\n \n \n ", + "\n \n ", + "\n \n \n ", + "\n ", + "\n \n \n
BenefitDescription
Concurrency and ScalabilityBuilt-in support for lightweight processes and efficient message passing.
Fault-Tolerance\"Let it crash\" philosophy and automatic recovery from failures.
Hot Code SwappingUpdate code without stopping the system, providing zero downtime.
Distribution and ClusteringBuilt-in support for distributed systems and easy horizontal scaling.
Ecosystem and LibrariesGrowing ecosystem with a wide range of libraries and supportive community.
\n ", + "

These benefits make Elixir and Erlang a compelling choice for building scalable, fault-tolerant,", + " fault-tolerant, and maintainable systems, especially in domains like web development, real-time", + " real-time applications, and distributed systems.

", + "\n
", + "\n\n", + "
Conclusion
\n", + "

Elixir's combination of functional programming, concurrency, and fault-tolerance makes it a", + " makes it a powerful language for building scalable and maintainable RAG systems. By leveraging the", + " By leveraging the appropriate chunking approaches and utilizing Elixir's strengths, you can create", + " you can create efficient and high-quality retrieval augmented generation solutions.

", + "\n\n", + "
© Elixir RAG. All rights reserved.
" + ] + + assert result == expected_result + end + end + describe "rejects unsupported options" do test "rejects a chunk_overlap of -1" do opts = [ @@ -384,7 +489,7 @@ defmodule TextChunkerTest do assert result == { :error, - "invalid value for :format option: expected one of [:doc, :docx, :epub, :latex, :odt, :pdf, :rtf, :markdown, :plaintext, :elixir, :ruby, :php, :python, :vue, :javascript, :typescript], got: :made_up_format" + "invalid value for :format option: expected one of [:doc, :docx, :elixir, :epub, :html, :javascript, :latex, :markdown, :odt, :pdf, :php, :plaintext, :python, :rtf, :ruby, :typescript, :vue], got: :made_up_format" } end diff --git a/test/support/fixtures/document_fixtures/test_file.html b/test/support/fixtures/document_fixtures/test_file.html new file mode 100644 index 0000000..64f5602 --- /dev/null +++ b/test/support/fixtures/document_fixtures/test_file.html @@ -0,0 +1,63 @@ +

Elixir: A Powerful Language for Building Scalable Applications

+

Elixir is a dynamic, functional programming language designed for building scalable and maintainable applications. It runs on the Erlang Virtual Machine (VM), which is known for its robust concurrency and fault-tolerance capabilities. In this article, we'll explore the key features of Elixir and discuss different chunking approaches for Retrieval Augmented Generation (RAG).

+ +

Key Features of Elixir

+ + +
+

Chunking Approaches for Retrieval Augmented Generation

+

Retrieval Augmented Generation (RAG) is a technique that combines information retrieval with language generation to generate high-quality and informative text. Chunking, the process of breaking down text into smaller units, plays a crucial role in RAG. Let's explore different chunking approaches commonly used in Elixir:

+
    +
  1. Sentence-based Chunking: This approach splits the text into individual sentences using punctuation markers such as periods, question marks, and exclamation points. Each sentence becomes a separate chunk, allowing for fine-grained retrieval and generation.
    2. +
  2. Paragraph-based Chunking: With this approach, the text is divided into paragraphs based on the presence of newline characters or specific paragraph delimiters. Paragraphs provide a coherent and self-contained unit of information suitable for RAG.
    3. +
  3. Semantic Chunking: Semantic chunking involves analyzing the text and identifying meaningful semantic units or phrases. This can be achieved using techniques like named entity recognition, noun phrase extraction, or dependency parsing. Semantic chunks capture the core concepts and ideas within the text.
    4. +
  4. Custom Chunking: Elixir provides the flexibility to define custom chunking rules based on specific requirements. For example, you can chunk text based on a certain number of words, specific delimiters, or regular expressions that match particular patterns.
    5. +
+

The choice of chunking approach depends on the nature of the text and the desired granularity of retrieval and generation. Elixir's powerful string manipulation and pattern matching capabilities make it easy to implement various chunking strategies efficiently.

+
+ +
+

Benefits of Elixir/Erlang

+

Elixir and Erlang offer several advantages over other language stacks when it comes to building scalable and fault-tolerant systems. Let's take a look at some of the key benefits:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BenefitDescription
Concurrency and ScalabilityBuilt-in support for lightweight processes and efficient message passing.
Fault-Tolerance"Let it crash" philosophy and automatic recovery from failures.
Hot Code SwappingUpdate code without stopping the system, providing zero downtime.
Distribution and ClusteringBuilt-in support for distributed systems and easy horizontal scaling.
Ecosystem and LibrariesGrowing ecosystem with a wide range of libraries and supportive community.
+

These benefits make Elixir and Erlang a compelling choice for building scalable, fault-tolerant, and maintainable systems, especially in domains like web development, real-time applications, and distributed systems.

+
+ +
Conclusion
+

Elixir's combination of functional programming, concurrency, and fault-tolerance makes it a powerful language for building scalable and maintainable RAG systems. By leveraging the appropriate chunking approaches and utilizing Elixir's strengths, you can create efficient and high-quality retrieval augmented generation solutions.

+ +
© Elixir RAG. All rights reserved.
\ No newline at end of file