Class: Aoororachain::RecursiveTextSplitter

Inherits:
Object
  • Object
show all
Defined in:
lib/aoororachain/recursive_text_splitter.rb

Instance Method Summary collapse

Constructor Details

#initialize(size: 1024, overlap: 200) ⇒ RecursiveTextSplitter

Returns a new instance of RecursiveTextSplitter.



5
6
7
8
 
# File 'lib/aoororachain/recursive_text_splitter.rb', line 5

def initialize(size: 1024, overlap: 200)
  @size = size
  @overlap = overlap
end

Instance Method Details

#split_documents(documents) ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/aoororachain/recursive_text_splitter.rb', line 10

def split_documents(documents)
  original_documents = Array(documents)

  new_documents = []
  original_documents.each do |document|
    texts = split_text(document.content)
    texts.each do |text|
      new_documents << Document.new(text, document.metadata)
    end
  end

  new_documents
end

#split_text(text) ⇒ Object 



24
25
26
 
# File 'lib/aoororachain/recursive_text_splitter.rb', line 24

def split_text(text)
  split_recursive(text, 0, [])
end