Class: Baran::CharacterTextSplitter

Inherits:
TextSplitter show all
Defined in:
lib/baran/character_text_splitter.rb

Instance Attribute Summary collapse

Attributes inherited from TextSplitter

#chunk_overlap, #chunk_size

Instance Method Summary collapse

Methods inherited from TextSplitter

#chunks, #joined, #merged

Constructor Details

#initialize(chunk_size: 1024, chunk_overlap: 64, separator: nil) ⇒ CharacterTextSplitter

Returns a new instance of CharacterTextSplitter.



7
8
9
10
 
# File 'lib/baran/character_text_splitter.rb', line 7

def initialize(chunk_size: 1024, chunk_overlap: 64, separator: nil)
  super(chunk_size: chunk_size, chunk_overlap: chunk_overlap)
  @separator = separator || "\n\n"
end

Instance Attribute Details

#separator ⇒ Object

Returns the value of attribute separator.



5
6
7
 
# File 'lib/baran/character_text_splitter.rb', line 5

def separator
  @separator
end

Instance Method Details

#splitted(text) ⇒ Object 



12
13
14
15
 
# File 'lib/baran/character_text_splitter.rb', line 12

def splitted(text)
  splits = separator.empty? ? text.chars : text.split(separator)
  merged(splits, @separator)
end