10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
|
# File 'lib/text_splitters/recursive_character_text_splitter.rb', line 10
def split(text)
output = []
good_splits = []
separator = @separators.last
@separators.each do |s|
if text.include?(s)
separator = s
break
end
end
splits = text.split(separator)
splits.each do |s|
if s.length < @chunk_size
good_splits << s
else
if good_splits.any?
merged_text = merge_splits(good_splits, separator)
output.concat(merged_text)
good_splits = []
end
other_info = split(s)
output.concat(other_info)
end
end
if good_splits.any?
merged_text = merge_splits(good_splits, separator)
output.concat(merged_text)
end
output
end
|