Class: Langchain::Chunker::Markdown

Inherits:
Base
  • Object
show all
Defined in:
lib/langchain/chunker/markdown.rb

Overview

Simple text chunker

Usage:

Langchain::Chunker::Markdown.new(text).chunks

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(text, chunk_size: 1000, chunk_overlap: 200) ⇒ Markdown

Returns a new instance of Markdown.

Parameters:

  • text (String)
  • chunk_size (Integer) (defaults to: 1000)
  • chunk_overlap (Integer) (defaults to: 200)
  • separator (String) 


18
19
20
21
22
 
# File 'lib/langchain/chunker/markdown.rb', line 18

def initialize(text, chunk_size: 1000, chunk_overlap: 200)
  @text = text
  @chunk_size = chunk_size
  @chunk_overlap = chunk_overlap
end

Instance Attribute Details

#chunk_overlap ⇒ Object (readonly)

Returns the value of attribute chunk_overlap.



12
13
14
 
# File 'lib/langchain/chunker/markdown.rb', line 12

def chunk_overlap
  @chunk_overlap
end

#chunk_size ⇒ Object (readonly)

Returns the value of attribute chunk_size.



12
13
14
 
# File 'lib/langchain/chunker/markdown.rb', line 12

def chunk_size
  @chunk_size
end

#text ⇒ Object (readonly)

Returns the value of attribute text.



12
13
14
 
# File 'lib/langchain/chunker/markdown.rb', line 12

def text
  @text
end

Instance Method Details

#chunks ⇒ Array<Langchain::Chunk>

Returns:



25
26
27
28
29
30
31
32
33
34
 
# File 'lib/langchain/chunker/markdown.rb', line 25

def chunks
  splitter = Baran::MarkdownSplitter.new(
    chunk_size: chunk_size,
    chunk_overlap: chunk_overlap
  )

  splitter.chunks(text).map do |chunk|
    Langchain::Chunk.new(text: chunk[:text])
  end
end