Chunkers API

This module provides tools for splitting documents into segments.

Base Chunker

maticlib.core.text.chunkers.base.BaseChunker

BaseChunker(target_size=1000, overlap_size=200)

Bases: ABC

Abstract base class for all text chunkers.

Initializes the BaseChunker.

Parameters:

Name	Type	Description	Default
target_size	int	Maximum character length of a single chunk.	1000
overlap_size	int	Number of characters from the previous chunk to overlap.	200

Source code in maticlib/core/text/chunkers/base.py

def __init__(self, target_size: int = 1000, overlap_size: int = 200):
    """
    Initializes the BaseChunker.

    Args:
        target_size: Maximum character length of a single chunk.
        overlap_size: Number of characters from the previous chunk to overlap.
    """
    self.target_size = target_size
    self.overlap_size = overlap_size

chunk_documents

chunk_documents(documents)

Accepts a list of document dictionaries and chunks each one.

Parameters:

Name	Type	Description	Default
documents	List[Dict[str, Any]]	List of dicts with content (str) and metadata (dict) keys.	required

Returns:

Type	Description
List[TextSegment]	A flat list of TextSegments from all documents.

Source code in maticlib/core/text/chunkers/base.py

def chunk_documents(self, documents: List[Dict[str, Any]]) -> List[TextSegment]:
    """
    Accepts a list of document dictionaries and chunks each one.

    Args:
        documents: List of dicts with ``content`` (str) and ``metadata`` (dict) keys.

    Returns:
        A flat list of TextSegments from all documents.
    """
    segments = []
    for doc in documents:
        content = doc.get("content", "")
        meta = doc.get("metadata", {})
        segments.extend(self.chunk_text(content, base_metadata=meta))
    return segments

chunk_text abstractmethod

chunk_text(text, base_metadata=None, parent_id=None)

Split a single text string into multiple TextSegments.

Parameters:

Name	Type	Description	Default
text	str	The raw text content to split.	required
base_metadata	Optional[Dict[str, Any]]	Optional metadata dict to attach to each segment.	None
parent_id	Optional[str]	Optional parent segment ID for hierarchical chunking.	None

Returns:

Type	Description
List[TextSegment]	A list of TextSegment objects.

Source code in maticlib/core/text/chunkers/base.py

@abstractmethod
def chunk_text(
    self,
    text: str,
    base_metadata: Optional[Dict[str, Any]] = None,
    parent_id: Optional[str] = None,
) -> List[TextSegment]:
    """
    Split a single text string into multiple TextSegments.

    Args:
        text: The raw text content to split.
        base_metadata: Optional metadata dict to attach to each segment.
        parent_id: Optional parent segment ID for hierarchical chunking.

    Returns:
        A list of TextSegment objects.
    """
    pass

Separator Chunker

maticlib.core.text.chunkers.separator.SeparatorChunker

SeparatorChunker(
    separator="\n\n", target_size=1000, overlap_size=200
)

Bases: BaseChunker

Initializes the SeparatorChunker.

Parameters:

Name	Type	Description	Default
separator	str	The string used to split the text (default \n\n).	'\n\n'
target_size	int	Maximum character length of a single chunk.	1000
overlap_size	int	Number of characters from the previous chunk to overlap.	200

Source code in maticlib/core/text/chunkers/separator.py

def __init__(
    self,
    separator: str = "\n\n",
    target_size: int = 1000,
    overlap_size: int = 200,
):
    """
    Initializes the SeparatorChunker.

    Args:
        separator: The string used to split the text (default ``\\n\\n``).
        target_size: Maximum character length of a single chunk.
        overlap_size: Number of characters from the previous chunk to overlap.
    """
    super().__init__(target_size, overlap_size)
    self.separator = separator

chunk_text

chunk_text(text, base_metadata=None, parent_id=None)

Splits text by the separator and groups splits into chunks.

Parameters:

Name	Type	Description	Default
text	str	The raw text content to split.	required
base_metadata	Optional[Dict[str, Any]]	Optional metadata to attach to each segment.	None
parent_id	Optional[str]	Optional parent segment ID.	None

Returns:

Type	Description
List[TextSegment]	A list of TextSegment objects.

Source code in maticlib/core/text/chunkers/separator.py

def chunk_text(
    self,
    text: str,
    base_metadata: Optional[Dict[str, Any]] = None,
    parent_id: Optional[str] = None,
) -> List[TextSegment]:
    """
    Splits text by the separator and groups splits into chunks.

    Args:
        text: The raw text content to split.
        base_metadata: Optional metadata to attach to each segment.
        parent_id: Optional parent segment ID.

    Returns:
        A list of TextSegment objects.
    """
    base_metadata = base_metadata or {}
    if not text:
        return []

    splits = text.split(self.separator)
    chunks = []
    current_chunk = []
    current_length = 0

    for split in splits:
        split_len = len(split)
        if (
            current_length + split_len + len(self.separator) > self.target_size
            and current_chunk
        ):
            chunks.append(self.separator.join(current_chunk))

            # Handling overlap logic (simple implementation)
            # Keep adding from the end of current_chunk until overlap size is reached
            overlap_chunk = []
            overlap_length = 0
            for item in reversed(current_chunk):
                if (
                    overlap_length + len(item) + len(self.separator)
                    <= self.overlap_size
                ):
                    overlap_chunk.insert(0, item)
                    overlap_length += len(item) + len(self.separator)
                else:
                    break

            current_chunk = overlap_chunk
            current_length = overlap_length

        current_chunk.append(split)
        current_length += split_len + len(self.separator)

    if current_chunk:
        chunks.append(self.separator.join(current_chunk))

    segments = []
    total_chunks = len(chunks)
    for i, chunk in enumerate(chunks):
        meta = base_metadata.copy()
        meta.update(
            {
                "chunk_index": i,
                "total_chunks": total_chunks,
            }
        )
        if parent_id:
            meta["parent_id"] = parent_id

        segments.append(
            TextSegment(
                content=chunk, metadata=meta, segment_id=uuid.uuid4().hex[:12]
            )
        )

    return segments

Token Budget Chunker

maticlib.core.text.chunkers.token_budget.TokenBudgetChunker

TokenBudgetChunker(
    target_tokens=256, overlap_tokens=32, tokeniser=None
)

Bases: BaseChunker

Source code in maticlib/core/text/chunkers/token_budget.py

def __init__(
    self,
    target_tokens: int = 256,
    overlap_tokens: int = 32,
    tokeniser: Optional[Any] = None,
):
    super().__init__(target_size=target_tokens, overlap_size=overlap_tokens)
    self.tokeniser = tokeniser

    # Check if they are trying to use tiktoken explicitly
    if self.tokeniser and getattr(self.tokeniser, "__module__", "").startswith(
        "tiktoken"
    ):
        try:
            import tiktoken
        except ImportError as e:
            raise MissingDependencyError(
                "tiktoken is required for TokenBudgetChunker when using a tiktoken encoding. "
                "Install it with: pip install maticlib[chunking]"
            ) from e

Semantic Difference Chunker

maticlib.core.text.chunkers.semantic_difference.SemanticDifferenceChunker

SemanticDifferenceChunker(
    embedding_model,
    similarity_threshold=0.75,
    min_chunk_size=100,
    buffer_sentences=1,
)

Bases: BaseChunker

Source code in maticlib/core/text/chunkers/semantic_difference.py

def __init__(
    self,
    embedding_model: BaseEmbeddings,
    similarity_threshold: float = 0.75,
    min_chunk_size: int = 100,
    buffer_sentences: int = 1,
):
    super().__init__()
    if np is None:
        raise MissingDependencyError(
            "numpy is required for SemanticDifferenceChunker. "
            "Install it with: pip install maticlib[chunking]"
        )
    self.embedding_model = embedding_model
    self.similarity_threshold = similarity_threshold
    self.min_chunk_size = min_chunk_size
    self.buffer_sentences = buffer_sentences

Hierarchical Chunker

maticlib.core.text.chunkers.hierarchical.HierarchicalChunker

HierarchicalChunker(
    separators=None,
    target_size=1000,
    overlap_size=200,
    language=None,
)

Bases: BaseChunker

Source code in maticlib/core/text/chunkers/hierarchical.py

def __init__(
    self,
    separators: Optional[List[str]] = None,
    target_size: int = 1000,
    overlap_size: int = 200,
    language: Optional[str] = None,
):
    super().__init__(target_size, overlap_size)
    if language:
        self.separators = self._get_separators_for_language(language)
    else:
        self.separators = separators or self.DEFAULT_SEPARATORS