Skip to content

Document Loaders API

This module provides document loaders to extract text from various sources.

Base Loader

maticlib.io.BaseLoader 

BaseLoader(chunker=None)

Bases: ABC

Initialize the loader.

Parameters:

Name Type Description Default
chunker Optional[BaseChunker]

An optional chunker to split the loaded documents into segments. If not provided, the entire document is yielded as a single segment.

 None
Source code in maticlib/io/base.py 
 8
 9
10
11
12
13
14
15
16
def __init__(self, chunker: Optional[BaseChunker] = None):
    """
    Initialize the loader.

    Args:
        chunker: An optional chunker to split the loaded documents into segments.
                 If not provided, the entire document is yielded as a single segment.
    """
    self.chunker = chunker

load abstractmethod 

load(source, metadata=None)

Load a document from the given source (file path, URL, etc.) and yield TextSegments.

Source code in maticlib/io/base.py 
18
19
20
21
22
23
24
25
26
@abstractmethod
def load(
    self, source: str, metadata: Optional[Dict[str, Any]] = None
) -> Iterable[TextSegment]:
    """
    Load a document from the given source (file path, URL, etc.)
    and yield TextSegments.
    """
    pass

load_async async 

load_async(source, metadata=None)

Load a document asynchronously.

Source code in maticlib/io/base.py 
29
30
31
32
33
34
35
36
37
38
39
40
async def load_async(
    self, source: str, metadata: Optional[Dict[str, Any]] = None
) -> Iterable[TextSegment]:
    """
    Load a document asynchronously.
    """
    import asyncio

    loop = asyncio.get_running_loop()
    return await loop.run_in_executor(
        None, lambda: list(self.load(source, metadata))
    )

Text Loader

maticlib.io.TextLoader 

TextLoader(chunker=None)

Bases: BaseLoader

Loads plain text (.txt) files as TextSegments.

Source code in maticlib/io/base.py 
 8
 9
10
11
12
13
14
15
16
def __init__(self, chunker: Optional[BaseChunker] = None):
    """
    Initialize the loader.

    Args:
        chunker: An optional chunker to split the loaded documents into segments.
                 If not provided, the entire document is yielded as a single segment.
    """
    self.chunker = chunker

load 

load(source, metadata=None)

Reads a plain text file and yields TextSegments.

Parameters:

Name Type Description Default
source str

Absolute or relative path to the .txt file.

 required
metadata Optional[Dict[str, Any]]

Optional metadata dict to attach to each segment.

 None

Returns:

Type Description
Iterable[TextSegment]

An iterable of TextSegment objects.
Source code in maticlib/io/file.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
def load(
    self, source: str, metadata: Optional[Dict[str, Any]] = None
) -> Iterable[TextSegment]:
    """
    Reads a plain text file and yields TextSegments.

    Args:
        source: Absolute or relative path to the .txt file.
        metadata: Optional metadata dict to attach to each segment.

    Returns:
        An iterable of TextSegment objects.
    """
    if not os.path.exists(source):
        raise DocumentLoadError(f"File not found: {source}")

    base_meta = metadata or {}
    base_meta["source"] = source
    base_meta["loader"] = "TextLoader"

    try:
        with open(source, "r", encoding="utf-8") as f:
            content = f.read()
    except Exception as e:
        raise DocumentLoadError(f"Failed to read file {source}: {e}")

    if self.chunker:
        parent_id = uuid.uuid4().hex[:12]
        yield from self.chunker.chunk_text(
            content, base_metadata=base_meta, parent_id=parent_id
        )
    else:
        yield TextSegment(
            content=content, metadata=base_meta, segment_id=uuid.uuid4().hex[:12]
        )

PDF Loader

maticlib.io.PDFLoader 

PDFLoader(chunker=None)

Bases: BaseLoader

Loads PDF files page-by-page as TextSegments. Requires pypdf.

Source code in maticlib/io/base.py 
 8
 9
10
11
12
13
14
15
16
def __init__(self, chunker: Optional[BaseChunker] = None):
    """
    Initialize the loader.

    Args:
        chunker: An optional chunker to split the loaded documents into segments.
                 If not provided, the entire document is yielded as a single segment.
    """
    self.chunker = chunker

load 

load(source, metadata=None)

Reads a PDF and yields one TextSegment per page.

Parameters:

Name Type Description Default
source str

Absolute or relative path to the .pdf file.

 required
metadata Optional[Dict[str, Any]]

Optional metadata dict to attach to each segment.

 None

Returns:

Type Description
Iterable[TextSegment]

An iterable of TextSegment objects, one per page.
Source code in maticlib/io/file.py 
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
def load(
    self, source: str, metadata: Optional[Dict[str, Any]] = None
) -> Iterable[TextSegment]:
    """
    Reads a PDF and yields one TextSegment per page.

    Args:
        source: Absolute or relative path to the .pdf file.
        metadata: Optional metadata dict to attach to each segment.

    Returns:
        An iterable of TextSegment objects, one per page.
    """
    try:
        import pypdf
    except ImportError as e:
        raise MissingDependencyError(
            "pypdf is required for PDFLoader. Install it with: pip install maticlib[rag]"
        ) from e

    if not os.path.exists(source):
        raise DocumentLoadError(f"File not found: {source}")

    base_meta = metadata or {}
    base_meta["source"] = source
    base_meta["loader"] = "PDFLoader"

    try:
        with open(source, "rb") as f:
            reader = pypdf.PdfReader(f)
            content = ""
            for page in reader.pages:
                content += page.extract_text() + "\n\n"
    except Exception as e:
        raise DocumentLoadError(f"Failed to read PDF {source}: {e}")

    if self.chunker:
        parent_id = uuid.uuid4().hex[:12]
        yield from self.chunker.chunk_text(
            content, base_metadata=base_meta, parent_id=parent_id
        )
    else:
        yield TextSegment(
            content=content, metadata=base_meta, segment_id=uuid.uuid4().hex[:12]
        )

DOCX Loader

maticlib.io.DOCXLoader 

DOCXLoader(chunker=None)

Bases: BaseLoader

Loads Microsoft Word (.docx) files as TextSegments. Requires python-docx.

Source code in maticlib/io/base.py 
 8
 9
10
11
12
13
14
15
16
def __init__(self, chunker: Optional[BaseChunker] = None):
    """
    Initialize the loader.

    Args:
        chunker: An optional chunker to split the loaded documents into segments.
                 If not provided, the entire document is yielded as a single segment.
    """
    self.chunker = chunker

load 

load(source, metadata=None)

Reads a .docx file and yields TextSegments.

Parameters:

Name Type Description Default
source str

Absolute or relative path to the .docx file.

 required
metadata Optional[Dict[str, Any]]

Optional metadata dict to attach to each segment.

 None

Returns:

Type Description
Iterable[TextSegment]

An iterable of TextSegment objects.
Source code in maticlib/io/file.py 
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
def load(
    self, source: str, metadata: Optional[Dict[str, Any]] = None
) -> Iterable[TextSegment]:
    """
    Reads a .docx file and yields TextSegments.

    Args:
        source: Absolute or relative path to the .docx file.
        metadata: Optional metadata dict to attach to each segment.

    Returns:
        An iterable of TextSegment objects.
    """
    try:
        import docx
    except ImportError as e:
        raise MissingDependencyError(
            "python-docx is required for DOCXLoader. Install it with: pip install maticlib[rag]"
        ) from e

    if not os.path.exists(source):
        raise DocumentLoadError(f"File not found: {source}")

    base_meta = metadata or {}
    base_meta["source"] = source
    base_meta["loader"] = "DOCXLoader"

    try:
        doc = docx.Document(source)
        content = "\n".join([para.text for para in doc.paragraphs])
    except Exception as e:
        raise DocumentLoadError(f"Failed to read DOCX {source}: {e}")

    if self.chunker:
        parent_id = uuid.uuid4().hex[:12]
        yield from self.chunker.chunk_text(
            content, base_metadata=base_meta, parent_id=parent_id
        )
    else:
        yield TextSegment(
            content=content, metadata=base_meta, segment_id=uuid.uuid4().hex[:12]
        )

WebPage Loader

maticlib.io.WebPageLoader 

WebPageLoader(chunker=None)

Bases: BaseLoader

Fetches a URL and extracts readable text using BeautifulSoup. Requires beautifulsoup4 and httpx.

Source code in maticlib/io/base.py 
 8
 9
10
11
12
13
14
15
16
def __init__(self, chunker: Optional[BaseChunker] = None):
    """
    Initialize the loader.

    Args:
        chunker: An optional chunker to split the loaded documents into segments.
                 If not provided, the entire document is yielded as a single segment.
    """
    self.chunker = chunker

load 

load(source, metadata=None)

Fetches a web page and yields TextSegments.

Parameters:

Name Type Description Default
source str

The URL of the page to fetch.

 required
metadata Optional[Dict[str, Any]]

Optional metadata dict to attach to each segment.

 None

Returns:

Type Description
Iterable[TextSegment]

An iterable of TextSegment objects with the page's cleaned text.
Source code in maticlib/io/web.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
def load(
    self, source: str, metadata: Optional[Dict[str, Any]] = None
) -> Iterable[TextSegment]:
    """
    Fetches a web page and yields TextSegments.

    Args:
        source: The URL of the page to fetch.
        metadata: Optional metadata dict to attach to each segment.

    Returns:
        An iterable of TextSegment objects with the page's cleaned text.
    """

    try:
        from bs4 import BeautifulSoup
    except ImportError as e:
        raise MissingDependencyError(
            "beautifulsoup4 is required for WebPageLoader. Install it with: pip install maticlib[rag]"
        ) from e

    base_meta = metadata or {}
    base_meta["source"] = source
    base_meta["loader"] = "WebPageLoader"

    try:
        with httpx.Client(follow_redirects=True, timeout=10.0) as client:
            response = client.get(source)
            response.raise_for_status()
            html = response.text
    except Exception as e:
        raise DocumentLoadError(f"Failed to fetch URL {source}: {e}")

    try:
        soup = BeautifulSoup(html, "html.parser")
        # Remove scripts and styles
        for script in soup(["script", "style"]):
            script.extract()

        # Get text and clean up whitespace
        text = soup.get_text(separator="\n")
        lines = (line.strip() for line in text.splitlines())
        chunks_text = (line for line in lines if line)
        content = "\n".join(chunks_text)
    except Exception as e:
        raise DocumentLoadError(f"Failed to parse HTML from {source}: {e}")

    if self.chunker:
        parent_id = uuid.uuid4().hex[:12]
        yield from self.chunker.chunk_text(
            content, base_metadata=base_meta, parent_id=parent_id
        )
    else:
        yield TextSegment(
            content=content, metadata=base_meta, segment_id=uuid.uuid4().hex[:12]
        )