Building RAG Systems

# Example code for document processing pipeline
from langchain_community.document_loaders import PyPDFLoader, DirectoryLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
import os

# Load documents from a directory
loader = DirectoryLoader('./documents/', glob="**/*.pdf", loader_cls=PyPDFLoader)
documents = loader.load()

print(f"Loaded {len(documents)} document pages")

# Text splitting with overlap
text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    length_function=len,
    add_start_index=True,
)

chunks = text_splitter.split_documents(documents)
print(f"Split into {len(chunks)} chunks")

# Add metadata to chunks
for i, chunk in enumerate(chunks):
    chunk.metadata["chunk_id"] = i
    # Extract and add more metadata as needed
    if "page" in chunk.metadata:
        chunk.metadata["source"] = f"Page {chunk.metadata['page']} from {chunk.metadata['source']}"