fgg

from llama_index.core.ingestion import (
    IngestionPipeline,
    IngestionCache,
)

from llama_index.core.node_parser import (
    SentenceSplitter
)

from config.settings import (
    CHUNK_SIZE,
    CHUNK_OVERLAP
)


def run_pipeline(documents):

    cache = IngestionCache(
        cache_dir="./cache"
    )

    pipeline = IngestionPipeline(
        transformations=[
            SentenceSplitter(
                chunk_size=CHUNK_SIZE,
                chunk_overlap=CHUNK_OVERLAP
            )
        ],
        cache=cache
    )

    nodes = pipeline.run(
        documents=documents
    )

    return nodes