RAG-пайплайны на основе веб-данных: от парсинга до векторной базы

from playwright.sync_api import sync_playwright
 
def scrape_clean_text(url: str) -> str:
    with sync_playwright() as p:
        browser = p.chromium.launch()
        page = browser.new_page()
        page.goto(url, wait_until="networkidle")
 
        text = page.evaluate("""
            () => {
                document.querySelectorAll('script,style,nav,header,footer,aside').forEach(el => el.remove());
                return document.body.innerText;
            }
        """)
        browser.close()
        return " ".join(text.split())  # схлопнуть лишние пробелы/переносы

def chunk_by_headings(text: str, max_chars: int = 1000, overlap: int = 100) -> list[str]:
    paragraphs = text.split("\n\n")
    chunks, current = [], ""
 
    for para in paragraphs:
        if len(current) + len(para) > max_chars and current:
            chunks.append(current.strip())
            current = current[-overlap:] + para  # хвост предыдущего чанка для контекста
        else:
            current += " " + para
 
    if current.strip():
        chunks.append(current.strip())
    return chunks

import openai
import psycopg2
 
def embed_and_store(chunks: list[str], source_url: str, conn):
    for chunk in chunks:
        embedding = openai.embeddings.create(
            model="text-embedding-3-small", input=chunk
        ).data[0].embedding
 
        conn.execute(
            "INSERT INTO documents (content, embedding, source_url, scraped_at) "
            "VALUES (%s, %s, %s, NOW())",
            (chunk, embedding, source_url),
        )

def answer_question(question: str, conn) -> str:
    q_embedding = embed(question)
 
    rows = conn.execute(
        "SELECT content, source_url FROM documents "
        "ORDER BY embedding <-> %s LIMIT 5",
        (q_embedding,),
    ).fetchall()
 
    context = "\n\n".join(f"[{r.source_url}]\n{r.content}" for r in rows)
    prompt = f"Ответь на вопрос, используя только контекст ниже. Если ответа нет в контексте, скажи об этом.\n\nКонтекст:\n{context}\n\nВопрос: {question}"
 
    return llm_call(prompt)