Como RAG funciona: da teoria ao pipeline

Sem RAG:
Usuário: "Qual o procedimento pra failover do banco?"
LLM: "Em geral, failover envolve..." (resposta genérica, pode estar errada)

Com RAG:
Usuário: "Qual o procedimento pra failover do banco?"
[Sistema busca nos runbooks → encontra o doc "DR-003: Failover PostgreSQL"]
LLM recebe: prompt + conteúdo do doc DR-003
LLM: "Segundo o procedimento DR-003, execute: 1. Verificar replicação..." (resposta específica)

# Criar resource group
az group create --name rg-rag-demo --location eastus2

# Criar Azure AI Search
az search service create \
  --name rag-demo-search \
  --resource-group rg-rag-demo \
  --sku standard \
  --partition-count 1 \
  --replica-count 1

# Criar Azure OpenAI (pra embeddings e chat)
az cognitiveservices account create \
  --name rag-demo-openai \
  --resource-group rg-rag-demo \
  --kind OpenAI \
  --sku S0 \
  --location eastus2

# Deploy do modelo de embedding
az cognitiveservices account deployment create \
  --name rag-demo-openai \
  --resource-group rg-rag-demo \
  --deployment-name text-embedding-3-small \
  --model-name text-embedding-3-small \
  --model-version "1" \
  --model-format OpenAI \
  --sku-capacity 1 \
  --sku-name Standard

# Deploy do modelo de chat
az cognitiveservices account deployment create \
  --name rag-demo-openai \
  --resource-group rg-rag-demo \
  --deployment-name gpt-4o \
  --model-name gpt-4o \
  --model-version "2024-08-06" \
  --model-format OpenAI \
  --sku-capacity 1 \
  --sku-name Standard

# Criar índice via REST API
az rest --method PUT \
  --url "https://rag-demo-search.search.windows.net/indexes/runbooks?api-version=2024-07-01" \
  --skip-authorization-header \
  --headers "Content-Type=application/json" "api-key=<admin-key>" \
  --body '{
    "name": "runbooks",
    "fields": [
      {"name": "id", "type": "Edm.String", "key": true, "filterable": true},
      {"name": "title", "type": "Edm.String", "searchable": true},
      {"name": "content", "type": "Edm.String", "searchable": true},
      {"name": "source_file", "type": "Edm.String", "filterable": true},
      {"name": "chunk_index", "type": "Edm.Int32", "filterable": true},
      {"name": "embedding", "type": "Collection(Edm.Single)",
       "searchable": true, "retrievable": false, "stored": false,
       "dimensions": 1536, "vectorSearchProfile": "rag-profile"}
    ],
    "vectorSearch": {
      "algorithms": [{"name": "hnsw-config", "kind": "hnsw", 
        "hnswParameters": {"m": 4, "efConstruction": 400, "efSearch": 500, "metric": "cosine"}}],
      "profiles": [{"name": "rag-profile", "algorithm": "hnsw-config"}]
    }
  }'

import os
import tiktoken
from azure.search.documents import SearchClient
from azure.search.documents.models import VectorizedQuery
from azure.core.credentials import AzureKeyCredential
from openai import AzureOpenAI

# Configuração
search_client = SearchClient(
    endpoint="https://rag-demo-search.search.windows.net",
    index_name="runbooks",
    credential=AzureKeyCredential(os.environ["SEARCH_KEY"])
)

openai_client = AzureOpenAI(
    azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
    api_key=os.environ["AZURE_OPENAI_KEY"],
    api_version="2024-10-21"
)

TOKENIZER = tiktoken.get_encoding("cl100k_base")

def chunk_text(text, chunk_size=800, overlap=200):
    """Divide texto em chunks aproximados por token."""
    if overlap >= chunk_size:
        raise ValueError("overlap precisa ser menor que chunk_size")

    token_ids = TOKENIZER.encode(text)
    chunks = []
    start = 0
    step = chunk_size - overlap

    while start < len(token_ids):
        end = start + chunk_size
        chunk = TOKENIZER.decode(token_ids[start:end])
        chunks.append(chunk)
        start += step

    return chunks

def get_embedding(text):
    """Gera embedding via Azure OpenAI."""
    response = openai_client.embeddings.create(
        input=text,
        model="text-embedding-3-small"
    )
    return response.data[0].embedding

def index_document(file_path, title):
    """Processa e indexa um documento."""
    with open(file_path, "r", encoding="utf-8") as f:
        content = f.read()

    chunks = chunk_text(content)
    documents = []

    for i, chunk in enumerate(chunks):
        documents.append(
            {
                "id": f"{os.path.basename(file_path)}-{i}",
                "title": title,
                "content": chunk,
                "source_file": file_path,
                "chunk_index": i,
                "embedding": get_embedding(chunk),
            }
        )

    search_client.upload_documents(documents=documents)
    print(f"Indexado: {title} ({len(chunks)} chunks)")

def rag_query(question, top_k=5):
    """Busca documentos relevantes e gera resposta."""

    question_vector = get_embedding(question)

    vector_query = VectorizedQuery(
        vector=question_vector,
        kind="vector",
        fields="embedding",
        k_nearest_neighbors=top_k,
    )

    # Hybrid search: vector + keyword
    results = search_client.search(
        search_text=question,
        vector_queries=[vector_query],
        top=top_k,
    )

    # Montar contexto com os chunks encontrados
    context_parts = []
    for result in results:
        context_parts.append(f"[{result['title']}]\n{result['content']}")

    context = "\n\n---\n\n".join(context_parts)

    # Gerar resposta com o LLM
    response = openai_client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {
                "role": "system",
                "content": (
                    "Responda a pergunta usando APENAS o contexto fornecido. "
                    "Se a informação não estiver no contexto, diga que não encontrou."
                ),
            },
            {"role": "user", "content": f"Contexto:\n{context}\n\nPergunta: {question}"},
        ],
        temperature=0.1,
    )

    return response.choices[0].message.content