Como RAG funciona: da teoria ao pipeline

Sem RAG:
Usuário: "Qual o procedimento pra failover do banco?"
LLM: "Em geral, failover envolve..." (resposta genérica, pode estar errada)

Com RAG:
Usuário: "Qual o procedimento pra failover do banco?"
[Sistema busca nos runbooks → encontra o doc "DR-003: Failover PostgreSQL"]
LLM recebe: prompt + conteúdo do doc DR-003
LLM: "Segundo o procedimento DR-003, execute: 1. Verificar replicação..." (resposta específica)

# Criar resource group
az group create --name rg-rag-demo --location eastus2

# Criar Azure AI Search
az search service create \
  --name rag-demo-search \
  --resource-group rg-rag-demo \
  --sku standard \
  --partition-count 1 \
  --replica-count 1

# Criar Azure OpenAI (pra embeddings e chat)
az cognitiveservices account create \
  --name rag-demo-openai \
  --resource-group rg-rag-demo \
  --kind OpenAI \
  --sku S0 \
  --location eastus2

# Deploy do modelo de embedding
az cognitiveservices account deployment create \
  --name rag-demo-openai \
  --resource-group rg-rag-demo \
  --deployment-name text-embedding-3-small \
  --model-name text-embedding-3-small \
  --model-version "1" \
  --model-format OpenAI \
  --sku-capacity 1 \
  --sku-name Standard

# Deploy do modelo de chat
az cognitiveservices account deployment create \
  --name rag-demo-openai \
  --resource-group rg-rag-demo \
  --deployment-name gpt-4o \
  --model-name gpt-4o \
  --model-version "2024-08-06" \
  --model-format OpenAI \
  --sku-capacity 1 \
  --sku-name Standard

# Criar índice via REST API
az rest --method PUT \
  --url "https://rag-demo-search.search.windows.net/indexes/runbooks?api-version=2024-07-01" \
  --headers "Content-Type=application/json" "api-key=<admin-key>" \
  --body '{
    "name": "runbooks",
    "fields": [
      {"name": "id", "type": "Edm.String", "key": true, "filterable": true},
      {"name": "title", "type": "Edm.String", "searchable": true},
      {"name": "content", "type": "Edm.String", "searchable": true},
      {"name": "source_file", "type": "Edm.String", "filterable": true},
      {"name": "chunk_index", "type": "Edm.Int32", "filterable": true},
      {"name": "embedding", "type": "Collection(Edm.Single)",
       "searchable": true, "retrievable": false, "stored": false,
       "dimensions": 1536, "vectorSearchProfile": "rag-profile"}
    ],
    "vectorSearch": {
      "algorithms": [{"name": "hnsw-config", "kind": "hnsw", 
        "hnswParameters": {"m": 4, "efConstruction": 400, "efSearch": 500, "metric": "cosine"}}],
      "profiles": [{"name": "rag-profile", "algorithm": "hnsw-config"}]
    }
  }'

import os
from azure.search.documents import SearchClient
from azure.core.credentials import AzureKeyCredential
from openai import AzureOpenAI

# Configuração
search_client = SearchClient(
    endpoint="https://rag-demo-search.search.windows.net",
    index_name="runbooks",
    credential=AzureKeyCredential(os.environ["SEARCH_KEY"])
)

openai_client = AzureOpenAI(
    azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
    api_key=os.environ["AZURE_OPENAI_KEY"],
    api_version="2024-06-01"
)

def chunk_text(text, chunk_size=800, overlap=200):
    """Divide texto em chunks com overlap."""
    words = text.split()
    chunks = []
    start = 0
    while start < len(words):
        end = start + chunk_size
        chunk = " ".join(words[start:end])
        chunks.append(chunk)
        start = end - overlap
    return chunks

def get_embedding(text):
    """Gera embedding via Azure OpenAI."""
    response = openai_client.embeddings.create(
        input=text,
        model="text-embedding-3-small"
    )
    return response.data[0].embedding

def index_document(file_path, title):
    """Processa e indexa um documento."""
    with open(file_path, "r") as f:
        content = f.read()
    
    chunks = chunk_text(content)
    documents = []
    
    for i, chunk in enumerate(chunks):
        doc = {
            "id": f"{os.path.basename(file_path)}-{i}",
            "title": title,
            "content": chunk,
            "source_file": file_path,
            "chunk_index": i,
            "embedding": get_embedding(chunk)
        }
        documents.append(doc)
    
    search_client.upload_documents(documents=documents)
    print(f"Indexado: {title} ({len(chunks)} chunks)")

from azure.search.documents.models import VectorizedQuery

def rag_query(question, top_k=5):
    """Busca documentos relevantes e gera resposta."""
    
    question_vector = get_embedding(question)

    # Hybrid search: vector + keyword
    results = search_client.search(
        search_text=question,  # keyword search
        vector_queries=[
            VectorizedQuery(
                vector=question_vector,
                k_nearest_neighbors=top_k,
                fields="embedding",
                kind="vector"
            )
        ],
        top=top_k
    )
    
    # Montar contexto com os chunks encontrados
    context_parts = []
    for result in results:
        context_parts.append(f"[{result['title']}]\n{result['content']}")
    
    context = "\n\n---\n\n".join(context_parts)
    
    # Gerar resposta com o LLM
    response = openai_client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": 
             "Responda a pergunta usando APENAS o contexto fornecido. "
             "Se a informação não estiver no contexto, diga que não encontrou."},
            {"role": "user", "content": 
             f"Contexto:\n{context}\n\nPergunta: {question}"}
        ],
        temperature=0.1
    )
    
    return response.choices[0].message.content