Observability & evaluation

# In your .env or shell — no code changes needed
LANGSMITH_TRACING=true
LANGSMITH_API_KEY=lsv2-...
LANGSMITH_PROJECT=jarvis-prod

# Optional: tag a specific run for findability
from langsmith import traceable

@traceable(name="weekly-digest", tags=["scheduled"])
def run_weekly_digest():
    ...

// In your .env or shell — no code changes needed
LANGSMITH_TRACING=true
LANGSMITH_API_KEY=lsv2-...
LANGSMITH_PROJECT=jarvis-prod

import { traceable } from "langsmith/traceable";

const runWeeklyDigest = traceable(
  async () => { /* ... */ },
  { name: "weekly-digest", tags: ["scheduled"] },
);

from langsmith import Client

ls = Client()
ds = ls.create_dataset(dataset_name="jarvis-hr-cases", description="HR-routed user requests with expected answers.")
ls.create_examples(
    dataset_id=ds.id,
    inputs=[
        {"messages": [{"role": "user", "content": "How much leave do I have left?"}]},
        {"messages": [{"role": "user", "content": "What is parental leave policy?"}]},
    ],
    outputs=[
        {"expected_route": "hr", "expected_tool_called": "leave_balance"},
        {"expected_route": "hr", "expected_kb_cited": True},
    ],
)

import { Client } from "langsmith";

const ls = new Client();
const ds = await ls.createDataset("jarvis-hr-cases", { description: "HR routed cases." });
await ls.createExamples({
  datasetId: ds.id,
  inputs: [
    { messages: [{ role: "user", content: "How much leave do I have left?" }] },
  ],
  outputs: [
    { expected_route: "hr", expected_tool_called: "leave_balance" },
  ],
});

from langsmith.evaluation import evaluate

# Heuristic evaluator — pure code
def called_open_ticket(run, example) -> dict:
    tool_calls = [m for m in run.outputs["messages"] if m.get("tool_calls")]
    called = any(tc["name"] == "open_it_ticket" for m in tool_calls for tc in m["tool_calls"])
    return {"key": "called_open_ticket", "score": int(called)}

# LLM-as-judge evaluator
from langchain_anthropic import ChatAnthropic
judge = ChatAnthropic(model="claude-haiku-4-5-20251001", temperature=0)

JUDGE = """Score 0-1 on whether the assistant's final reply is HELPFUL and SAFE.
Question: {input}
Reply: {output}
Return JSON: {"score": <0|1>, "reason": ""}"""

def helpfulness(run, example):
    out = judge.invoke(JUDGE.format(
        input=example.inputs["messages"][-1]["content"],
        output=run.outputs["messages"][-1]["content"],
    ))
    import json; r = json.loads(out.content)
    return {"key": "helpfulness", "score": r["score"], "comment": r["reason"]}

# Run the eval
results = evaluate(
    lambda inputs: jarvis.invoke(inputs),
    data="jarvis-hr-cases",
    evaluators=[called_open_ticket, helpfulness],
)

import { evaluate } from "langsmith/evaluation";

function calledOpenTicket(run: any) {
  const tcs = (run.outputs.messages ?? []).flatMap((m: any) => m.tool_calls ?? []);
  return { key: "called_open_ticket", score: tcs.some((tc: any) => tc.name === "open_it_ticket") ? 1 : 0 };
}

const judge = new ChatAnthropic({ model: "claude-haiku-4-5-20251001", temperature: 0 });

async function helpfulness(run: any, example: any) {
  const out = await judge.invoke(`Score 0-1 helpfulness. Q: ${example.inputs.messages.at(-1).content}
A: ${run.outputs.messages.at(-1).content}
Return JSON {"score": 0|1, "reason": "..."}`);
  const r = JSON.parse(out.content as string);
  return { key: "helpfulness", score: r.score, comment: r.reason };
}

await evaluate(async (inputs) => jarvis.invoke(inputs), {
  data: "jarvis-hr-cases",
  evaluators: [calledOpenTicket, helpfulness],
});