From 4308f3fdb79c9ad847bcd5c5eabe5fdaa9002aeb Mon Sep 17 00:00:00 2001 From: Frank Huynh Date: Thu, 6 Aug 2026 22:28:12 +0700 Subject: [PATCH] feat(gooddata-eval): add KDA-skill agentic evaluator Adds an agentic evaluation runner for the Key Driver Analysis (KDA) skill: drives the create/execute KDA tool-call flow through a live chat session, evaluates completion (kda_triggered/executed/success/turn_completed) plus informational per-field correctness (Measure/Date Attribute/Periods/ Filters/Summary, all kda_-prefixed scores, not yet gated on strict_pass), and logs whole-turn latency plus pass_at_k/pass_power_k for gdc-nas's combo_report.py to bucket into its own daily-report table. Trace selection is deliberately NOT attempted here: a conversation can have several Langfuse traces sharing one session_id (title generation, a disambiguation turn, the actual KDA turn), and picking the right one requires checking observations, which are ingested asynchronously just like latency. This module links whichever trace the default (max-latency) selector finds, like every other skill; combo_report.py resolves the real KDA trace itself, well after the run, when ingestion has settled -- and owns that logic entirely rather than importing it from here, since the cross-repo import never actually resolves in the report-generation job. Also from PR review, including two follow-up independent re-reviews: - log_quality_and_value_scores (used by every agentic skill, not just KDA) no longer treats an unresolved latency/cost as the worst possible outcome -- drops that weighted term and renormalizes instead. - Dedupe _is_asking_clarification (copy-pasted across kda_skill.py, metric_skill.py, and a drifted third copy in conversation.py) into a shared _clarification.py. Only the bare "?"-anywhere check is tightened to require the message end on a question; the "could you"/"please"/"clarif" substring checks stay as broad as before, since they weren't the source of the original false-positive and conversation.py's multi-turn driver relies on their recall. - All 6 informational KDA scores are kda_-prefixed, closing off any future collision with another skill's own score of the same shape. - pass_power_k is now logged (mirrors visualization.py's own pass_at_K/pass_power_K) instead of being computed and discarded. - Replaced an unconditional print() with _log.info, named the 0.01-absolute-tolerance magic number, and fixed a couple of stale/ missing comments (a docstring reference to a removed function, a missing copyright header). JIRA: QA-28800 --- .../gooddata_eval/core/agentic/__init__.py | 14 + .../core/agentic/_clarification.py | 25 + .../gooddata_eval/core/agentic/_langfuse.py | 30 +- .../core/agentic/conversation.py | 10 +- .../gooddata_eval/core/agentic/kda_skill.py | 472 ++++++++++++++++ .../core/agentic/metric_skill.py | 10 +- .../tests/test_agentic_kda_skill.py | 532 ++++++++++++++++++ .../tests/test_agentic_langfuse_trace.py | 58 ++ .../gooddata-eval/tests/test_clarification.py | 33 ++ 9 files changed, 1162 insertions(+), 22 deletions(-) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py create mode 100644 packages/gooddata-eval/tests/test_agentic_kda_skill.py create mode 100644 packages/gooddata-eval/tests/test_agentic_langfuse_trace.py create mode 100644 packages/gooddata-eval/tests/test_clarification.py diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py index 639bee5b7..89e93dde8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py @@ -30,6 +30,14 @@ evaluate_agentic_guardrail, run_agentic_guardrail, ) +from gooddata_eval.core.agentic.kda_skill import ( + AgenticKdaSummary, + KdaEvaluation, + KdaRunResult, + KdaSkillAssertionError, + evaluate_agentic_kda_skill, + run_agentic_kda_skill, +) from gooddata_eval.core.agentic.metric_skill import ( AgenticMetricSummary, MetricRunResult, @@ -56,6 +64,7 @@ "AgenticAlertSummary", "AgenticGeneralQuestionSummary", "AgenticGuardrailSummary", + "AgenticKdaSummary", "AgenticMetricSummary", "AgenticSearchSummary", "AgenticRunSummary", @@ -69,6 +78,9 @@ "GeneralQuestionResult", "GuardrailAssertionError", "GuardrailResult", + "KdaEvaluation", + "KdaRunResult", + "KdaSkillAssertionError", "MetricRunResult", "MetricSkillAssertionError", "RunResult", @@ -81,6 +93,7 @@ "evaluate_agentic_conversation", "evaluate_agentic_general_question", "evaluate_agentic_guardrail", + "evaluate_agentic_kda_skill", "evaluate_agentic_metric_skill", "evaluate_agentic_search_tool", "evaluate_agentic_visualization", @@ -88,6 +101,7 @@ "run_agentic_conversation", "run_agentic_general_question", "run_agentic_guardrail", + "run_agentic_kda_skill", "run_agentic_metric_skill", "run_agentic_search_tool", "run_agentic_visualization", diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py new file mode 100644 index 000000000..c6137e1fe --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py @@ -0,0 +1,25 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Shared "is this a clarifying question?" heuristic for agentic skill runners.""" + +from __future__ import annotations + + +def is_asking_clarification(text: str) -> bool: + """True if ``text`` reads as the agent asking the user for input, not a final answer. + + Only the bare ``"?"``-anywhere check is tightened to require the message actually END + on a question -- a "?" anywhere in the text also matches a final answer that merely + quotes or rhetorically references a question, which would wrongly keep a single-turn + case going into a simulated-reply retry and could mask a real turn-1 failure behind an + artificial turn-2 pass. The other phrase checks stay substring-anywhere as before: they + weren't the source of that false-positive, and conversation.py's multi-turn, multi-skill + driver (up to 20 clarification rounds, not just KDA's single-turn case) relies on their + broader recall -- narrowing them too would risk the opposite failure, a real + disambiguation message going undetected and being graded as if it were the final answer. + """ + if not text: + return False + t = text.strip().lower() + if t.endswith("?"): + return True + return "could you" in t or "please" in t or "clarif" in t diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py index 67630ce2f..5a69918cf 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py @@ -31,7 +31,12 @@ def __init__(self, raw: dict) -> None: self.id: str = raw.get("id", "") self.metadata: dict = raw.get("metadata") or {} self.session_id: str | None = raw.get("sessionId") or raw.get("session_id") - self.latency: float = float(raw.get("latency") or 0.0) + # None (missing/null) is preserved, not coerced to 0.0 -- a trace that hasn't + # finished ingesting has UNKNOWN latency, not zero latency, and callers (e.g. + # log_quality_and_value_scores below, and every skill's own `pt.latency if pt + # else None` gating) rely on that distinction to not treat "unknown" as the best + # possible outcome. + self.latency: float | None = float(raw["latency"]) if raw.get("latency") is not None else None self.total_cost: float = float(raw.get("totalCost") or raw.get("total_cost") or 0.0) @@ -358,11 +363,24 @@ def log_quality_and_value_scores( data_type="NUMERIC", comment=f"{passed}/{total} strict checks passed", ) - speed = 0.0 if latency_sec is None else max(0.0, 1.0 - latency_sec / _MAX_LATENCY_SEC) - cost_factor = 0.0 if cost_usd is None else max(0.0, 1.0 - cost_usd / _MAX_COST_USD) - value = _QUALITY_WEIGHT * quality + _SPEED_WEIGHT * speed + _COST_WEIGHT * cost_factor + # An unresolved latency/cost (trace not yet settled, price not available) is UNKNOWN, + # not the best (1.0) or worst (0.0) possible outcome -- substituting either would + # silently pull value_score toward one extreme. Drop that weighted term instead and + # renormalize over whichever components do have a real value, so value_score always + # reflects only the signals actually measured for this run. + components = [(_QUALITY_WEIGHT, quality)] + speed = None if latency_sec is None else max(0.0, 1.0 - latency_sec / _MAX_LATENCY_SEC) + if speed is not None: + components.append((_SPEED_WEIGHT, speed)) + cost_factor = None if cost_usd is None else max(0.0, 1.0 - cost_usd / _MAX_COST_USD) + if cost_factor is not None: + components.append((_COST_WEIGHT, cost_factor)) + weight_total = sum(w for w, _ in components) + value = sum(w * v for w, v in components) / weight_total latency_str = "unknown" if latency_sec is None else f"{latency_sec:.2f}s" cost_str = "unknown" if cost_usd is None else f"${cost_usd:.4f}" + speed_str = "n/a" if speed is None else f"{speed:.2f}" + cost_factor_str = "n/a" if cost_factor is None else f"{cost_factor:.2f}" score_safe( langfuse, trace_id, @@ -371,8 +389,8 @@ def log_quality_and_value_scores( data_type="NUMERIC", comment=( f"{_QUALITY_WEIGHT}*quality({quality:.2f}) + " - f"{_SPEED_WEIGHT}*speed({speed:.2f}) + " - f"{_COST_WEIGHT}*cost({cost_factor:.2f}); " + f"{_SPEED_WEIGHT}*speed({speed_str}) + " + f"{_COST_WEIGHT}*cost({cost_factor_str}), renormalized /{weight_total:.1f}; " f"latency={latency_str}; cost={cost_str}" ), ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index a7c3034fe..73ee7342d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -11,6 +11,7 @@ from gooddata_sdk import GoodDataSdk from pydantic import BaseModel +from gooddata_eval.core.agentic._clarification import is_asking_clarification from gooddata_eval.core.agentic.alert_skill import render_alert_proposal from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids from gooddata_eval.core.chat.sse_client import ChatClient @@ -192,13 +193,6 @@ def _check_output_correct(turn: TurnDefinition, chat_result: ChatResult) -> bool return None -def _is_asking_clarification(text: str) -> bool: - if not text: - return False - t = text.lower() - return "?" in t or "could you" in t or "please" in t or "clarif" in t - - def _get_sim_user_response(agent_message: str, turn: TurnDefinition, expected_output: dict | None) -> str: """Generate a simulated user reply to an agent clarification question.""" otype = turn.expected_output_type @@ -327,7 +321,7 @@ def run_agentic_conversation( response_text = (chat_result.text_response or "").strip() if not response_text and chat_result.alert_proposals: response_text = render_alert_proposal(chat_result.alert_proposals[-1]) - asking = _is_asking_clarification(response_text) or bool(chat_result.alert_proposals) + asking = is_asking_clarification(response_text) or bool(chat_result.alert_proposals) if asking and clarification_turns < max_clarification_turns: clarification_turns += 1 total_clarification_turns += 1 diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py new file mode 100644 index 000000000..2c90f2671 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -0,0 +1,472 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Agentic KDA (Key Driver Analysis)-skill evaluation runner.""" + +from __future__ import annotations + +import json +import logging +import os +from dataclasses import dataclass +from typing import Any + +from gooddata_eval.core.agentic._clarification import is_asking_clarification +from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.models import ToolCallEvent + +_log = logging.getLogger(__name__) + +# A single run per case; callers that want pass_at_k/pass_power_k variance pass k +# explicitly (e.g. gdc-nas's KDA_RUN_K for the daily cron). +_DEFAULT_K = 1 +# KDA cases are designed to resolve in one turn (unlike alert/metric skills), so this is +# only a safety net for the rare disambiguation turn -- a title collision (see the +# handoff's known-collision cases) or a metric-vs-fact form choice -- not a general +# multi-turn budget. +_DEFAULT_MAX_ITERATIONS = 2 +# summary_correct is informational only (see KdaEvaluation docstring), so an absolute +# tolerance this tight is harmless today -- but it's an effectively-exact-match bar on +# revenue-scale values, and would fail near-constantly if ever promoted to strict_pass. A +# relative tolerance is the right fix for that follow-up ticket, not this constant. +_DEFAULT_SUMMARY_ABS_TOLERANCE = 0.01 + + +def _to_number(value: object) -> float | int | None: + """Convert string/number to int or float, None on failure. Mirrors alert_skill._to_number + -- the API is contractually numeric here, but this guards against a malformed response + raising a raw ValueError instead of failing the check cleanly.""" + if value is None: + return None + try: + f = float(str(value)) + return int(f) if f == int(f) else f + except (ValueError, TypeError): + return None + + +def _normalize_measure(m: dict) -> tuple[Any, Any, Any]: + return (m.get("type"), m.get("id"), m.get("aggregation")) + + +def _measure_matches(actual: object, expected: dict | list[dict] | None) -> bool: + """expected may be a single candidate dict or a list of candidate dicts (mirrors + metric_skill's expected_output: dict | list -- e.g. case 1 accepts either the + catalog metric id or the mathematically equivalent ad-hoc fact+SUM). + + ``actual`` is typed ``object``, not ``dict``, and checked with ``isinstance`` (mirroring + alert_skill._deep_subset) because it comes from a tool call the LLM constructed -- + a malformed call could put a non-dict value there. + """ + if not isinstance(actual, dict) or expected is None: + return False + candidates = expected if isinstance(expected, list) else [expected] + actual_norm = _normalize_measure(actual) + return any(actual_norm == _normalize_measure(c) for c in candidates if isinstance(c, dict)) + + +def _filters_match(actual: object, expected: list) -> bool: + actual = actual or [] + try: + return json.dumps(actual, sort_keys=True) == json.dumps(expected, sort_keys=True) + except TypeError: + return False + + +def _within_tolerance(actual: object, expected: object, tolerance: float) -> bool: + a, e = _to_number(actual), _to_number(expected) + if a is None or e is None: + return False + return abs(a - e) <= tolerance + + +def generate_simulated_kda_response(agent_message: str, measure_candidates: dict | list[dict] | None) -> str: + """Generate a user reply to keep the KDA-skill conversation going (gpt-4o-mini). + + Used only when the agent asks a clarifying question instead of triggering KDA + directly (e.g. a title collision between two metrics). Picks *any* candidate from + ``measure_candidates`` -- not necessarily the one an eventual correctness ticket + would require -- because the current scope only needs KDA to trigger, not the + resulting measure to be exactly right (see KdaEvaluation docstring). + + Always uses OpenAI regardless of which provider the combo under test runs -- this is + test-harness plumbing to keep a disambiguation turn moving, not the system under test, + and CI always has ``OPENAI_API_KEY`` from Vault for every combo (see + rw_e2e_test_tavern.yml) independent of the combo's own provider/model. + """ + try: + from openai import OpenAI # noqa: PLC0415 + except ImportError as exc: + raise RuntimeError("openai package is required for generate_simulated_kda_response") from exc + + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + raise OSError("OPENAI_API_KEY environment variable is not set") + + client = OpenAI(api_key=api_key) + candidates = measure_candidates if isinstance(measure_candidates, list) else [measure_candidates or {}] + candidate_desc = "; or ".join( + f"{c.get('type')} '{c.get('id')}'" + (f" (aggregation {c['aggregation']})" if c.get("aggregation") else "") + for c in candidates + ) + prompt = ( + f"You are simulating a user in a conversation with a BI assistant that runs key driver " + f"analysis. The assistant said: '{agent_message}'. " + f"The user is happy to proceed with any of the following: {candidate_desc}. " + f"Reply briefly as the user, picking whichever of those the assistant offered." + ) + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[{"role": "user", "content": prompt}], + max_tokens=150, + ) + return response.choices[0].message.content or "Please proceed with either option." + + +def _extract_kda_calls(tool_call_events: list[ToolCallEvent]) -> tuple[dict | None, dict | None]: + """Return (create_args, execute_result) for the LAST create/execute *pair* -- not the + last create and last execute picked independently. Taking the last pair (not the + first) matches the observed retry-loop behaviour (kda_1 fails, kda_2 retries): the + last attempt is what actually determined the answer the chatbot gave. A new create + call clears any earlier execute_result, since that result belongs to the create it + followed, not to this one -- without that reset, `create_1 -> execute_1(success) -> + create_2 (never executed)` would wrongly pair create_2's args with execute_1's result. + """ + create_args: dict | None = None + execute_result: dict | None = None + for tc in tool_call_events: + if tc.function_name == "create_key_driver_analysis": + create_args = tc.parsed_arguments() + execute_result = None + elif tc.function_name == "execute_key_driver_analysis" and tc.result: + execute_result = tc.parsed_result() + return create_args, execute_result + + +# Trace selection: a conversation can have more than one Langfuse trace sharing the same +# session_id (a conversation-title-generation trace, a disambiguation turn, the actual KDA +# turn), and picking the real one requires checking observations, which are ingested +# asynchronously just like latency -- so this module doesn't try to pick one at all. It +# links whichever trace the default selector finds, exactly like every other skill; +# combo_report.py re-resolves the real KDA trace itself, well after the run, when +# ingestion has settled. + + +@dataclass +class KdaEvaluation: + """Evaluation scores for a single KDA-skill run. + + Scope: this suite currently asserts only that the KDA process runs to completion -- + the tool chain triggers, executes successfully, and the chatbot delivers a final + answer. Per-field correctness (Measure/Date Attribute/Periods/Filters/Summary + matching the expected values) is computed and logged for visibility but + intentionally excluded from ``strict_pass`` -- that verification is scoped to a + follow-up ticket, not this one. + """ + + # Core: gates strict_pass. + kda_triggered: bool + executed: bool + success: bool + turn_completed: bool + + # Informational only: computed and logged, but not required for strict_pass. + measure_correct: bool + date_attribute_correct: bool + analyzed_period_correct: bool + reference_period_correct: bool + filters_correct: bool + summary_correct: bool + + @property + def strict_pass(self) -> bool: + return all([self.kda_triggered, self.executed, self.success, self.turn_completed]) + + +@dataclass +class KdaRunResult: + """Outcome of one run (one conversation, one message) for a KDA case.""" + + conversation_id: str + eval: KdaEvaluation + actual_create_args: dict | None + actual_execute_result: dict | None + + +@dataclass +class AgenticKdaSummary: + """Aggregated outcome of K runs for a KDA case.""" + + run_results: list[KdaRunResult] + pass_at_k: bool + pass_power_k: bool + best: KdaRunResult + + +def _evaluate_run( + create_args: dict | None, + execute_result: dict | None, + turn_completed: bool, + expected: dict, +) -> KdaEvaluation: + kda_triggered = create_args is not None + executed = execute_result is not None + # Checked against the tool's own result, not compared to expected_output -- this + # scope only cares whether KDA itself reported success, not input/output correctness. + success = executed and execute_result.get("success") is True + + # Informational only (see KdaEvaluation docstring) -- still computed so a follow-up + # ticket can promote these to strict_pass without redoing the extraction logic. + measure_correct = kda_triggered and _measure_matches(create_args.get("measure"), expected.get("Measure")) + date_attribute_correct = kda_triggered and create_args.get("date_attribute_id") == expected.get("Date Attribute") + analyzed_period_correct = kda_triggered and create_args.get("analyzed_period") == expected.get("Analyzed Period") + reference_period_correct = kda_triggered and create_args.get("reference_period") == expected.get("Reference Period") + filters_correct = kda_triggered and _filters_match(create_args.get("filters"), expected.get("Filters", [])) + + summary_correct = False + if executed and success: + data = execute_result.get("data") or {} + actual_summary = data.get("summary") or {} + expected_summary = expected.get("Summary") or {} + tolerance = expected_summary.get("absolute_tolerance", _DEFAULT_SUMMARY_ABS_TOLERANCE) + summary_correct = ( + _within_tolerance(actual_summary.get("reference_value"), expected_summary.get("reference_value"), tolerance) + and _within_tolerance( + actual_summary.get("analyzed_value"), expected_summary.get("analyzed_value"), tolerance + ) + and _within_tolerance(actual_summary.get("change"), expected_summary.get("change"), tolerance) + ) + + return KdaEvaluation( + kda_triggered=kda_triggered, + executed=executed, + success=success, + turn_completed=turn_completed, + measure_correct=measure_correct, + date_attribute_correct=date_attribute_correct, + analyzed_period_correct=analyzed_period_correct, + reference_period_correct=reference_period_correct, + filters_correct=filters_correct, + summary_correct=summary_correct, + ) + + +def run_agentic_kda_skill( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, +) -> AgenticKdaSummary: + """Run the KDA-skill agentic evaluation K times and return a summary. + + Each run is normally a single message in a single turn -- the agent_kda_skill + dataset is designed so every question resolves unambiguously -- but if the agent + asks a clarifying question instead of triggering KDA (a title collision, or a + metric-vs-fact form choice), a simulated user reply nudges it forward for up to + ``max_iterations`` turns, so a disambiguation turn doesn't block measuring whether + KDA itself triggers and completes. + """ + run_results: list[KdaRunResult] = [] + client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + + def _run_once(conv_id: str) -> KdaRunResult: + create_args: dict | None = None + execute_result: dict | None = None + turn_completed = False + current_question = question + + for iteration in range(max_iterations): + chat_result = client.send_message(conv_id, current_question) + c_args, e_result = _extract_kda_calls(chat_result.tool_call_events or []) + turn_completed = bool((chat_result.text_response or "").strip()) + if c_args is not None: + create_args, execute_result = c_args, e_result + break + response_text = (chat_result.text_response or "").strip() + if iteration >= max_iterations - 1 or not is_asking_clarification(response_text): + break + try: + current_question = generate_simulated_kda_response(response_text, expected_output.get("Measure")) + except Exception as exc: # noqa: BLE001 -- safety net, not the assertion; end only this run + _log.warning("Simulated KDA user reply failed for conversation %s: %s", conv_id, exc) + break + + ev = _evaluate_run(create_args, execute_result, turn_completed, expected_output) + return KdaRunResult( + conversation_id=conv_id, + eval=ev, + actual_create_args=create_args, + actual_execute_result=execute_result, + ) + + try: + conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() + try: + run_results.append(_run_once(conv_id_0)) + finally: + if initial_conversation_id is None: # only delete conversations we created + client.delete_conversation(conv_id_0) + + for _ in range(1, k): + conv_id = client.create_conversation() + try: + run_results.append(_run_once(conv_id)) + finally: + client.delete_conversation(conv_id) + finally: + client.close() + + pass_at_k = any(r.eval.strict_pass for r in run_results) + pass_power_k = all(r.eval.strict_pass for r in run_results) + best = max( + run_results, + key=lambda r: sum([r.eval.kda_triggered, r.eval.executed, r.eval.success, r.eval.turn_completed]), + ) + return AgenticKdaSummary( + run_results=run_results, + pass_at_k=pass_at_k, + pass_power_k=pass_power_k, + best=best, + ) + + +class KdaSkillAssertionError(AssertionError): + """Raised when a KDA-skill evaluation fails.""" + + __tracebackhide__ = True + + +def evaluate_agentic_kda_skill( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "agent_kda_skill", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, +) -> None: + """Run KDA-skill evaluation, log to Langfuse, and raise KdaSkillAssertionError on failure.""" + from datetime import datetime as _dt # noqa: PLC0415 + from datetime import timezone as _tz # noqa: PLC0415 + + from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client # noqa: PLC0415 + + if langfuse is None: + langfuse = try_make_langfuse_client() + window_start = _dt.now(_tz.utc) + summary = run_agentic_kda_skill( + host=host, + token=token, + workspace_id=workspace_id, + question=question, + expected_output=expected_output, + k=k, + max_iterations=max_iterations, + initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, + ) + + if langfuse is not None and dataset_item_id: + from gooddata_eval.core.agentic._langfuse import ( # noqa: PLC0415 + build_run_context, + find_traces_per_conversation, + log_quality_and_value_scores, + observe, + score_safe, + ) + + run_name_base, run_metadata = build_run_context( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ) + # No custom selector: which trace is the real KDA turn is re-checked later by + # combo_report.py instead (see "Trace selection" above), so this is the same + # default (max-latency) selection every other skill uses. + traces_by_conv = find_traces_per_conversation( + langfuse, + [r.conversation_id for r in summary.run_results], + window_start, + ) + suffix_needed = len(summary.run_results) > 1 + for run_idx, run in enumerate(summary.run_results): + pt = traces_by_conv.get(run.conversation_id) + run_name = f"{run_name_base}_run{run_idx}" if suffix_needed else run_name_base + ev = run.eval + # Gates strict_pass -- current scope is completion only (see KdaEvaluation docstring). + strict_checks = { + "kda_triggered": ev.kda_triggered, + "kda_executed": ev.executed, + "kda_success": ev.success, + "kda_turn_completed": ev.turn_completed, + } + # Informational only -- logged for visibility / a future correctness ticket, + # NOT part of strict_checks/strict_pass. See KdaEvaluation docstring. All + # kda_-prefixed, like the strict scores above: a bare "filters_correct" once + # collided with alert_skill's own filters_correct in gdc-nas's combo_report.py, + # which reads score dicts across skills by name -- prefixing every KDA score + # name (not just the one that happened to collide) closes that off for good. + informational_checks = { + "kda_measure_correct": ev.measure_correct, + "kda_date_attribute_correct": ev.date_attribute_correct, + "kda_analyzed_period_correct": ev.analyzed_period_correct, + "kda_reference_period_correct": ev.reference_period_correct, + "kda_filters_correct": ev.filters_correct, + "kda_summary_correct": ev.summary_correct, + } + # pt can be any trace of the conversation, not necessarily the KDA turn + # (see "Trace selection" above) -- only trust it as a rough approximation + # (this feeds value_score, not the daily report) when KDA triggered at all. + kda_latency_sec = pt.latency if pt and ev.kda_triggered else None + _log.info("[kda-report] %s: strict_pass=%s latency_sec=%s", run_name, ev.strict_pass, kda_latency_sec) + with observe(langfuse, pt.id if pt else None, dataset_item_id, run_name, run_metadata) as tid: + for score_name, value in {**strict_checks, **informational_checks}.items(): + score_safe(langfuse, tid, name=score_name, value=float(value), data_type="BOOLEAN") + # Mirrors visualization.py's own pass_at_K/pass_power_K logging -- pass_power_k + # is otherwise computed and never read anywhere, discarding exactly the + # cross-run consistency signal that raising k is meant to produce. + score_safe(langfuse, tid, name=f"pass_at_{k}", value=float(summary.pass_at_k), data_type="BOOLEAN") + score_safe( + langfuse, tid, name=f"pass_power_{k}", value=float(summary.pass_power_k), data_type="BOOLEAN" + ) + log_quality_and_value_scores( + langfuse, + tid, + strict_checks=strict_checks, + latency_sec=kda_latency_sec, + cost_usd=pt.total_cost if pt and ev.kda_triggered else None, + ) + + if not summary.pass_at_k: + best = summary.best + ev = best.eval + message = ( + f"KDA skill assertion failed. strict_pass={ev.strict_pass} " + f"(kda_triggered={ev.kda_triggered}, executed={ev.executed}, " + f"success={ev.success}, turn_completed={ev.turn_completed}). " + f"Informational only, not part of strict_pass: " + f"measure_correct={ev.measure_correct}, date_attribute_correct={ev.date_attribute_correct}, " + f"analyzed_period_correct={ev.analyzed_period_correct}, " + f"reference_period_correct={ev.reference_period_correct}, " + f"filters_correct={ev.filters_correct}, summary_correct={ev.summary_correct}. " + f"Actual create args: {best.actual_create_args}. " + f"Actual execute result: {best.actual_execute_result}." + ) + raise KdaSkillAssertionError(message) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 2e2b5b9b1..2471e7db8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -10,6 +10,7 @@ from gooddata_sdk import GoodDataSdk +from gooddata_eval.core.agentic._clarification import is_asking_clarification from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.models import ToolCallEvent @@ -165,13 +166,6 @@ def _delete_metric(sdk: GoodDataSdk, workspace_id: str, metric_id: str) -> None: print(f"[CLEANUP] Failed to delete metric {metric_id}: {exc}") -def _is_asking_clarification(text: str) -> bool: - if not text: - return False - t = text.lower() - return "?" in t or "could you" in t or "please provide" in t or "clarif" in t - - def _execute_single_metric_run( client: ChatClient, sdk: GoodDataSdk, @@ -203,7 +197,7 @@ def _execute_single_metric_run( metric_id_to_delete = candidate.get("metric_id") break response_text = (chat_result.text_response or "").strip() - if _is_asking_clarification(response_text): + if is_asking_clarification(response_text): current_question = generate_simulated_response(response_text, primary_expected) else: break diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py new file mode 100644 index 000000000..50f9a3c42 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -0,0 +1,532 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +import json +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic.kda_skill import ( + KdaEvaluation, + KdaSkillAssertionError, + _extract_kda_calls, + _filters_match, + _measure_matches, + _normalize_measure, + _to_number, + _within_tolerance, + evaluate_agentic_kda_skill, + run_agentic_kda_skill, +) +from gooddata_eval.core.models import ChatResult + +_EXPECTED = {"Measure": {"type": "metric", "id": "revenue"}} + + +def _tool_call(name: str, result: dict | None = None, arguments: dict | None = None): + return { + "functionName": name, + "functionArguments": "{}" if arguments is None else json.dumps(arguments), + "result": None if result is None else json.dumps(result), + } + + +def _kda_chat_result(*, success: bool = True, text: str = "Here is the analysis.") -> ChatResult: + return ChatResult.model_validate( + { + "textResponse": text, + "toolCallEvents": [ + _tool_call("create_key_driver_analysis", arguments={"measure": {"type": "metric", "id": "revenue"}}), + _tool_call("execute_key_driver_analysis", result={"success": success, "data": {"summary": {}}}), + ], + "reasoningStepCount": 1, + } + ) + + +def _no_kda_chat_result(text: str = "I could not find that metric.") -> ChatResult: + return ChatResult.model_validate({"textResponse": text, "toolCallEvents": [], "reasoningStepCount": 1}) + + +# --------------------------------------------------------------------------- # +# Pure helpers +# --------------------------------------------------------------------------- # +def test_to_number_int(): + assert _to_number("42") == 42 + + +def test_to_number_float(): + assert _to_number("4.5") == 4.5 + + +def test_to_number_none_on_garbage(): + assert _to_number("not-a-number") is None + assert _to_number(None) is None + + +def test_normalize_measure(): + assert _normalize_measure({"type": "metric", "id": "revenue", "aggregation": "SUM"}) == ( + "metric", + "revenue", + "SUM", + ) + + +def test_measure_matches_single_candidate(): + assert _measure_matches({"type": "metric", "id": "revenue"}, {"type": "metric", "id": "revenue"}) is True + + +def test_measure_matches_list_of_candidates(): + actual = {"type": "fact", "id": "order_value", "aggregation": "SUM"} + expected = [{"type": "metric", "id": "revenue"}, {"type": "fact", "id": "order_value", "aggregation": "SUM"}] + assert _measure_matches(actual, expected) is True + + +def test_measure_matches_false_when_actual_not_a_dict(): + assert _measure_matches("revenue", {"type": "metric", "id": "revenue"}) is False + + +def test_measure_matches_false_when_expected_none(): + assert _measure_matches({"type": "metric", "id": "revenue"}, None) is False + + +def test_filters_match_equal_ignores_key_order(): + assert _filters_match([{"b": 2, "a": 1}], [{"a": 1, "b": 2}]) is True + + +def test_filters_match_false_on_mismatch(): + assert _filters_match([{"a": 1}], [{"a": 2}]) is False + + +def test_filters_match_treats_none_actual_as_empty_list(): + assert _filters_match(None, []) is True + + +def test_filters_match_false_on_non_serializable_value(): + assert _filters_match([{"a", "not json serializable"}], []) is False + + +def test_within_tolerance_true(): + assert _within_tolerance(100.0, 100.5, 1.0) is True + + +def test_within_tolerance_false_when_exceeds(): + assert _within_tolerance(100.0, 105.0, 1.0) is False + + +def test_within_tolerance_false_on_non_numeric(): + assert _within_tolerance("n/a", 100.0, 1.0) is False + + +def test_extract_kda_calls_takes_last_execute_on_retry(): + events = ( + _kda_chat_result(success=False).tool_call_events + + ChatResult.model_validate( + { + "toolCallEvents": [ + _tool_call("execute_key_driver_analysis", result={"success": True, "data": {"summary": {}}}), + ], + } + ).tool_call_events + ) + create_args, execute_result = _extract_kda_calls(events) + assert create_args == {"measure": {"type": "metric", "id": "revenue"}} + assert execute_result == {"success": True, "data": {"summary": {}}} + + +def test_extract_kda_calls_does_not_pair_a_new_create_with_an_earlier_execute(): + # create_1 -> execute_1(success) -> create_2 (never executed): create_2's args must + # not get paired with execute_1's stale result -- that would wrongly report the run + # as executed/succeeded when the actual last attempt never ran. + events = ChatResult.model_validate( + { + "toolCallEvents": [ + _tool_call("create_key_driver_analysis", arguments={"measure": {"type": "metric", "id": "a"}}), + _tool_call("execute_key_driver_analysis", result={"success": True, "data": {"summary": {}}}), + _tool_call("create_key_driver_analysis", arguments={"measure": {"type": "metric", "id": "b"}}), + ] + } + ).tool_call_events + create_args, execute_result = _extract_kda_calls(events) + assert create_args == {"measure": {"type": "metric", "id": "b"}} + assert execute_result is None + + +def test_extract_kda_calls_none_when_no_tool_calls(): + create_args, execute_result = _extract_kda_calls([]) + assert create_args is None + assert execute_result is None + + +def test_extract_kda_calls_ignores_execute_call_with_no_result(): + events = ChatResult.model_validate( + {"toolCallEvents": [_tool_call("execute_key_driver_analysis", result=None)]} + ).tool_call_events + _, execute_result = _extract_kda_calls(events) + assert execute_result is None + + +# --------------------------------------------------------------------------- # +# KdaEvaluation.strict_pass +# --------------------------------------------------------------------------- # +def _evaluation(**overrides) -> KdaEvaluation: + fields = { + "kda_triggered": True, + "executed": True, + "success": True, + "turn_completed": True, + "measure_correct": True, + "date_attribute_correct": True, + "analyzed_period_correct": True, + "reference_period_correct": True, + "filters_correct": True, + "summary_correct": True, + } + fields.update(overrides) + return KdaEvaluation(**fields) + + +def test_strict_pass_true_when_all_core_checks_pass(): + assert _evaluation().strict_pass is True + + +def test_strict_pass_false_when_any_core_check_fails(): + assert _evaluation(success=False).strict_pass is False + + +# --------------------------------------------------------------------------- # +# run_agentic_kda_skill +# --------------------------------------------------------------------------- # +def test_run_agentic_kda_skill_triggers_and_succeeds(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _kda_chat_result(success=True) + + with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + ) + + assert summary.pass_at_k is True + assert summary.best.eval.kda_triggered is True + assert summary.best.eval.executed is True + assert summary.best.eval.success is True + mock_client.close.assert_called_once() + + +def test_run_agentic_kda_skill_no_tool_call(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _no_kda_chat_result() + + with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + ) + + assert summary.pass_at_k is False + assert summary.best.eval.kda_triggered is False + + +def test_run_agentic_kda_skill_resolves_after_clarification_turn(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [ + _no_kda_chat_result("Could you clarify which revenue measure you mean?"), + _kda_chat_result(success=True), + ] + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch( + "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", + return_value="The revenue metric is fine.", + ) as mock_simulate, + ): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=2, + ) + + mock_simulate.assert_called_once() + assert summary.pass_at_k is True + assert mock_client.send_message.call_count == 2 + + +def test_run_agentic_kda_skill_gives_up_after_max_iterations_of_clarification(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _no_kda_chat_result("Could you clarify which measure?") + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch( + "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", + return_value="Please use revenue.", + ), + ): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=2, + ) + + assert summary.pass_at_k is False + assert mock_client.send_message.call_count == 2 + + +def test_run_agentic_kda_skill_survives_simulated_reply_failure(): + # The simulated-user helper is a safety net, not the assertion under test -- if it + # raises, only the current run ends early; earlier completed runs are preserved. + mock_client = MagicMock() + mock_client.create_conversation.side_effect = ["conv-1", "conv-2"] + mock_client.send_message.side_effect = [ + _kda_chat_result(success=True), # run 0: triggers KDA immediately + _no_kda_chat_result("Could you clarify which measure?"), # run 1: asks, then helper blows up + ] + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch( + "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", + side_effect=RuntimeError("openai down"), + ), + ): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=2, + max_iterations=2, + ) + + assert len(summary.run_results) == 2 + assert summary.run_results[0].eval.kda_triggered is True + assert summary.run_results[1].eval.kda_triggered is False + assert summary.pass_at_k is True # run 0 still counts + + +def test_run_agentic_kda_skill_uses_initial_conversation_for_run_0(): + mock_client = MagicMock() + mock_client.send_message.return_value = _kda_chat_result(success=True) + with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + initial_conversation_id="existing-conv", + ) + mock_client.create_conversation.assert_not_called() + mock_client.delete_conversation.assert_not_called() + + +def test_run_agentic_kda_skill_creates_fresh_conversations_for_remaining_runs(): + mock_client = MagicMock() + mock_client.create_conversation.side_effect = ["fresh-1", "fresh-2"] + mock_client.send_message.return_value = _kda_chat_result(success=True) + with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=3, + max_iterations=1, + initial_conversation_id="existing-conv", + ) + assert mock_client.create_conversation.call_count == 2 + assert mock_client.delete_conversation.call_count == 2 + + +# --------------------------------------------------------------------------- # +# evaluate_agentic_kda_skill +# --------------------------------------------------------------------------- # +def test_evaluate_agentic_kda_skill_raises_on_failure(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _no_kda_chat_result() + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic._langfuse.try_make_langfuse_client", return_value=None), + pytest.raises(KdaSkillAssertionError), + ): + evaluate_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + langfuse=None, + ) + + +def test_evaluate_agentic_kda_skill_does_not_raise_on_success(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _kda_chat_result(success=True) + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic._langfuse.try_make_langfuse_client", return_value=None), + ): + evaluate_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + langfuse=None, + ) + + +def test_evaluate_agentic_kda_skill_never_treats_fallback_trace_latency_as_kda_latency(): + # Regression test: when KDA never triggered, whatever trace find_traces_per_conversation's + # default (max-latency) selector picks is NOT a real KDA turn -- its latency/cost must not + # be logged as the KDA run's own value_score inputs. + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _no_kda_chat_result() + + fallback_trace = MagicMock(id="fallback-trace", latency=999.0, total_cost=5.0) + mock_langfuse = MagicMock() + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic._langfuse.build_run_context", return_value=("run-base", {})), + patch( + "gooddata_eval.core.agentic._langfuse.find_traces_per_conversation", + return_value={"conv-1": fallback_trace}, + ), + patch("gooddata_eval.core.agentic._langfuse.observe") as mock_observe, + patch("gooddata_eval.core.agentic._langfuse.score_safe"), + patch("gooddata_eval.core.agentic._langfuse.log_quality_and_value_scores") as mock_log_scores, + pytest.raises(KdaSkillAssertionError), + ): + mock_observe.return_value.__enter__.return_value = "fallback-trace" + evaluate_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + langfuse=mock_langfuse, + dataset_item_id="item-1", + ) + + mock_log_scores.assert_called_once() + assert mock_log_scores.call_args.kwargs["latency_sec"] is None + assert mock_log_scores.call_args.kwargs["cost_usd"] is None + + +def test_evaluate_agentic_kda_skill_reports_trace_latency_when_kda_triggered(): + # When KDA did trigger, whatever trace find_traces_per_conversation returns is + # reported as-is -- same as every other skill. Picking the *right* trace among a + # conversation's several is combo_report.py's job now, not this module's (see + # kda_skill.py's "Trace selection" comment) -- there is no "matched" concept here. + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _kda_chat_result(success=True) + + found_trace = MagicMock(id="trace-1", latency=76.0, total_cost=0.02) + mock_langfuse = MagicMock() + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic._langfuse.build_run_context", return_value=("run-base", {})), + patch( + "gooddata_eval.core.agentic._langfuse.find_traces_per_conversation", + return_value={"conv-1": found_trace}, + ), + patch("gooddata_eval.core.agentic._langfuse.observe") as mock_observe, + patch("gooddata_eval.core.agentic._langfuse.score_safe"), + patch("gooddata_eval.core.agentic._langfuse.log_quality_and_value_scores") as mock_log_scores, + ): + mock_observe.return_value.__enter__.return_value = "trace-1" + evaluate_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + langfuse=mock_langfuse, + dataset_item_id="item-1", + ) + + mock_log_scores.assert_called_once() + assert mock_log_scores.call_args.kwargs["latency_sec"] == 76.0 + assert mock_log_scores.call_args.kwargs["cost_usd"] == 0.02 + + +def test_evaluate_agentic_kda_skill_logs_pass_at_k_and_pass_power_k(): + # pass_power_k (did EVERY one of k runs pass, not just one) was computed and never + # logged anywhere -- discarding exactly the cross-run consistency signal raising k is + # meant to produce. Mirrors visualization.py's own pass_at_K/pass_power_K scores. + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _kda_chat_result(success=True) + + found_trace = MagicMock(id="trace-1", latency=10.0, total_cost=0.01) + mock_langfuse = MagicMock() + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic._langfuse.build_run_context", return_value=("run-base", {})), + patch( + "gooddata_eval.core.agentic._langfuse.find_traces_per_conversation", + return_value={"conv-1": found_trace}, + ), + patch("gooddata_eval.core.agentic._langfuse.observe") as mock_observe, + patch("gooddata_eval.core.agentic._langfuse.score_safe") as mock_score_safe, + patch("gooddata_eval.core.agentic._langfuse.log_quality_and_value_scores"), + ): + mock_observe.return_value.__enter__.return_value = "trace-1" + evaluate_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + langfuse=mock_langfuse, + dataset_item_id="item-1", + ) + + logged = {c.kwargs["name"]: c.kwargs["value"] for c in mock_score_safe.call_args_list} + assert logged["pass_at_1"] == 1.0 + assert logged["pass_power_1"] == 1.0 diff --git a/packages/gooddata-eval/tests/test_agentic_langfuse_trace.py b/packages/gooddata-eval/tests/test_agentic_langfuse_trace.py new file mode 100644 index 000000000..bfabdc904 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_langfuse_trace.py @@ -0,0 +1,58 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +from unittest.mock import MagicMock + +from gooddata_eval.core.agentic._langfuse import _TraceObj, log_quality_and_value_scores + + +def test_trace_obj_latency_none_when_missing(): + assert _TraceObj({"id": "t1"}).latency is None + + +def test_trace_obj_latency_none_when_explicitly_null(): + assert _TraceObj({"id": "t1", "latency": None}).latency is None + + +def test_trace_obj_latency_preserves_real_zero(): + # A real 0.0 (start == end) must stay 0.0, not be confused with "unknown". + assert _TraceObj({"id": "t1", "latency": 0.0}).latency == 0.0 + + +def test_trace_obj_latency_preserves_real_value(): + assert _TraceObj({"id": "t1", "latency": 45.3}).latency == 45.3 + + +def _value_score(langfuse: MagicMock) -> float: + for call in langfuse.create_score.call_args_list: + if call.kwargs.get("name") == "value_score": + return call.kwargs["value"] + raise AssertionError("value_score was never logged") + + +def test_value_score_uses_all_three_components_when_all_known(): + langfuse = MagicMock() + log_quality_and_value_scores(langfuse, "t1", strict_checks={"a": True, "b": True}, latency_sec=10.0, cost_usd=0.0) + # speed = 1 - 10/60 ≈ 0.833, cost_factor = 1.0 (cost_usd=0.0) -- same value as before this + # fix (all three known -> weights already summed to 1.0, nothing to renormalize). + assert abs(_value_score(langfuse) - (0.6 * 1.0 + 0.2 * (1 - 10 / 60) + 0.2 * 1.0)) < 0.001 + + +def test_value_score_renormalizes_instead_of_treating_unknown_latency_as_worst(): + langfuse = MagicMock() + log_quality_and_value_scores(langfuse, "t1", strict_checks={"a": True}, latency_sec=None, cost_usd=0.0) + # Unresolved latency drops the 0.2-weighted speed term entirely instead of scoring it 0.0 + # (which would silently drag value_score down for a run that may well have been fast). + # Renormalized over quality(0.6) + cost(0.2) = 0.8. + assert abs(_value_score(langfuse) - (0.6 * 1.0 + 0.2 * 1.0) / 0.8) < 0.001 + + +def test_value_score_renormalizes_instead_of_treating_unknown_cost_as_worst(): + langfuse = MagicMock() + log_quality_and_value_scores(langfuse, "t1", strict_checks={"a": True}, latency_sec=0.0, cost_usd=None) + assert abs(_value_score(langfuse) - (0.6 * 1.0 + 0.2 * 1.0) / 0.8) < 0.001 + + +def test_value_score_is_quality_only_when_latency_and_cost_both_unknown(): + langfuse = MagicMock() + log_quality_and_value_scores(langfuse, "t1", strict_checks={"a": True, "b": False}, latency_sec=None, cost_usd=None) + assert abs(_value_score(langfuse) - 0.5) < 0.001 diff --git a/packages/gooddata-eval/tests/test_clarification.py b/packages/gooddata-eval/tests/test_clarification.py new file mode 100644 index 000000000..228304ab1 --- /dev/null +++ b/packages/gooddata-eval/tests/test_clarification.py @@ -0,0 +1,33 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +import pytest +from gooddata_eval.core.agentic._clarification import is_asking_clarification + + +@pytest.mark.parametrize( + "text", + ["Could you clarify which metric?", "Please provide the date range.", "Did you mean revenue?"], +) +def test_is_asking_clarification_true(text): + assert is_asking_clarification(text) is True + + +def test_is_asking_clarification_false_on_plain_statement(): + assert is_asking_clarification("Here is the key driver analysis result.") is False + + +def test_is_asking_clarification_false_on_empty(): + assert is_asking_clarification("") is False + + +def test_is_asking_clarification_false_when_question_mark_is_not_the_final_answer(): + text = 'The user asked "what changed?" so here is the key driver breakdown they requested.' + assert is_asking_clarification(text) is False + + +def test_is_asking_clarification_true_for_a_phrase_not_at_the_end_of_the_message(): + # Regression guard: only the bare "?"-anywhere check was tightened to require it be at + # the end. "could you"/"please"/"clarif" stay substring-anywhere -- conversation.py's + # multi-turn driver relies on their broader recall for messages that don't end in "?". + text = "Please let me know which region you'd like before I run the analysis." + assert is_asking_clarification(text) is True