Skip to content

datarobot_genai.dragent.eval.scorer_factory

scorer_factory

Build moderation OOTB scorers from NAT EvalBuilder judge LLMs.

Each custom NAT evaluator plugin should stay thin (config + evaluate_item). This module centralizes how a workflow llm_name is resolved and mapped onto the same scorer types used by moderation_config.yaml OOTB guards.

LitellmJudgeTarget dataclass

litellm.acompletion model id and kwargs for in-process LLM judges.

Source code in datarobot_genai/dragent/eval/scorer_factory.py 
39
40
41
42
43
44
@dataclass(frozen=True)
class LitellmJudgeTarget:
    """``litellm.acompletion`` model id and kwargs for in-process LLM judges."""

    model: str
    completion_kwargs: dict[str, Any]

resolve_langchain_judge_llm async 

resolve_langchain_judge_llm(builder: EvalBuilder, llm_name: str) -> object

Resolve llm_name from the eval/workflow config to a LangChain chat model.

Source code in datarobot_genai/dragent/eval/scorer_factory.py 
47
48
49
50
51
async def resolve_langchain_judge_llm(builder: EvalBuilder, llm_name: str) -> object:
    """Resolve ``llm_name`` from the eval/workflow config to a LangChain chat model."""
    from nat.builder.framework_enum import LLMFrameworkEnum

    return await builder.get_llm(llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN)

build_litellm_judge_target async 

build_litellm_judge_target(builder: EvalBuilder, llm_name: str) -> LitellmJudgeTarget

Map the workflow judge LLM to a litellm target (agent goal accuracy, etc.).

Source code in datarobot_genai/dragent/eval/scorer_factory.py 
54
55
56
57
58
async def build_litellm_judge_target(builder: EvalBuilder, llm_name: str) -> LitellmJudgeTarget:
    """Map the workflow judge LLM to a litellm target (agent goal accuracy, etc.)."""
    llm = await resolve_langchain_judge_llm(builder, llm_name)
    model, completion_kwargs = langchain_chat_model_to_litellm(llm)
    return LitellmJudgeTarget(model=model, completion_kwargs=completion_kwargs)

build_faithfulness_evaluator async 

build_faithfulness_evaluator(builder: EvalBuilder, llm_name: str) -> FaithfulnessEvaluator

LlamaIndex faithfulness judge (same wiring as OOTBFaithfulnessGuard).

Source code in datarobot_genai/dragent/eval/scorer_factory.py 
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
async def build_faithfulness_evaluator(
    builder: EvalBuilder, llm_name: str
) -> FaithfulnessEvaluator:
    """LlamaIndex faithfulness judge (same wiring as ``OOTBFaithfulnessGuard``)."""
    from datarobot_dome._import_utils import require_extra

    llm = await resolve_langchain_judge_llm(builder, llm_name)

    try:
        from llama_index.core import Settings
        from llama_index.core.evaluation import FaithfulnessEvaluator
    except ImportError as e:
        raise require_extra("llama-index-core", "llm-eval", e) from e

    llamaindex_llm = wrap_langchain_judge_for_llamaindex(llm)
    Settings.llm = llamaindex_llm
    Settings.embed_model = cast(Any, None)
    return FaithfulnessEvaluator()

build_task_adherence_scorer async 

build_task_adherence_scorer(builder: EvalBuilder, llm_name: str) -> TaskCompletionMetric

DeepEval task completion judge (same wiring as OOTBTaskAdherenceGuard).

Source code in datarobot_genai/dragent/eval/scorer_factory.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
async def build_task_adherence_scorer(builder: EvalBuilder, llm_name: str) -> TaskCompletionMetric:
    """DeepEval task completion judge (same wiring as ``OOTBTaskAdherenceGuard``)."""
    from datarobot_dome._import_utils import require_extra

    llm = await resolve_langchain_judge_llm(builder, llm_name)

    try:
        from datarobot_dome._deepeval_adapter import ModerationDeepEvalLLM
    except ImportError as e:
        raise require_extra("deepeval", "llm-eval", e) from e
    try:
        from deepeval.metrics import TaskCompletionMetric
    except ImportError as e:
        raise require_extra("deepeval", "llm-eval", e) from e

    deepeval_llm = ModerationDeepEvalLLM(llm)
    return TaskCompletionMetric(model=deepeval_llm, include_reason=True)

build_guideline_adherence_scorer async 

build_guideline_adherence_scorer(builder: EvalBuilder, llm_name: str, agent_guideline: str) -> GuidelineEvaluator

LlamaIndex guideline judge (same wiring as OOTBAgentGuidelineAdherence).

Source code in datarobot_genai/dragent/eval/scorer_factory.py 
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
async def build_guideline_adherence_scorer(
    builder: EvalBuilder,
    llm_name: str,
    agent_guideline: str,
) -> GuidelineEvaluator:
    """LlamaIndex guideline judge (same wiring as ``OOTBAgentGuidelineAdherence``)."""
    from datarobot_dome._import_utils import require_extra

    if not agent_guideline.strip():
        raise ValueError("agent_guideline is required for guideline adherence evaluation")

    llm = await resolve_langchain_judge_llm(builder, llm_name)

    try:
        from llama_index.core.evaluation import GuidelineEvaluator
        from llama_index.core.evaluation.guideline import EvaluationData
        from llama_index.core.output_parsers import PydanticOutputParser
    except ImportError as e:
        raise require_extra("llama-index-core", "llm-eval", e) from e

    class _FixedPydanticOutputParser(PydanticOutputParser):
        def format(self, query: str) -> str:
            return query + "\n\n" + self.get_format_string(escape_json=False)

    langchain_llm = wrap_langchain_judge_for_llamaindex(llm)
    return GuidelineEvaluator(
        llm=langchain_llm,
        guidelines=agent_guideline,
        output_parser=_FixedPydanticOutputParser(output_cls=EvaluationData),
    )