XidZddlmZddlZddlZddlZddlZddlmZddl m Z m Z ddl m Z ddlmZmZmZmZmZmZddlmZmZmZmZmZdd lmZdd lmZdd lm Z dd l!m"Z"m#Z#m$Z$m%Z%ejLe'Z(Gd deZ)GddedZ*GddeZ+GddedZ,GddZ-ee+e,e.fZ/GddeZ0ee0e.fZ1Gdde-Z2 d'dZ3dZ4d(dZ5Gd d!Z6 d)d"Z7 d*d#Z8 d+d$Z9 d,d%Z:eee e jJe e jHgee+e,ffee;e jJe;e jHgee+e,fffZr?r model_configrrKr'r(r)r2r27s H@E:0E:8#Hn#8!G]!2!%J%: 6ND65=AO:A;59M29659M29!E> )H-L'"#r(r2ceZdZUdZded<y)EvaluationResultszqBatch evaluation results. This makes it easy for your evaluator to return multiple metrics at once. zlist[EvaluationResult]resultsNr!r'r(r)rOrOas $#!r(rOcReZdZdZe d ddZ d ddZy) RunEvaluatorzEvaluator interface class.Ncy)zEvaluate an example.Nr')rJrunexampleevaluator_run_ids r) evaluate_runzRunEvaluator.evaluate_runosr(cKtjfd}tjj d|d{S7w)z#Evaluate an example asynchronously.ctjdi5jcdddS#1swYyxYw)Nr')rhtracing_contextrW)current_contextrVrUrTrJsr)_run_with_contextz5RunEvaluator.aevaluate_run.._run_with_contexts@##6o6 I((g7GH I I Is4=N)rZget_tracing_contextasyncioget_running_looprun_in_executor)rJrTrUrVr]r\s```` @r) aevaluate_runzRunEvaluator.aevaluate_runxsH002 I I--/??FWXXXXsAA A ANNrTrrUOptional[Example]rVOptional[uuid.UUID]rL*Union[EvaluationResult, EvaluationResults])r"r#r$r%rrWrbr'r(r)rRrRls}$&*04 # ###. # 4 ##&*04 Y  Y# Y. Y 4 Yr(rRcHeZdZUdZded< ded< dZded< dZd ed <y) ComparisonEvaluationResultzFeedback scores for the results of comparative evaluations. These are generated by functions that compare two or more runs, returning a ranking or other feedback. rr3z'dict[Union[uuid.UUID, str], SCORE_TYPE]scoresNr<r=z6Optional[Union[str, dict[Union[uuid.UUID, str], str]]]r6)r"r#r$r%r&r=r6r'r(r)riris8 H@ 33459M296FJG CJr(riceZdZdZ d d dZ d ddZ ddZ ddZeddZ d ddZ d dfd Z d dd Z dd Z xZS)DynamicRunEvaluatoraA dynamic evaluator that wraps a function and transforms it into a `RunEvaluator`. This class is designed to be used with the `@run_evaluator` decorator, allowing functions that take a `Run` and an optional `Example` as arguments, and return an `EvaluationResult` or `EvaluationResults`, to be used as instances of `RunEvaluator`. Attributes: func (Callable): The function that is wrapped by this evaluator. ct|\}|rt|\}d fd }t||ddlm}|*|j |||_t |dd|_tj|r8| td|j |||_t |dd|_y|j tttttgt f|||_t |dd|_y) zInitialize the `DynamicRunEvaluator` with a given function. Args: func (Callable): A function that takes a `Run` and an optional `Example` as arguments, and returns a dict or `ComparisonEvaluationResult`. cf|S|jd|jd\}}}|S)NrTrUgetinputs_ traced_inputsprepare_inputss r)process_inputsz4DynamicRunEvaluator.__init__..process_inputss>% $2 5!6::i#8% !Q=! r(rrNrvr"rlFunc was provided as a coroutine function, but afunc was also provided. If providing both, func should be a regular function to avoid ambiguity.rrr9rLr9)_normalize_evaluator_funcr langsmithrensure_traceableafuncgetattr_nameinspectiscoroutinefunction TypeErrorr r rr r_RUNNABLE_OUTPUTfuncrJrr}rvrrus @r)__init__zDynamicRunEvaluator.__init__s(";4!@~ &?&F #UN ! d D)  $55n6DJ! 4IJDJ  & &t , 3 %55^6DJ!z3HIDJ#44XsHW$568HHI4P-5DI!z3HIDJr(c<ttrjs|_S stddvr|r|jd<t fddDrtdtdid|iS#t $r}td|d}~wwxYw) NziExpected an EvaluationResult object, or dict with a metric 'key' and optional 'score'; got empty result: r3c3&K|]}|v ywNr').0kresults r) z@DynamicRunEvaluator._coerce_evaluation_result..sJq1F?J)r4rr6zrExpected an EvaluationResult object, or dict with a metric 'key' and optional 'score' or categorical 'value'; got r=z[Expected an EvaluationResult object, or dict with a metric 'key' and optional 'score'; got r')rDr2r= ValueErrorrallr)rJrr= allow_no_keyes ` r)_coerce_evaluation_resultz-DynamicRunEvaluator._coerce_evaluation_results f. /'''4$M  FFLXOF"| $ u J,IJJ OOUhX$Q &P&PQ Q 44:8=  sAA>> BBBcd|vrB|j}|dDcgc]}|j||c}|d<tdi|S|jtt||dScc}w)NrP)r=T)r=rr')copyrrOr r9)rJrPr=cprs r)_coerce_evaluation_resultsz.DynamicRunEvaluator._coerce_evaluation_resultss  B!+..q .NByM%*r* *-- w }4.  sA)ct|tr|js||_|St|}|j ||Sr)rDr2r=_format_evaluator_resultr)rJrr=s r)_format_resultz"DynamicRunEvaluator._format_resultsB f. /'''4$M)&1..v}EEr(ct|dSzCheck if the evaluator function is asynchronous. Returns: bool: `True` if the evaluator function is asynchronous, `False` otherwise. r}hasattrrIs r)is_asynczDynamicRunEvaluator.is_async$tW%%r(ct|dsPtj}|jr t d|j |j ||S|tj}d|ji}t|ddrt|j|d<|j||||d}|j||S) aEvaluate a run using the wrapped function. This method directly invokes the wrapped function with the provided arguments. Args: run (Run): The run to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: Union[EvaluationResult, EvaluationResults]: The result of the evaluation. rtCannot call `evaluate_run` on an async run evaluator from within an running event loop. Use `aevaluate_run` instead.Nr> session_id experimentrun_idr5langsmith_extra)rr_get_event_loop is_running RuntimeErrorrun_until_completerbuuiduuid4idr~rrrr)rJrTrUrV running_loopr5rs r)rWz DynamicRunEvaluator.evaluate_run-s"tV$"113L&&("R $66t7I7I#w7WXX  ##zz| $3SVV#< 3 d +%(%8H\ "  '7XN  ""6+;<rrrr) rsuperrbrrrr~rrr}r)rJrTrUrVr5r __class__s r)rbz!DynamicRunEvaluator.aevaluate_runSs&tW%.sG<< <  ##zz| $3SVV#< 3 d +%(%8H\ "zz  '7XN"   ""6+;<<=  s" B&B"A'B& B$ B&$B&c&|j||S)aMake the evaluator callable, allowing it to be used like a function. This method enables the evaluator instance to be called directly, forwarding the call to `evaluate_run`. Args: run (Run): The run to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: Union[EvaluationResult, EvaluationResults]: The result of the evaluation. )rW)rJrTrUs r)__call__zDynamicRunEvaluator.__call__ts  g..r(c"d|jdS))Represent the DynamicRunEvaluator object.zrrIs r)__repr__zDynamicRunEvaluator.__repr__s&tzzl!44r(r)rXCallable[[Run, Optional[Example]], Union[_RUNNABLE_OUTPUT, Awaitable[_RUNNABLE_OUTPUT]]]r}zIOptional[Callable[[Run, Optional[Example]], Awaitable[_RUNNABLE_OUTPUT]]])F)rzUnion[EvaluationResult, dict]r= uuid.UUIDrboolrLr2)rPzUnion[dict, EvaluationResults]r=rrLrg)rzMUnion[EvaluationResult, EvaluationResults, dict, str, int, bool, float, list]r=rrLrgrLrrcrd)rTrrUrerVrf)rTrrUrerLrgrLr)r"r#r$r%rrrrpropertyrrWrbrr __classcell__)rs@r)rlrls;, 8J 8J 8J|# -!   < / !  4  " F  F ! F 4 F&&&*04 $= $=#$=. $= 4 $=R&*04 = =#=. =D6://!2/ 3/"5r(rlct|S)zmCreate a run evaluator from a function. Decorator that transforms a function into a `RunEvaluator`. )rlrs r) run_evaluatorrs t $$r(i'c^t|}t|tkDr|dtdz dz}|S)Nz...))reprlen_MAXSIZE)objss r) _maxsize_reprrs1 S A 1v n1  & Hr(ceZdZdZ d d dZed dZ d ddZ d ddZ d ddZ ddZ e dd Z dd Z y)DynamicComparisonRunEvaluatorz4Compare predictions (as traces) from 2 or more runs.Nc t|\}|rt|\}d fd }t||ddlm}|*|j |||_t |dd|_tj|r8| td|j |||_t |dd|_y|j tttttt gt"f|||_t |dd|_y) zInitialize the `DynamicRunEvaluator` with a given function. Args: func (Callable): A function that takes a `Run` and an optional `Example` as arguments, and returns an `EvaluationResult` or `EvaluationResults`. cf|S|jd|jd\}}}|S)NrunsrUrorqs r)rvz>DynamicComparisonRunEvaluator.__init__..process_inputss>% $2 6"FJJy$9% !Q=! r(rrNrwr"rlrxry)$_normalize_comparison_evaluator_funcrr{rr|r}r~rrrrr r rrr r_COMPARISON_OUTPUTrrs @r)rz&DynamicComparisonRunEvaluator.__init__s(("Fd!K~ &J5&Q #UN ! d D)  $55n6DJ! 4IJDJ  & &t , 3 %55^6DJ!z3HIDJ#44!#(9:*,  .5 DI!z3HIDJr(ct|dSrrrIs r)rz&DynamicComparisonRunEvaluator.is_asyncrr(cXt|dsPtj}|jr t d|j |j ||Stj}|j|}|j||||d}|j|||S)zCompare runs to score preferences. Args: runs: A list of runs to compare. example: An optional example to be used in the evaluation. rrrtagsr) rr_rrrr acompare_runsrr _get_tagsr_format_results)rJrrUrr=rrs r) compare_runsz*DynamicComparisonRunEvaluator.compare_runsstV$"113L&&("R $66&&tW5 ~~d#  '4dC  ##FM4@@r(cKt|ds|j||Stj}|j |}|j ||||dd{}|j |||S7w)aEvaluate a run asynchronously using the wrapped async function. This method directly invokes the wrapped async function with the provided arguments. Args: runs (Run): The runs to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: ComparisonEvaluationResult: The result of the evaluation. r}rrN)rrrrrr}r)rJrrUr=rrs r)rz+DynamicComparisonRunEvaluator.acompare_runs s tW%$$T73 3 ~~d#zz  '4dC"   ##FM4@@  sAA: A8!A:c&|j||S)aMake the evaluator callable, allowing it to be used like a function. This method enables the evaluator instance to be called directly, forwarding the call to `evaluate_run`. Args: run (Run): The run to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: ComparisonEvaluationResult: The result of the evaluation. )r)rJrrUs r)rz&DynamicComparisonRunEvaluator.__call__'s  w//r(c"d|jdS)rz>r(cg}|D]^}|jdt|jzt|dds8|jdt|jz`|S)zExtract tags from runs.zrun:rNz experiment:)appendrrr~r)rrrTs r)rz'DynamicComparisonRunEvaluator._get_tags<s` AC KKSVV, -sL$/ MC,??@ A r(ct|tr|js||_|St|tr9t ||Dcic]\}}|j |c}}|j |d}n4t|trd|vr |j |d<nd|}t| tdid|i|Scc}}w#t$r}td||d}~wwxYw)N)rjr3r=r3zXExpected 'dict', 'list' or 'ComparisonEvaluationResult' result object. Received: result=r=zExpected a dictionary with a 'key' and dictionary of scores mappingrun IDs to numeric scores, or ComparisonEvaluationResult object, got r') rDrir=listziprrr9rr)rJrr=rrTr4msgrs r)rz-DynamicComparisonRunEvaluator._format_resultsGs f8 9'''4$M  %;>tV;LMZS%3665=Mzz!.F  %F" $ u -%+I/ S/ ! -"M >J >J >J@&&AEA!A,=A #A@AEA!A,=A #A:AE0!0,=0 #0"?"="!" " $ "r(rct|S)z.Create a comaprison evaluator from a function.)rrs r)comparison_evaluatorrls ) ..r(c d tj jjDcgc]!\}}|j|j k7s |#}}} jjDcgc]-\}}|j tjjur|/c}} |r6t fd|Ds2t|Dcgc] }| vs| c}dk7rd d}t|t fd|Dr|ddgk(rdfStjrF d fd  dfd }td r td n |j|_|fS d fd dfd }td r td n |j|_|fScc}}wcc}}wcc}w)N)rTrUrroutputsreference_outputs attachmentsc32K|]}|vxs|vywrr'rpnameargs_with_defaultssupported_argss r)rz,_normalize_evaluator_func..) GLE^ # Bu0B'B B UInvalid evaluator function. Must have at least one argument. Supported arguments are . Please see https://docs.smith.langchain.com/evaluation/how_to_guides/evaluation/evaluate_llm_application#use-custom-evaluatorsc32K|]}|vxs|vywrr'rs r)rz,_normalize_evaluator_func..)CH>5,>#>>rrTrUc|||r |jni|jxsi|r|jxsini|r|jxsinid}i}g}i}jj D]e\}}||vs |j |j |jfvr|j||n||||<|dvrt||n||||<g|||fSN)rTrUrrrrr)rTrU rrrr parametersitemskindPOSITIONAL_OR_KEYWORDPOSITIONAL_ONLYrr rTrUarg_mapkwargsargsrt param_nameparamsigs r)_prepare_inputsz2_normalize_evaluator_func.._prepare_inputs&07gnnR"{{0b@G7#6#6#<"RBI)>Br  " ),)=)=)?%J!W, ::!77!11*!KK (;<181DF:. *-??*'**=>!(!4&j1V]22r(cJK||\}}}|i|d{S7wrr'rTrUrrrsr rs r)awrapperz+_normalize_evaluator_func..awrappers2%4C$A!vq!4262222 #!#r"c|||r |jni|jxsi|r|jxsini|r|jxsinid}i}g}i}jj D]e\}}||vs |j |j |jfvr|j||n||||<|dvrt||n||||<g|||fSrrrs r)r z2_normalize_evaluator_func.._prepare_inputsr r(c.||\}}}|i|Srr'r s r)wrapperz*_normalize_evaluator_func..wrappers&$3C$A!vqT,V,,r()rTrrUrerLtuple[list, dict, dict])rTrrUrerLrr signaturerrr VAR_KEYWORDdefault Parameteremptyrrrrrr~r" rrpall_argsarrrr rr rs ` @@@@r)rzrzvsN   D !C&)nn&:&:&<X(%!--@WXHX,,. E1 99G--33 3    PX   HDq1C(CD E J11?0@AF G oLT   Tz  & &t , 3 3#4 3( 3> 3 3#4 3! 34,j)&&   o. . 3 3#4 3( 3> - 4,j)%%   _- -kYEs!F:F:2G G Gc d tj jjDcgc]!\}}|j|j k7s |#}}} jjDcgc]-\}}|j tjjur|/c}} |r6t fd|Ds2t|Dcgc] }| vs| c}dk7rd d}t|t fd|Dr|ddgk(rdfStjrF d fd  dfd }td r td n |j|_|fS d fd  dfd }td r td n |j|_|fScc}}wcc}}wcc}w)NrrUrrrrc32K|]}|vxs|vywrr'rs r)rz7_normalize_comparison_evaluator_func..rrrrrc32K|]}|vxs|vywrr'rs r)rz7_normalize_comparison_evaluator_func.. rrrrUc|||r |jni|Dcgc]}|jxsic}|r|jxsinid}i}g}i} jjD]e\}}||vs |j|j |j fvr|j||n||||<|dvrt||n||||<g|||fScc}wNr)rrU rrrrrrrrrr rrUrTrrrrtrrr s r)r z=_normalize_comparison_evaluator_func.._prepare_inputs*!&07gnnR=ABc 1r 1BBI)>Br  " ),)=)=)?%J!W, ::!77!11*!KK (;<181DF:. *-@@*'**=>!(!4&j1V]22+ CC cJK||\}}}|i|d{S7wrr'rrUrrrsr rs r)rz6_normalize_comparison_evaluator_func..awrapperHs2%4D'$B!vq!4262222rr"c|||r |jni|Dcgc]}|jxsic}|r|jxsinid}i}g}i} jjD]e\}}||vs |j|j |j fvr|j||n||||<|dvrt||n||||<g|||fScc}wr#r$r%s r)r z=_normalize_comparison_evaluator_func.._prepare_inputsWr&r'c.||\}}}|i|Srr'r)s r)rz5_normalize_comparison_evaluator_func..wrapperus(%4D'$B!vqT,V,,r()rrrUrerLr)rrrUrerLrrrs ` @@@@r)rrs-SN   D !C&)nn&:&:&<X(%!--@WXHX,,. E1 99G--33 3    PX   HDq1C(CD E J11?0@AF G oLT  Tz  & &t , 3# 3.? 3( 3< 3# 3.? 3# 34,j)&&   _, , 3# 3.? 3( 3< -# -.? -# -4,j)%%   O+ +kYEs!GG2G G  G c@t|tttfrd|i}|S|st d|t|t r't d|Dst d|dd|i}|St|trd|i}|St|tr |St d|) Nr4zdExpected a non-empty dict, str, bool, int, float, list, EvaluationResult, or EvaluationResults. Got c3<K|]}t|tywr)rDr9)rxs r)rz+_format_evaluator_result..s71:a&7sz8Expected a list of dicts or EvaluationResults. Received .rPrzZExpected a dict, str, bool, int, float, list, EvaluationResult, or EvaluationResults. Got ) rDrrFrErrrrr9)rs r)rrs&4,-6"* M);;A( D   FD !777J6(RST V$ M FC 6" M FD ! M &&,X /  r(c d tjjjDcgc]\}}| }}}jjDcgc]-\}}|jtj j ur|/c}}|r6t fd|Ds=t|Dcgc] }|vs| c}dk7rd d}|r |d|dz }t|t fd|Dr|dd gk(rS d fd }td rtd |_ |S|j|_ |Scc}}wcc}}wcc}w) Nrexamplesrrrrc32K|]}|vxs|vywrr'rs r)rz/_normalize_summary_evaluator..rrrrr/z Received arguments c3&K|]}|v ywrr')rrrs r)rz/_normalize_summary_evaluator..s?U.(?rrr2c |||Dcgc]}|jc}|Dcgc]}|jxsic}|Dcgc]}|jxsic}d}i}g} jjD]K\}}||vs |j|j |j fvr|j||D||||<M |i|} t| tr| St| Scc}wcc}wcc}w)Nr1) rrrrrrrrrrDr2r) rr2rUrTrrrrrrrr s r)rz-_normalize_summary_evaluator..wrappers $9ABg7>>B9=>#CKK-2->KS%Tgoo&;&;%T GFD%(^^%9%9%; A! E(zz33--& GJ$78-4Z-@z* A4*6*F&"23 +F3 3)C>%TsC+C0 C5 r")rzSequence[schemas.Run]r2zSequence[schemas.Example]rLrg) rrrrrrrrrrrr~r") rrrrrrrrr rs ` @@@r)_normalize_summary_evaluatorr6sSN   D !C&)nn&:&:&<=(%=H=,,. E1 99G--33 3    PX   HDq1C(CD E J11?0@ C   )(15 5Co?h? ?8PD  4' 43L 4 7 4:*1z)BGD* % IPHXHX w>Es E %2E; EE)rr)rr)rrrLr)rr rLztuple[Union[Callable[[Run, Optional[Example]], _RUNNABLE_OUTPUT], Callable[[Run, Optional[Example]], Awaitable[_RUNNABLE_OUTPUT]]], Optional[Callable[..., dict]]])rr rLztuple[Union[Callable[[Sequence[Run], Optional[Example]], _COMPARISON_OUTPUT], Callable[[Sequence[Run], Optional[Example]], Awaitable[_COMPARISON_OUTPUT]]], Optional[Callable[..., dict]]])rz;Union[EvaluationResults, dict, str, int, bool, float, list]rLzUnion[EvaluationResults, dict])rr rLSUMMARY_EVALUATOR_T)>r% __future__rr_rloggingrabcrcollections.abcrr functoolsrtypingrr r r r r pydanticrrrrrtyping_extensionsrr{rrZrlangsmith.schemasrrrr getLoggerr"rGrr+r2rOrRr9rrirrlrrrrrrzrrrr7r6r'r(r)rBsE" /TS''BB   8 $'y' 6Ye 6'y'T" "YY8)+r(