
    1j                        d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	m	Z	m
Z
 ddlmZ  ed      ZddZddZdd	Zd
ZddZddZddZedk(  r& ej,                   eej.                  dd              yy)u  
aria-eval-judge — CLI fuer eval-driven-agent-dev Skill (KAR-75)

Laedt golden-dataset.yaml + judge-prompt.md, faehrt jeden Case durch den Judge
(LLM via Anthropic SDK), berechnet Precision/Recall, schreibt Run-JSON nach
<dataset-dir>/runs/<date>.json.

Usage:
  python3 aria-eval-judge.py     --dataset /root/aria/evals/<feature>/golden-dataset.yaml     --judge-prompt /root/aria/evals/<feature>/judge-prompt.md     [--model claude-haiku-4-5-20251001]     [--max-cases 10]
    )annotationsN)datetimetimezone)Pathz%/root/aria/brain/youtube/.config.yamlc                    t         j                  j                  d      } | r| S 	 t        j                  t
        j                  d            }|j                  di       j                  d      }|r|S 	 y# t        $ r Y yw xY w)u   Liest eval-Judge-Modell aus ARIA_EVAL_MODEL env, dann .config.yaml models.triage_model,
    dann fällt zurück auf claude-haiku-4-5-20251001.ARIA_EVAL_MODELzutf-8)encodingmodelstriage_modelzclaude-haiku-4-5-20251001)osenvirongetyaml	safe_load_ARIA_CONFIG_PATH	read_text	Exception)env_valcfgmodels      %/root/aria/scripts/aria-eval-judge.py_get_default_modelr      s     jjnn./Gnn.88'8JK"%)).9L  '  &s   AA5 5	B Bc                x    | j                         5 }t        j                  |      cd d d        S # 1 sw Y   y xY w)N)openr   r   )pfs     r   	load_yamlr   (   s.    	
 !Q~~a ! ! !s   09c                f   t        j                  |j                  di       dd      }t        j                  |j                  di       dd      }| j                  d|      j                  d|      j                  d|      j                  d	|      j                  d
d      j                  dd      S )zLReplace {{input}}, {{expected}} placeholders with case data as JSON strings.input   Findentensure_asciiexpectedz{{input_text_or_json}}z	{{input}}z {{expected_verdict_and_reasons}}z{{expected}}z{{actual_output}}u1   (see input — actual = input for unit-eval mode)z
{{actual}})jsondumpsr   replace)templatecase
input_jsonexpected_jsons       r   render_promptr,   -   s    DHHWb1!%PJJJtxx
B7PUVM	):	6	j	)	3]	C		/	$&Y	Z	R	S    zDu bist ein strenger, objektiver Eval-Judge fuer KI-Agenten-Outputs. Antworte ausschliesslich als JSON mit den Feldern: verdict, scores, reason, confidence.c                   	 | j                   j                  |ddt        ddidgd|dg      }dj                  d |j                  D              }d }	 d|v ra|j                  d      }|D ]K  }|j                         }|j                  d      r|dd  j                         }|j                  d      sI|} n |j                  d      }	|j                  d      }
|	dk7  r |
dk7  rt        j                  ||	|
dz          }||j                   j"                  |j                   j$                  dfS # t        $ r}d t	        |      d	d	d
fcY d }~S d }~ww xY w# t        j                  t        f$ r d }Y sw xY w)Ni,  texttype	ephemeral)r0   r/   cache_controluser)rolecontent)r   
max_tokenssystemmessagesr   )errorinput_tokensoutput_tokens c              3  N   K   | ]  }t        |d       s|j                    yw)r/   N)hasattrr/   ).0bs     r   	<genexpr>zcall_judge.<locals>.<genexpr>R   s     Fa71f3E166Fs   %%z```r%      {}   )r:   r;   )r8   create_JUDGE_SYSTEM_PROMPTr   strjoinr5   splitstrip
startswithfindrfindr%   loadsJSONDecodeError
ValueErrorusager:   r;   )clientr   promptresper/   parsedpartsr   firstlasts              r   
call_judger\   B   s   N%% #0&,k%:  &&9: & 
 77F4<<FFDFD=JJu%E $GGI<<'!"A<<$De$ 		#tzz#B;42:ZZU46 23F 

//11  %  Ns1vq1MMMN    *- s7   .D0 AE 5AE 0	E9E	EEE54E5c                $    | dz  dz  |dz  dz  z   S )Ni@B g      ?g      @ )in_tokout_toks     r   
haiku_costra   h   s!    Y#%9)<(CCCr-   c                   t        j                         }|j                  dd       |j                  dd       |j                  dt                      |j                  dt        d	       |j                  d
d       |j                  |       }t        |j                        }t        |j                        }|j                         r|j                         s"t        d| d| t        j                         yt        |      }|j                         }|j                  dg       d |j                    }|st        d       yt"        j$                  j                  d      }|s(|j&                  st        dt        j                         y|j&                  sddlm}	  |	|      }
nd }
|j,                  dz  }|j/                  d       t1        j2                         j5                  d      }|| dz  }g }dx}x}x}}d}|D ]c  }t7        ||      }|j&                  rCt        d|d    d       t        |d d         t        d!       |j9                  |d   dd"       _t;        |
|j<                  |      \  }}t?        |d#   |d$         }||z  }|j                  d%i       j                  d&d'      }|xs i j                  d&d(      }||k(  }|d)k(  r|d)k(  r|d*z  }n|d*z  }n|d'k(  r|d*z  }n
|d)k(  r|d*z  }|j9                  |d   ||||xs i j                  d+      |xs i j                  d,      |xs i j                  d-      ||d.	       tA        jB                  d/       f ||z   r|||z   z  nd }||z   r|||z   z  nd }||j<                  tE        |      tG        |      ||||||||t1        j2                  tH        jJ                        jM                         d0}|jO                  tQ        jR                  |dd12             t        d3| d4| d5| d6| d7| d8| d9| d:|d;       t        d<|        ||z   dk(  rdS d*S )=Nz	--datasetT)requiredz--judge-promptz--model)defaultz--max-casesd   )r0   rd   z	--dry-run
store_true)actionz	Missing: z or )filer    casesz	No cases.r   ANTHROPIC_API_KEYz<ANTHROPIC_API_KEY not set. Use --dry-run to preview prompts.)	Anthropic)api_keyruns)exist_okz%Y-%m-%d-%H%M%Sz.jsong        z	--- case idz ---i  z...)case_iddry_runr:   r;   r$   verdictpassr9   failrF   scoresreason
confidence)	rp   expected_verdictactual_verdictmatchru   rv   rw   cost_usdrS   g?)run_idr   datasetcases_totaltptnfpfn	precisionrecallr{   results	timestampFr!   z[aria-eval-judge] run z: TP=z TN=z FP=z FN=z precision=z recall=z cost=$z.4fz  saved to )*argparseArgumentParseradd_argumentr   int
parse_argsr   r}   judge_promptexistsprintsysstderrr   r   r   	max_casesr   r   rq   	anthropicrk   parentmkdirr   nowstrftimer,   appendr\   r   ra   timesleeprI   lenr   utc	isoformat
write_textr%   r&   )argvr   argsdataset_pathprompt_pathr}   r(   ri   rl   rk   rT   runs_dirr|   run_pathr   r   r   r   r   
total_costr)   rU   rX   rS   costrx   ry   rz   r   r   summarys                                  r   mainr   l   sE   !ANN;N.NN#dN3NN9&8&:N;NN=sCN8NN;|N4<<D%Lt(()K (:(:(<	,tK=9

K%G$$&HKK$%5t~~6Ek1jjnn01G4<<LSVS]S]^<<'7+""V+HNNDN!\\^$$%67FVHE**HGBb2J %x.<<Id4j\./&#,%LNNtDzdCD"64::v>%/1GHd
88J377	6J ,B++Iw?#33v%'aa'a6)aDz 0,|((2|((2!<R,,\:

 
	 	

3K%N $&7b2gI "RR27^dF |$5z"B\\(,,/99;G 

715IJ	"6(%t4t4t4t L k&C8HJ K	Kz
"#bQ1%A%r-   __main__rF   )returnrI   )r   r   r   dict)r(   rI   r)   r   r   rI   )r   rI   rU   rI   r   ztuple[dict | None, dict])r_   r   r`   r   r   float)r   z	list[str]r   r   )__doc__
__future__r   r   r%   r   r   r   r   r   r   pathlibr   r   r   r   r,   rH   r\   ra   r   __name__exitr   r^   r-   r   <module>r      s    # * * * ' @A ' !
 ^ #LDb&J zCHHT#((12,  r-   