
    1j7                    f   d Z ddlmZ ddlZddlZddlZddlZddlmZmZ ddl	m
Z
  e
d      Zedz  Zedz  d	z  Zed
z  dz  Zed
z  ZdZdZddZddZddZddZddZddZ	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 d dZd!dZedk(  r& ej8                   eej:                  dd              yy)"u*  
aria-eval-ci — KAR-745: Automated Eval CI Runner

Findet alle /root/aria/evals/*/golden-dataset.yaml, faehrt aria-eval-judge.py
gegen jedes Dataset mit passendem judge-prompt.md, berechnet P/R/F1 + Delta
zum letzten Lauf, schreibt Markdown-Report, und exitiert mit != 0 bei Regression.

Run-History: /root/aria/state/eval-ci-history.json
Report:      /root/aria/state/eval-ci-report-<TIMESTAMP>.md

Usage:
  python3 aria-eval-ci.py [--dry-run] [--model MODEL] [--max-cases N]
                          [--regression-threshold FLOAT] [--evals-dir DIR]
    )annotationsN)datetimetimezone)Pathz
/root/ariaevalsscriptszaria-eval-judge.pystatezeval-ci-history.jsonzclaude-haiku-4-5-20251001g?c                     t         j                         r(	 t        j                  t         j	                               S dg iS # t        j
                  t        f$ r Y dg iS w xY w)Nruns)HISTORY_FILEexistsjsonloads	read_textJSONDecodeErrorOSError     "/root/aria/scripts/aria-eval-ci.pyload_historyr   '   s`    	::l44677 B< $$g. 	B<	s   &A A A c                    t         j                  j                  dd       t         j                  t	        j
                  | dd             y )NTparentsexist_ok   F)indentensure_ascii)r   parentmkdir
write_textr   dumps)historys    r   save_historyr#   0   s6    dT:DJJwquMNr   c                    t        | j                  dg             D ]B  }|j                  dg       D ]+  }|j                  d      |k(  s|j                  d      c c S  D y)zEReturn the most recent metric snapshot for this dataset key, or None.r   datasetskeymetricsN)reversedget)r"   dataset_keyrundss       r   last_metrics_for_datasetr-   5   s_    FB/0 )''*b) 	)Bvve}+vvi((	)) r   c                    g }t        | j                  d            D ]C  }|j                  }|dz  }|j                         s%|j	                  |j
                  ||d       E |S )z<Return list of {key, dataset_path, judge_prompt_path} dicts.z*/golden-dataset.yamlzjudge-prompt.md)r&   dataset_pathjudge_prompt_path)sortedglobr   r   appendname)	evals_dirr%   ds_pathfeature_dirjudge_prompts        r   find_datasetsr9   B   sq    H)..)@AB nn"%66""$ ###!-
 	 Or   c                   t         j                  t        t              dt        | d         dt        | d         d|dt        |      g
}|r|j	                  d       t        j                  |dd	      }|j                  j                         }|j                  j                         }|j                  d
v }i }	d}
|s|r|j                         D ]q  }|j                  d      r3|j                         }i }|D ]!  }d|v s|j                  dd      \  }}|||<   # t        |j                  dd            }t        |j                  dd            }t        |j                  dd            }t        |j                  dd            }|j                  dd      }|j                  dd      }|dk7  rt!        |      nd}|dk7  rt!        |      nd}d}||||z   }|dkD  rd|z  |z  |z  nd}|||||||||z   |z   |z   dkD  r||z   ||z   |z   |z   z  ndd}	d|v sO|j                  dd      d   j                         }
t | d   ||j                  |	|||
|dS )z
    Invoke aria-eval-judge.py as subprocess.
    Returns a result dict with keys: key, success, metrics, error, runs_path.
    z	--datasetr/   z--judge-promptr0   --model--max-cases	--dry-runT)capture_outputtext)r      Nz[aria-eval-judge]=r@   TPr   TNFPFN	precisionNonerecallr   g        )tptnfpfnrF   rH   f1	pass_ratezsaved tor&   )r&   success
returncoder'   stdoutstderr	runs_pathdry_run)sys
executablestrJUDGE_SCRIPTr3   
subprocessr+   rR   striprS   rQ   
splitlines
startswithsplitintr)   float)r,   model	max_casesrU   cmdresultrR   rS   rP   r'   rT   linepartskvtokenkvrI   rJ   rK   rL   precision_str
recall_strrF   rH   rM   denoms                              r   	run_judgern   X   s    	LSN+,#b!4565s9~C 

;^^F ]]  "F]]  "F6)GG Iw%%' 	BD23

" "Ee|${{321 !1" a)a)a)a) "{F ;VVHf5
4AV4KE-0QU	.8F.Bz*#'(V-?&/E=BQY!i-&058CBBb!*$EG"Wr\TVEVZ[D["r'b2glR.?!@ae T! JJz15b9??A	9	B> %y''	 	r   c                    |si S i }dD ]<  }| j                  |      }|j                  |      }|(|+t        ||z
  d      ||<   > |S )zICompute metric deltas vs previous run. Returns empty dict if no previous.)rF   rH   rM   rN      )r)   round)currentpreviousdeltar&   curprevs         r   compute_deltarw      s_    	E9 .kk#||C ?t/sTz1-E#J	.
 Lr   c                v   g }| D ]  }|d   s
|d   r|d   st        ||d         }t        |d   |      }|j                         D ]n  \  }}|| k  s|j                  d|d    d| dt	        |      dd	|d
d|j                  di       j                  |       d|d   j                  |              p  |S )zFReturn list of ALERT strings for any metric drops exceeding threshold.rU   rP   r'   r&   zALERT: [z] z dropped by .1%z (threshold z.0%u   ) — prev=_prev_metricsz curr=)r-   rw   itemsr3   absr)   )	resultsr"   	thresholdalertsrrv   rt   metricdrops	            r   check_regressionr      s     F Y<q|1Y<'5:a	lD1!KKM 	LFDyj qxj6(,s4yo N""+CAEE/SU<V<Z<Z[a<b;c diL,,V457	 Mr   c                8   t        j                  t        j                        j	                  d      }d|   d| d| d|rdnd  g}|r=|j                  d       |D ]  }|j                  d	|         |j                  d
       |j                  d       |j                  d
       |j                  d       |j                  d       |D ]   }	|	d   }
|	d   r|j                  d|
 d       $|	d   s|j                  d|
 d       ?|	d   }t        ||
      }t        ||      }d5d}d5d}|j                  d|
 d ||j                  d             d ||j                  d             d ||j                  d             d ||j                  d             d ||j                  d             d ||j                  d             d ||j                  d             d       # |j                  d
       |r\|j                  d       |D ]5  }|j                  d       |j                  |       |j                  d       7 |j                  d
       |j                  d       |D ]  }	|j                  d |	d           |	d   r|j                  d!       n|	d   r|	d   }|j                  d"|j                  d#       d$|j                  d%       d&|j                  d'       d(|j                  d)              |	j                  d*      r]|j                  d+|	d*    d,       nD|j                  d-|	d.    d/       |	j                  d0      r|j                  d1|	d0   d d2  d3       |j                  d
        d4j                  |      S )6Nz%Y-%m-%d %H:%M UTCu   # Aria Eval-CI Report — z	**Run:** z  **Model:** z  **Mode:** zdry-runlivez## Regressionsz-  z## Dataset ResultszL| Dataset | Pass% | Precision | Recall | F1 | Delta P | Delta R | Delta F1 |zL|---------|-------|-----------|--------|----|---------|---------|----------|r&   rU   z| u0    | dry-run | — | — | — | — | — | — |rP   u.    | ERROR | — | — | — | — | — | — |r'   c                    | | dS dS )Nry      —r   )rj   s    r   fmtzbuild_report.<locals>.fmt   s    !"aW9E9r   c                (    | y| dk\  rdnd}| | dS )Nr   r   +r   ry   r   )rj   signs     r   dfmtzbuild_report.<locals>.dfmt   s(    yq&3bDVAc7##r   z | rN   rF   rH   rM   z |z## Alert Detailsz```z## Run Detailsz### zMode: dry-run (no LLM calls)zTP=rI   z TN=rJ   z FP=rK   z FN=rL   rT   zRun JSON: ``zERROR (exit rQ   )rS   z```
i  z
```
)rj   zfloat | NonereturnrX   )
r   nowr   utcstrftimer3   r-   rw   r)   join)run_idr}   r"   r   ra   rU   r   linesar   r&   mrv   rt   r   r   s                   r   build_reportr      s    ,,x||
$
-
-.B
CC
$VH-
C5eWLgSY8Z[	E %& 	#ALL2aS"	#R	LL%&	LL	LL_`	LL_` 
hY<LL2cU"RST|LL2cU"PQRiL'5a&	:	$ 	 QUU;'() *QUU;'() *QUU8_%& 'QUU4[!" #eii,-. /eii)*+ ,eiio&'r+		
+
@ 
LL'( 	!ALL3 LLOLL3 	! 	R	LL!" tAeH:&'Y<LL78y\)ALLaeeDk]$quuT{m4d}DtV uu[!{1[>*:!<=LL<,'8:;uuXuQx[#%6$7u=>R" 99Ur   c                   t        j                  t              }|j                  ddd       |j                  dt               |j                  dt
        d	
       |j                  dt        t        d       |j                  dt        t              d       |j                  |       }t        |j                        }t        |      }|st        d| t        j                          y|j"                  rXt        d       |D ]/  }t        d|d   dd|d           t        dddd|d           1 t        dt%        |       d       yt'               }t)        j*                         j-                  d       }t        d!| d"t%        |       d#       g }|D ]  }t        d$|d    d%dd&'       t/        ||j0                  |j2                  d()      }	|j5                  |	       |	d*   r[|	d+   }
t        d,|
j7                  d-d      d.d/|
j7                  d0       d1|
j7                  d2       d3|
j7                  d4              t        d5|	d6    d7        |D ]  }	t9        ||	d         }|xs i |	d8<    t;        |||j<                        }t?        |||||j0                  d()      }t@        d9| d:z  }t@        jC                  d&d&;       |jE                  |d<=       t        d>|        |t)        j*                  tF        jH                        jK                         |j0                  |D 	cg c]  }	|	d   |	d+   |	d*   d? c}	d@}|jM                  dAg       j5                  |       |dA   dBd  |dA<   tO        |       tQ        dC |D              }tQ        dD |D              }t        dE| dF| dGt%        |       dH       |r|D ]  }t        |        yI|dk(  rdS dIS c c}	w )JN)descriptionr=   
store_truez;List datasets and preview prompts WITHOUT making LLM calls.)actionhelpr;   )defaultr<   d   )typer   z--regression-thresholdzNMetric drop fraction that triggers ALERT + non-zero exit (default 0.05 = 5pp).)r   r   r   z--evals-dirz5Root directory to scan for golden-dataset.yaml files.)r   r   z7[eval-ci] No datasets with judge-prompt.md found under )filer   u6   [eval-ci] DRY RUN — listing datasets (no LLM calls):z  r&   40sz
  dataset=r/   r   z
  judge  =r0   z
[eval-ci] z4 dataset(s) found. Pass --dry-run to skip LLM calls.r   z%Y-%m-%d-%H%M%Sz[eval-ci] Starting run u    — z dataset(s)z  [eval-ci] Running: z ...T)endflushF)rU   rP   r'   z pass_rate=rN   ry   z P=rF   z R=rH   z F1=rM   z ERROR (exit rQ   r   rz   zeval-ci-report-z.mdr   zutf-8)encodingz
[eval-ci] Report: )r&   r'   rP   )r   	timestampra   r%   r   ic              3  P   K   | ]  }|d    s	|j                  d      rd   yw)rP   rU   r@   N)r)   .0r   s     r   	<genexpr>zmain.<locals>.<genexpr>  s!     Mqq|AEE)<LMs   
&&&c              3  ,   K   | ]  }|d    r	d  yw)rP   r@   Nr   r   s     r   r   zmain.<locals>.<genexpr>  s     8q1Y<8s   
z[eval-ci] Summary: z dataset(s) OK, z error(s), z	 alert(s)r@   ))argparseArgumentParser__doc__add_argumentDEFAULT_MODELr_   r`   DEFAULT_REGRESSION_THRESHOLDrX   	EVALS_DIR
parse_argsr   r5   r9   printrV   rS   rU   lenr   r   r   r   rn   ra   rb   r3   r)   r-   r   regression_thresholdr   
REPORT_DIRr   r    r   r   	isoformat
setdefaultr#   sum)argvparserargsr5   r%   r,   r"   r   r}   r   r   rv   r   	report_mdreport_path	run_entrypassedfailedr   s                      r   mainr   .  s   $$9F
J  
 	=9
C=
 ,]	   ID  
 T"DT^^$IY'HG	{SZ]ZdZde||FG 	DBBr%yoZ>0B/CDEBr#hj,?)@(ABC	D 	S]O+_`a nG\\^$$%67F	#F85X{
KLG 6%bi[52TJb$**dnneDqY<)AaeeK3C8 9UU;'(AEE(O+<DtO
 M!L/!2!456  ('5:!ZR/(
 gw0I0IJF VWgvtzzSXYI<<KTD19w7	 
./ \\(,,/99; 
 	 xY<Y<
	I vr")))4focd+GFO MGMMF8G88F	x'7x{3v;-W`
ab 	A!H	!1""1
s   4O&
__main__r@   )r   dict)r"   r   r   rG   )r"   r   r*   rX   r   dict | None)r5   r   r   
list[dict])
r,   r   ra   rX   rb   r_   rU   boolr   r   )rr   r   rs   r   r   r   )r}   r   r"   r   r~   r`   r   	list[str])r   rX   r}   r   r"   r   r   r   ra   rX   rU   r   r   rX   )r   r   r   r_   )r   
__future__r   r   r   rZ   rV   r   r   pathlibr   	ARIA_ROOTr   rY   r   r   r   r   r   r#   r-   r9   rn   rw   r   r   r   __name__exitr   r   r   r   <module>r      s7   #    
 ' 		9$';;7"%;; 
+# O
,FZ
"  	4WWW W 	W
 W W 	W|e#P zCHHT#((12,  r   