
    LNj5                    @   d Z ddlmZ ddlZej
                  j                  dd       ddlmZ	  e	d      Z
ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlmZmZmZ ddlmZ ddlZ ed	      Z ed
      Z ed      Z ed      ZdZg dZd&dZ d'dZ!d(dZ"d)dZ#d*dZ$d+dZ%d Z&d,dZ'd-dZ(d.dZ)d/dZ*d0dZ+d1dZ,d2dZ-d Z.d Z/d Z0d  Z1d3d!Z2d" Z3e4d#k(  r7e
jk                  d$        ejl                   e3ejn                  d%d              yy)4u  
Aria Knowledge Pipeline (AKP) — Podcast-Audio-Connector Stage 1 (KAR-85)

Liest Podcast-RSS-Feed-Liste, pullt neue Episoden, downloaded Audio temporaer,
transcribed via groq-whisper, schreibt JSON nach brain/akp/raw/podcast/.

Cheap-Filter (Audio-spezifisch):
- duration 20-180 min (skip kurze Trailer + extra-lange Marathons)
- age <14 days
- title-blocklist + Werbung-Patterns
- skip wenn Show in `youtube-channel-subscriptions.md` parallel abgedeckt
  (z.B. Latent Space hat YouTube-Version, vermeidet Doppel-Ingest)

Cost-Cap (separat von Triage/Deep):
- groq-whisper-large-v3-turbo: $0.04/h Audio
- Bei 5 Episoden/Tag á 60 min = $0.20/Tag
- Daily-Cap konfigurierbar in .config.yaml
    )annotationsNz/root/aria/lib)
get_loggerzaria-akp-podcast-ingest)datetimetimezone	timedelta)Pathz%/root/aria/brain/youtube/.config.yamlz7/root/aria/brain/05-Referenzen/podcast-subscriptions.mdz /root/aria/brain/akp/raw/podcastz+/root/aria/scripts/aria-transcribe-route.pyg      ?)z\bsponsor\bz\bsponsored by\bz\bbrought to you by\bc                     t         j                         5 } t        j                  |       cd d d        S # 1 sw Y   y xY wN)CONFIG_PATHopenyaml	safe_load)fs    -/root/aria/scripts/aria-akp-podcast-ingest.pyload_configr   0   s1    				 !q~~a ! ! !s   4=c                Z    t        j                  |       }t         j                  |_        |S r
   )sqlite3connectRowrow_factory)db_pathcons     r   db_connr   5   s     
//'
"CkkCOJ    c                   g }t        j                  d      }| j                         sg S | j                         j	                         D ]S  }|j                  |j                               }|s%|j                  |j                  d      j                  d             U t               }g }|D ])  }||v r|j                  |       |j                  |       + |S )Nz^-\s+(https?://\S+)   z,;))recompileexists	read_text
splitlinesmatchstripappendgrouprstripsetadd)memory_pathouturl_relinemseendedupedus           r   parse_feedsr1   ;   s    
CZZ./F	%%'224 1LL&JJqwwqz((/01 5DB' '9hW^^A&' Nr   c                    t        j                  dd|       }t        j                  dd|      j                  d      j                         }|d d S )Nz	https?:// z[^\w]+-<   )r   subr#   lower)urlss     r   	feed_slugr:   K   sE    
|R%A
y#q!'',224ASb6Mr   c                L   dD ]G  }t        | |d       }|st        j                  |j                               j	                         d d c S  t        | dd      xs d}t        | dd      xs d}t        j                  ||z   j                               j	                         d d S )N)idguid   linkr3   title)getattrhashlibsha1encode	hexdigest)entrykeyvr?   r@   s        r   
episode_idrI   Q   s     =E3%<<
+557<<= 5&"%+DE7B'-2E<<--/0::<SbAAr   c                j   | j                  dg       xs g D ]f  }t        |t              s|j                  d      xs d}|j                  d      xs |j                  d      }|j                  d      s_|sb|dfc S  | j                  dd      }d}|r|j	                  d	      }	 |D cg c]  }t        |       }}t        |      d
k(  r|d   dz  |d   dz  z   |d   z   }n0t        |      dk(  r|d   dz  |d   z   }nt        |      dk(  r|d   }d|fS d|fS c c}w # t        $ r Y d|fS w xY w)z6Return (audio_url, duration_seconds_estimate or None).
enclosurestyper3   hrefr8   zaudio/Nitunes_duration:   r     r   r5      )get
isinstancedict
startswithsplitintlen
ValueError)rF   enctrM   dur_strdurationpartsps           r   find_audio_urlra   [   sO   yyr*0b "c4 %2A776?4cggenD||H%$Tz!" ii)2.GHc"		%*+SV+E+5zQ 8d?U1X]:U1XEUq 8b=583Uq 8 >4> ,  	>	s%   (D# ,D>AD# D# #	D21D2c                    dD ]4  }| j                  |      }|s	 t        |d d dt        j                  ic S  y # t        t
        f$ r Y Iw xY w)N)published_parsedupdated_parsed   tzinfo)rS   r   r   utc	TypeErrorrZ   )rF   rG   rH   s      r   episode_published_atri   v   s^    5 IIcN2A<x||<<	  z* s   ;AAc                2   | j                  dd      xs d}t        |       }|r4t        j                  t        j
                        |z
  t        d      kD  ry||dk  ry|dkD  ry	t        D ])  }t        j                  ||t        j                        s) y
 y )Nr@   r3      )daystoo_oldi  too_short_lt20mini0*  too_long_gt3had_title)rS   ri   r   nowr   rg   r   AD_PATTERNS_TITLEr   search
IGNORECASE)rF   r^   r@   pubpats        r   cheap_filterrw      s    IIgr"(bE
u
%C
X\\*S0I24FFg&h"  99S%/ r   c                R   |dz  }ddt        |      dddddg}|r|d	|gz  }|j                  |        	 t        j                  |d
d
d      }t        |j                  d            }|r|d   S |j                  dk(  r|j                         r|S y# t        j                  $ r Y yw xY w)zIUse yt-dlp to download audio (handles redirects, partial fetch, formats).zepisode.mp3zyt-dlpz-oz--no-warningsz--no-playlistz--quietz--socket-timeout120z--proxyTiX  capture_outputtexttimeoutNz	episode.*r   )	strr$   
subprocessrunTimeoutExpiredlistglob
returncoder   )	audio_urltmp_dirproxyout_pathcmdr
candidatess          r   download_audior      s    &HT3x=/I'95BC	5!!JJyNN3t$L gll;/0J!}||qX__. $$ s   B B&%B&c                    	 t        j                  dt        t              t        |       gddd      }|j
                  dk7  ry|j                  j                         }|r|S dS # t         j                  $ r Y yw xY w)uM  Call aria-transcribe-route.py — duration-based router (Groq ≤60s, Transkriptor >60s).

    Podcasts sind typischerweise 20-180 min → routet immer zu Transkriptor (Pro-Plan flat-rate,
    bessere Quality + Speaker-Diarization). Falls Transkriptor failed → automatic Fallback
    auf Groq (im Router selbst implementiert).
    python3Ti  rz   Nr   )r   r   r~   WHISPER_SCRIPTr   r   stdoutr#   )
audio_pathr   r|   s      r   
transcriber      sw    NNN+S_=dD
 	||q88>>D4!T! $$ s   2A$ $A:9A:c                H    | j                  d|f      j                         d uS )Nz'SELECT 1 FROM ingested WHERE video_id=?)executefetchone)r   eids     r   already_ingestedr      s%    ;;@3&IRRT\```r   c                    | j                  d      D cg c]  }|d   	 }}d|vr"| j                  d       | j                          yyc c}w )u   Fix 09.07.2026: whisper_daily_usd verglich gegen total_usd (= triage+deep+
    whisper gemischt) — ein teurer Deep-Tag (>3$) hat damit den Podcast-Ingest
    ausgehungert, obwohl 0$ Whisper verbraucht waren. Eigene Spalte trennt das.zPRAGMA table_info(daily_costs)r   whisper_usdz=ALTER TABLE daily_costs ADD COLUMN whisper_usd REAL DEFAULT 0Nr   commit)r   r   colss      r   ensure_whisper_columnr      sN     ++&FGHQAaDHDHD ST

 ! Is   A	c                   t        |        t        j                  t        j                        j                         j                         }| j                  d|f      j                         }|r|d   t        |d         S dS )Nz0SELECT whisper_usd FROM daily_costs WHERE date=?r   g        )
r   r   rq   r   rg   date	isoformatr   r   float)r   todayrows      r   todays_whisper_costr      sm    #LL&++-779E
++:UHhj   CF$65Q=?C?r   c                    t        |        t        j                  t        j                        j                         j                         }| j                  d|||||f       | j                          y )NzINSERT INTO daily_costs (date, triage_usd, deep_usd, total_usd, whisper_usd) VALUES (?,0,0,?,?) ON CONFLICT(date) DO UPDATE SET total_usd=total_usd+?, whisper_usd=whisper_usd+?)	r   r   rq   r   rg   r   r   r   r   )r   deltar   s      r   add_whisper_costr      sY    #LL&++-779EKK	[	ueU+
 JJLr   c                   t         |z  }|j                  dd       ||  dz  }t        |      }d|  |||j                  dd      |j                  dd      xs ||||r|j	                  d      nd|r|j                         nd|t        |      t        j                  t        j                        j                         d	d
}	|j                  t        j                  |	dd             t        |      S )NT)parentsexist_okz.jsonpod_r@   r3   r?   z%Y%m%dpodcast)video_idchannel_slugchannel_handler@   r8   r   duration_secondsupload_datepublished_at
transcripttranscript_charsingested_atsourcerR   F)indentensure_ascii)PODCAST_RAW_DIRmkdirri   rS   strftimer   rY   r   rq   r   rg   
write_textjsondumpsr~   )
r   slugrF   r   r^   r   out_dirr   ru   payloads
             r   	write_rawr      s    $GMM$M.C5&H
u
%C3%L7B'yy$1	$14s||H-"+.B 
O||HLL1;;=G 

715IJx=r   c	                f    | j                  dd| |||d d |||||f	       | j                          y )NzINSERT OR REPLACE INTO ingested (video_id, channel, channel_handle, title, duration_seconds, published_at, url, transcript_chars, raw_path, source) VALUES (?,?,?,?,?,?,?,?,?,'podcast')r   i,  r   )	r   r   r   r@   r8   raw_pathcontent_charsr^   pub_isos	            r   store_ingestedr      sE    KK	/ utT5#;'3		"	 JJLr   c                ^    | j                  dd| ||d d ||f       | j                          y )NzsINSERT OR IGNORE INTO ingested (video_id, channel, title, url, skipped_reason, source) VALUES (?,?,?,?,?,'podcast')r      r   )r   r   r   r@   r8   reasons         r   store_skippedr      s6    KK}utU4C[#v6 JJLr   c                \    | j                  d      }| j                          |j                  S )NzHINSERT INTO run_log (stage, status) VALUES ('ingest-podcast', 'running'))r   r   	lastrowid)r   curs     r   log_run_startr     s$    
++`
aCJJL==r   c                R    | j                  d|||||f       | j                          y )NzUPDATE run_log SET finished_at=datetime('now'), status='completed', items_processed=?, items_skipped=?, items_failed=?, notes=? WHERE id=?r   )r   run_id	processedskippedfailednotess         r   log_run_finishr     s-    KK	Q	GVUF3
 JJLr   c                   t               }t        |d   d         }t        t              }|st	        d       yt
        j                  j                  d      st	        dt        j                         yt
        j                  j                  d      d	k(  rt
        j                  j                  d
      nd }t        |j                  di       j                  dt                    }d}d| v rd}t        |      }d}d}	d}
g }|D ]  }t        |      }	 t        j                  |      }|j&                  r&|j(                  s|j#                  | d       |
dz  }
W|j(                  d | D ]&  }t+        |      }t-        |d|       rt/        |      \  }}|s6t1        ||||j                  dd      |j                  dd      d       |	dz  }	et3        ||      }|r6t1        ||||j                  dd      |j                  dd      |       |	dz  }	|xs ddz  dz  }t5        |      |z   |kD  r|j#                  | d| d        nJt7        j8                  d      5 }t;        |      }t=        |||      }|r|j?                         j@                  dk  r'|j#                  | d| d        |
dz  }
	 d d d        UtC        |      }d d d        s|j#                  | d| d!       |
dz  }
|xs ddz  dz  }tE        ||       tG        ||||||      }tI        |      }tK        ||||j                  dd      |j                  dd      xs ||tM        |      ||r|jO                         nd	       |dz  }tQ        jR                  d"       ) t5        |      |kD  s|j#                  d#        n tU        ||||	|
d$jW                  |      d d%        t	        d&| d'|	 d(|
 d)t5        |      d*       |r|d d D ]  }t	        d+|         |jY                          y# t         $ r2}|j#                  | dt%        |      d d         |
dz  }
Y d }~Xd }~ww xY w# 1 sw Y   xY w),Npathsstate_dbz[akp-podcast] no feedsr   GROQ_API_KEYu<   [akp-podcast] GROQ_API_KEY not set — transcribe will fail.)fileAKP_PODCAST_USE_PROXY1YT_PROXY	cost_capswhisper_daily_usd   z
--backfill
   z: parse_err r5   r   z: bozo_no_entriesr   r@   r3   r?   no_audio_enclosurerQ   g{Gz?/z: cost-cap reachedzakp-pod-)prefixi  z: download_failz: transcribe_failg      ?z$cost-cap reached, stopping all feedsz; i  z[akp-podcast] processed=z	 skipped=z failed=z cost=$z.3fz  NOTE: )-r   r   r1   PODCAST_LIST_PATHprintosenvironrS   sysstderrr   DEFAULT_WHISPER_DAILY_CAP_USDr   r:   
feedparserparse	Exceptionr$   r~   bozoentriesrI   r   ra   r   rw   r   tempfileTemporaryDirectoryr   r   statst_sizer   r   r   ri   r   rY   r   timesleepr   joinclose)argvcfgr   feedsr   capmax_per_feedr   r   r   r   notes_linesr8   r   parsederF   r   r   r^   r   est_costtmpr   r   r   actual_costr   ru   ns                                 r   mainr    s   
-C
#g,z*
+C)*E&'::>>.)LSVS]S]^ +-**..9P*QUX*XBJJNN:&^bE
R(,,-@B_`
aCLt3FIGFK :~	%%c*F
 ;;v~~$'89:aKF^^M\2 )	EU#CtC5\2"0"7Ixc3eii.DeiiPVXZF[]qr1!%2Fc3eii.DeiiPVXZF[]cd1 (DD047H"3'(2S8""dV1SE1C#DE ,,J? 43s)+IwF
!Z__%6%>%>%E&&$q_'EFaKF4 4 (
3
4 ""dV1SE1B#CD!#+tt3d:KS+. dE:xSH&u-C3T599Wb+A599VUWCXCe\e#S_hSV\^`NIJJsOS)	X s#c)EFu:x 3	7FDIIk<RSWTW<XY	$YKy	&QXYlmpYqruXv
wxRa 	"AHQC.!	"IIK}  	$|CF3BK=ABaKF	:4 4s+   P%AQ#>Q#%	Q .'QQ #Q-__main__script_startr   )returnrU   )r   r~   r	  zsqlite3.Connection)r)   r   r	  z	list[str])r8   r~   r	  r~   )r	  r~   )r	  ztuple[str | None, int | None])r^   z
int | Noner	  
str | None)r   r~   r   r   r   r
  r	  zPath | None)r   r   r	  r
  )r   r~   r	  bool)r	  None)r	  r   )r   r   r	  r  )r3   )8__doc__
__future__r   r   _syspathinsertaria_loggingr   _get_logger_logrB   r   r   r   r   r   r   r   r   r   r   r   pathlibr   r   r   r   r   r   r   rr   r   r   r1   r:   rI   ra   ri   rw   r   r   r   r   r   r   r   r   r   r   r   r  __name__eventexitr    r   r   <module>r     s.  $ #  		  $ % 2,- L L L L L 2 2  :;RS 9:CD !$  !
 B6 ("(a@	0`F zJJ~CHHT#((12,  r   