
    =jc                    t    d Z ddlmZ ddlZddlZddlmZ  ej                  e      Z	d	dZ
d
dZddZddZy)u+   Dedupe — SHA256(url) gegen rolling state.    )annotationsN)Pathc                f    t        j                  | j                  d            j                         S )Nutf-8)hashlibsha256encode	hexdigest)urls    %/root/aria/scripts/ai-radar/dedupe.py	_hash_urlr      s#    >>#**W-.88::    c                    | j                         s
t               S | j                  d      j                         D ch c]#  }|j	                         s|j	                         % c}S c c}w )Nr   encoding)existsset	read_text
splitlinesstrip)
state_filelines     r   
_load_seenr      sR    u%/%9%97%9%K%V%V%XiT\`\f\f\hDJJLiiis   A(A(c                v    t        |      }g }| D ]&  }t        |d         }||vs|j                  |       ( |S )z>Behalte nur Items deren URL-Hash noch nicht in state_file ist.r   )r   r   append)itemsr   seenfreshitemhs         r   filter_unseenr!      sH    j!DE d5k"D=LL Lr   c                    |j                   j                  dd       |j                  dd      5 }| D ]"  }|j                  t	        |d         dz          $ 	 ddd       y# 1 sw Y   yxY w)	u-   Hänge URL-Hashes der Items an state_file an.T)parentsexist_okar   r   r   
N)parentmkdiropenwriter   )r   r   fhr   s       r   	mark_seenr,       sk    D48	w	/ 42 	4DHHYtE{+d23	44 4 4s   (A##A,)r   strreturnr-   )r   r   r.   zset[str])r   
list[dict]r   r   r.   r/   )r   r/   r   r   r.   None)__doc__
__future__r   r   loggingpathlibr   	getLogger__name__logr   r   r!   r,    r   r   <module>r9      s;    1 "   g!;j4r   