
    =j                    x    d Z ddlmZ ddlZddlZddlmZmZ  ej                  e
      Zh dZd	dZd
dZddZy)uT   Normalisierung — einheitliches Item-Schema {title, url, date, source, score, raw}.    )annotationsN)urlsplit
urlunsplit>   urldatetitlesourcec                6   t        | j                               }|j                  j                         }|j	                  d      r|dd  }|j
                  j                  d      xs d}t        |j                  j                         |||j                  df      S )Nzwww.   / )
r   stripnetloclower
startswithpathrstripr   schemequery)r   partsr   r   s       )/root/aria/scripts/ai-radar/normalizer.py_normalize_urlr      s}    SYY[!E\\!F ::S!(SDu||))+VT5;;KLL    c                   t        | t        j                        r7| j                  t        j                  j
                        j                         S t        | t        j                        r| j                         S t        | t              r| S t        j                  j                  t        j                  j
                        j                         S )N)

isinstancedtdatetime
astimezonetimezoneutc	isoformatr   strnow)values    r   _coerce_dater%      s    %%0::<<%!  %;;??2;;??+5577r   c           
        g }| D ]  }t         t        |      z
  }|rt        j                  d||       /	 t	        |d         j                         t        |d         t        |d         t	        |d         j                         j                         |j                  d      |j                  di       d}|j                  |        |S # t        $ r!}t        j                  d	||       Y d
}~d
}~ww xY w)zENimmt rohe Plattform-Items und normalisiert auf einheitliches Schema.z!item missing required keys %s: %rr   r   r   r	   scoreraw)r   r   r   r	   r'   r(   znormalize failed for %r: %sN)REQUIRED_KEYSsetlogwarningr"   r   r   r%   r   get	Exceptionappend)itemsoutraw_itemmissing
normalizedexcs         r   normalize_itemsr6       s    C #h-/KK;WhO	Xg./557%huo6$Xf%56hx01779??A!g.||E2.J 	

:#$ J	  	KK5xE	s   BC	C6C11C6)r   r"   returnr"   )r7   r"   )r0   
list[dict]r7   r8   )__doc__
__future__r   r   r   loggingurllib.parser   r   	getLogger__name__r+   r)   r   r%   r6    r   r   <module>r@      s;    Z "   -g!2M8r   