
    ,$j]B                       d dl mZ d dlZd dlZd dlZd dlmZmZ d dlm	Z	 d dl
mZ d dlmZmZ d dlmZmZmZmZmZmZmZmZmZ  e	d          Z e	d	          Z e	d
          ZdZ ej        d          Ze G d d                      Zd>dZ d?dZ!d@dZ"dAdZ#dBdZ$dCd#Z%dDd$Z&dEd&Z'dFd,Z(dGdHd2Z)dId3Z*dJd8Z+dKd;Z,dLd<Z-dLd=Z.dS )M    )annotationsN)	dataclassfield)Path)Any)Sessionselect)	ActivityEventAnalyticsPostSnapshotContentScriptExternalPostProviderTheme
VideoAssetVideoStatusWebsiteCompanionz,/home/agent/projects/TrueTraceShorts_WebSitezQ/home/agent/jarvis_runtime/autoshortsbot/AutoShortsBot/data/youtube_upload_auditszK/home/agent/jarvis_runtime/autoshortsbot/AutoShortsBot/data/post_candidatesz!https://truetraceshorts.pages.devzV(?:shorts/|watch\?v=|youtu\.be/|youtube_video_id['\" ]*[:=]['\" ]*)([A-Za-z0-9_-]{8,})c                      e Zd ZU ded<   dZded<   dZded<   dZded<   dZded<   dZded	<   dZ	ded
<   dZ
ded<   dZded<   dZded<   dZded<   dZded<   dZded<   dZded<    ee          Zded<   ddZdS )MappingSourcestridN
str | Nonecandidate_id
package_idslugtitleshort_titlehook
youtube_idyoutube_urlwebsite_slugwebsite_content_pathfamilyscript_text
video_path)default_factoryset[str]sourcesother'MappingSource'returnc           	         dD ]A}t          | |          s/t          ||          rt          | |t          ||                     B| j                            |j                   | S )N)r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   )getattrsetattrr'   update)selfr(   names      S/home/agent/projects/AutoShorts_Dashboard/backend/app/services/analytics_mapping.pymergezMappingSource.merge1   sr    
 	: 	:D 4&& :75$+?+? :dGE4$8$8999EM***    )r(   r)   r*   r)   )__name__
__module____qualname____annotations__r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r   setr'   r2    r3   r1   r   r      s/        GGG#L####!J!!!!DE"K""""D!J!!!!"K""""#L####'+++++F"K""""!J!!!!c222G2222     r3   r   pathr   r*   dict[str, Any] | Nonec                    	 t          j        |                     dd                    }t          |t                    r|nd S # t
          $ r Y d S w xY w)Nutf-8replaceencodingerrors)jsonloads	read_text
isinstancedict	Exception)r:   datas     r1   _safe_read_jsonrI   <   sa    z$..').LLMM!$--7tt47   tts   AA 
AAvaluer   r   c                    | sd S t          |           }t                              |          }|r|                    d          S t	          j        d|          r|S d S )N   z[A-Za-z0-9_-]{8,})r   
YOUTUBE_REsearchgroupre	fullmatch)rJ   textmatchs      r1   _youtube_idrT   D   sd     tu::Dd##E {{1~~	|($// 4r3   dict[str, str]c                   |                      dd          }|                    d          si S |                    dd          }|dk     ri S |d|         }i }|                                D ]}|                                r|                    d          sd	|vr0|                    d	d
          \  }}|                                                    d                              d          }|||                                <   |S )Nr=   r>   r?   z---z
---   r    :rL   "')rD   
startswithfind
splitlinesstripsplit)r:   rR   endrawrH   linekeyrJ   s           r1   _frontmatterre   P   s   >>79>==D??5!! 	
))GQ

C
Qww	
qu+CD   " "zz|| 	ts33 	s$ZZQ''
U##C((..s33!SYY[[Kr3   rR   r   c                    | sdS t          j        dd|                                           } t          j        dd|           } d                    |                                           S )N z#[\w-]+
[^a-z0-9]+rX   )rP   sublowerjoinr`   rR   s    r1   _normrm   b   sW     r6*b$**,,//D6-d++D88DJJLL!!!r3   r&   c                f    h dfdt          |                                           D             S )N>   anddonfornotonetheyetyoudontthatthiswithyourfirstc                B    h | ]}t          |          d k    |v|S )   )len).0tstops     r1   	<setcomp>z_tokens.<locals>.<setcomp>l   s,    KKK!c!ffqjjQd]]A]]]r3   )rm   r`   )rR   r   s    @r1   _tokensr   j   s:    tttDKKKKuT{{((**KKKKr3   recordsdict[str, MappingSource]recordNonec                    |j         p|j        p|j        p|j        }|| v r| |                             |           d S || |<   d S N)r   r   r   r   r2   )r   r   rd   s      r1   _addr   o   sT    


Nv2
Nfk
NVYC
g~~6"""""r3   c                    | sdS |                                  dk    s'|                                                     d          rdS | S )NEveryday Red FlagsEVERYDAY_RED_FLAGeveryday)upperrj   r\   )rJ   s    r1   _family_fromr   w   sJ     $##{{}}+++u{{}}/G/G
/S/S+##Lr3   list[MappingSource]c                 >   i } t           dz  dz  }|                                r"t          |                    d                    ng D ]t}t	          |          pi }t          |                    d                    }t          |                    d          p|j        |                    d          p|j        t          |                    d          pd          
                    d          r|                    d          nd |                    d          |                    d	          |                    d
          |                    d          ||                    d          |                    d          t          |                    d                    dh          }t          | |           vt           dz  dz  dz  }|                                r"t          |                    d                    ng D ]}t          |          }t          |                    d                    }t          |                    d          p|j        |                    d          p|j        t          |                    d          pd          
                    d          r|                    d          nd |                    d          p|j        |                    d	          |                    d
          |                    d          ||                    d          |                    d          p|j        t          |          t          |                    d                    dh          }t          | |           t                                          r't          t                              d                    ng D ]1}t	          |          pi }t          |                    d          p)|                    d          p|                    d                    }t          |                    d          p|j        |                    d          |                    d	          ||                    d          p|                    d          p|rd| nd t          |                    d          pd          
                    d          rdnd dh          }t          | |           3t                                          r't          t                              d                    ng D ]}t	          |          pi }|                    d          }t!          |                    d          t"                    r|                    d          ni }	|                    d	          p|	r|	                    d	          nd }
|                    d           p|                    d!          }t          |p|j        j        ||
t!          |                    d"          t                    r|                    d"          pdd d#         nd t!          |                    d"          t                    r|                    d"          nd |rt          |          nd t          |pd          
                    d          rdnd d$h%          }t          | |           t)          |                                           S )&NrH   
candidatesz*.jsonvideoUrlr   rg   package_r   r   
shortTitler   categorywebsite_candidate)r   r   r   r   r   r   r   r   r   r    r"   r'   srccontentredflagsz*.mdwebsite_markdown)r   r   r   r   r   r   r   r   r   r    r!   r"   r'   youtube_video_id
shorts_url	watch_urlr   zhttps://www.youtube.com/shorts/zerf-r   youtube_upload_audit)r   r   r   r   r   r"   r'   z**/review_package*.json
video_metaoutputmedia_cache_pathscript   review_package)r   r   r   r   r#   r$   r"   r'   )WEBSITE_ROOTexistssortedglobrI   rT   getr   stemr   r\   r   r   re   AUDITS_ROOTPACKAGES_ROOTrE   rF   parentr0   listvalues)r   cand_dirr:   rH   ytrecmd_dirfmcidmetar   r   s               r1   build_source_indexr      s[   (*Gf$|3H3;??3D3DLx}}X..///"  t$$**--..xx~~*$449),TXXd^^-Ar)B)B)M)Mj)Y)Yctxx~~~_c&!!((7##..&!!,,&)) 4 455()
 
 
 	WcE!I-
:F/5}}Fv{{6**+++B  $
++,,vvd||(ty2'*266$<<+=2'>'>'I'I*'U'U_rvvd|||[_,49&&//|,,z**449!$Tz 2 233'(
 
 
 	Wc6A6H6H6J6JR{''11222PR  t$$*"455h,9O9OhSWS[S[\gShShiixx''449.11((7##..  D$((;2G2G  Dvx  MCLrnpLrLrLr  C+.txx/G/G/M2+N+N+Y+YZ`+a+ak''gk+,
 
 
 	WcIVI]I]I_I_g}))*CDDEEEeg  t$$*hh~&&)3DHH\4J4JD)Q)QYtxx%%%WY!!J4&Idhhw&7&7&7T(##Ctxx0B'C'C&dk&5?@R@RTW5X5Xb$((8$$*DSD11^b.8(9K9KS.Q.Q[***W[&,6s6{{{$+.syb>>+D+DV+L+LV''RV%&	
 	
 	
 	Wc  !!!r3   scoreint
confidencereasondict[str, Any]c                   i ddd| j         p| j        d| j         d| j        d| j        p| j        p| j         d| j        d| j        d	| j        d
| j        d| j        r	| j        rdn
| j        rdndd| j        rt           d| j         dnd d| j        d| j	        d| j
        d| j        d| j        d||dddd                    |d          ||gt          | j                  dS )Ntypecontent_scriptr   r   r   r   youtube_titler"   content_familyr    website_statuszguide existszguide pendingzno guidewebsite_urlz
/redflags//r!   r   r   r#   r$   r   u$   High — matched by YouTube video IDu!   Medium — matched by exact titleu   Low — similar title only)highmediumlowu    Needs review — no strong match)r   confidence_labelr   match_reasonsr'   )r   r   r   r   r   r"   r    r!   
PUBLIC_URLr   r   r#   r$   r   r   r'   )r   r   r   r   s       r1   _candidate_payloadr      s    f!.VY 	+ 	f'	
 	#Jv|Jv7J 	 	&- 	&- 	+ 	F,?  ^FD_  ^..y  zM  g]fufu  S] 	&J]g*FF0CFFFFcg 	 ; 	f' 	v) 	v)  	f'!" 	#$ !:9/
 
 #j<
=
= &.))5   r3      r   r   limitlist[dict[str, Any]]c                   g }t          |           t          |           t                      D ]}|rL|j        |k    rAd|j        v rdnd|j        v rdnd}|                    t          |ddd|                      Q|j        |j        |j	        |j
        |j        g}rAt          fd	|D                       r&|                    t          |d
dd                     t          fd|D             d          }|dk    r<t          d|dz            }|                    t          ||dd| d                     |                    d d           i }	|D ]=}
|
                    d          p|
                    d          p|
d         }||	vr|
|	|<   >t#          |	                                          d |         S )Nr   zwebsite candidater   zmarkdown redflagzYouTube upload auditd   r   z!Matched by YouTube video ID from c              3  B   K   | ]}|t          |          k    V  d S r   )rm   )r   n
title_norms     r1   	<genexpr>z)mapping_candidates_for.<locals>.<genexpr>   s3      IIqIeAhh*4IIIIIIr3   T   r   zMatched by exact titlec              3  Z   K   | ]%}|t          t          |          z            V  &d S r   )r   r   )r   r   title_tokenss     r1   r   z)mapping_candidates_for.<locals>.<genexpr>   s;      JJ!Js<'!**455JJJJJJr3   r   )defaultrW   A      r   zSimilar title/keyword overlap: z shared termsc                    | d         S )Nr   r9   )items    r1   <lambda>z(mapping_candidates_for.<locals>.<lambda>   s
    T'] r3   Trd   reverser   r   r   )rm   r   r   r   r'   appendr   r   r   r   r   r   anymaxminsortr   r   r   )r   r   r   r   r   sourcenamesoverlapr   dedupr   rd   r   r   s               @@r1   mapping_candidates_forr      s+   JuJ5>>L!##   	#.J66,?3;,N,N((j|  AD  AL  kL  kLTfTf  RhF0c6CogmCoCoppqqqCOSXs7GR 	#IIIIeIIIII 	0b(D\]]^^^JJJJuJJJTUVVVa<<GbL))E0eUD|fmD|D|D|}}~~~OO22DOAAA')E  hh|$$N(@(@NDJeE#J''r3   c                    t          j        dd|                                                               d          } | pdS )Nrh   -r   )rP   ri   rj   r_   rl   s    r1   _slugifyr     s6    6-djjll3399#>>D9r3   sessionr   r0   r   c                f   |pd}|                      t          t                                        t          j        |k                                                        }|sPt          |dd          }|                     |           |                                  |                     |           |S )Nr   z?Mapped from real YouTube analytics and website/content sources.active)r   descriptionstatus)	execr	   r   wherer   r|   addcommitrefresh)r   r0   r"   s      r1   _get_or_create_familyr     s    ''D\\&----ekT.ABBCCIIKKF  T/p  zB  C  C  CFMr3   	candidate.tuple[VideoAsset, ContentScript | None, Theme]c                
   t          | |                    d          p|                    d                    }|                    d          p)|                    d          p|                    d          }|                     t          t                                        t          j        |k                                                        }|st          j	        d| 
                                                                          }t	          ||                    d          d|                    d	          pd
d||                    d          |                    d          p|                    d          p||j        |                    d          t          j        dd          }n|j        p|                    d          |_        |j        p|j        |_        |j        p|                    d          |_        |j        p)|                    d          p|                    d          |_        |                     |           |                                  |                     |           |                     t          t,                                        t,          j        |k                                                        }|st-          |j        |j        ||                    d          p|                    d          p||                    d          |                    d          |                    d          d|                    d          |                    d          |                    d          d
  
        }na|j        p|j        |_        |j        p|j        |_        i |j        pi |                    d          |                    d          d|_        |                     |           |                    d          r|                     t          t4                                        t4          j        |j        k                                                        }|st5          |j        |j        t7          t8                    |                    d          |                    d          p|                    d          |                    d          |                    d          rdnd          }|                     |           |                                  |                     |           |||fS )Nr"   r   r   r   r   zanalytics-map:r   analytics_mappingr$   rg   r   r   r   r#   FzYoutube 260606)r   r   r   	file_path	file_sizechecksumexternal_youtube_idworking_titletopic_idr   r   is_demosource_labelr   measuredr!   r    r   )r    r   )
	family_idvideo_asset_idr   r   r   r#   r   r   source_pathperformance_summary_jsonpushed	generated)r	  content_script_idwebsite_repo_pathr   r   generated_content_pathr   )r   r   r   r	   r   r   r   r|   hashlibsha256encode	hexdigestr   r   importedr  r  r   r  r   r   r   r   r  r	  r  r   r   r   )r   r   r"   r   videor  r   	companions           r1   materialize_candidater    s   "7IMM(,C,C,fy}}UeGfGfggF
--
'
'
]9==+>+>
])--P\B]B]CLL
++11*2IS2PQQRRXXZZE n>"83"8"8"?"?"A"ABBLLNN }}\22&mmL117R )l ; ;#--00YIMM/4R4RYVYY==//')
 
 
  %*$=$\|A\A\!469 +Jy}}\/J/J#1mY]]75K5Kmy}}]lOmOmKKNNOOE\\&//55m6NRU6UVVWW]]__F xi 8--((QIMM/,J,JQcx((!m44#--88!&<==6?mmN6S6Sgpgtgt  vF  hG  hG  &H  &H
 
 
 "+8vy & 5 A +xf.M.SQS  +xfofsfs  uC  gD  gD  Xa  Xe  Xe  fv  Xw  Xw  +x  +x  +x'KK}}^$$ #LL(8!9!9!?!?@P@_chck@k!l!lmmssuu	 
	#($x"()"%l"3"3]]>22mmG,,N	o0N0N'0}}5K'L'L#,==1G#H#HYxxk  I KK	"""NNOOE&&  r3   c           	        g }|                      t          t                                        t          j        dk                                                        }i }|D ]8}|                    |j                  }|r|j        |j        k    r
|||j        <   9|	                                D ]}|                      t          t                                        t          j        |j        k                                                        }|j        s|r|j        s|j        dk    r}t          |j        |j        d          }d |D             }t!          |          dk    r>|                    |j        |j        |j        pd|d         |d         d         d	           t'          |d
 d          t!          |          dS )NFignoredr   )r   c                *    g | ]}|d          dk    |S )r   r   r9   )r   cs     r1   
<listcomp>z+high_confidence_preview.<locals>.<listcomp>g  s&    CCCa<F)B)B)B)B)Br3   rL   r   r   )external_post_idr   viewstargetr   c                    | d         S )Nr  r9   )is    r1   r   z)high_confidence_preview.<locals>.<lambda>j  s
    7 r3   Tr   )itemscount)r   r	   r   r   r  allr   r  snapshot_atr   r   r|   r	  mapping_statusr   r   r   r   r  r   )	r   r#  	snapshotslatest_by_idsnapcurrentpostr   r   s	            r1   high_confidence_previewr-  Z  s   EV$9::@@AVA^bgAghhiimmooI57L 7 7""4#899 	7$*W-@@@26L./##%% e e||F<0066|7TX\Xm7mnnoouuww 	4 	T-@ 	DDW[dDdDd+DJ8MUVWWW
CC:CCCt99>>LLd.Cdjcgcmcrqr  C  DE  F  RV  WX  RY  Zb  Rc  d  d  e  e  eE';';TJJJUXY^U_U_```r3   c                   t          |           }g }|d         D ]}|d         }t          | |          \  }}}|                     t          t                                        t          j        |d         k                                                        }|s|j        |_	        |j        |_
        d|_        |                     |           |                     t          t                                        t          j        |d         k                                                        D ]#}	|j        |	_	        |                     |	           $|                     t          d|j        dd|d         |j        |r|j        nd |j        |d         |d	
                     |                    i ||j        |j        |r|j        nd d           |                                  dt%          |          |dS )Nr#  r   r  linkedanalytics_postanalytics_auto_linkedz.Analytics auto-linked by high-confidence matchr   )r  r	  r  r  r   r   )entity_type	entity_id
event_typelabelpayload_json)r	  r  r  T)oklinked_countr/  )r-  r  r   r	   r   r   r  r|   r   r	  r  r'  r   r   r%  r
   r   r   r   )
r   previewr/  r   r   r  r   r"   r,  r*  s
             r1   auto_link_high_confidencer:  m  s}   %g..GF  H Hh 5gv F Fvv||F<0066|7TX\]oXp7pqqrrxxzz 	#h	&DLL(=!>!>!D!DEZEkos  uG  pH  FH  "I  "I  J  J  N  N  P  P 	 	D"'(DKKM(g.B.23E.FZ_Zb  GM  zWy  zC  zC  SW  fl  fo  {  @H  {I  U[  \  \
 
 
 	 	 	 	  G  G	u{  iFhnhqhq  BF  G  G  G  	H  	H  	H  	HNNFvFFFr3   )r:   r   r*   r;   )rJ   r   r*   r   )r:   r   r*   rU   )rR   r   r*   r   )rR   r   r*   r&   )r   r   r   r   r*   r   )rJ   r   r*   r   )r*   r   )
r   r   r   r   r   r   r   r   r*   r   )r   )r   r   r   r   r   r   r*   r   )rR   r   r*   r   )r   r   r0   r   r*   r   )r   r   r   r   r*   r   )r   r   r*   r   )/
__future__r   rB   rP   r  dataclassesr   r   pathlibr   typingr   sqlmodelr   r	   app.models.corer
   r   r   r   r   r   r   r   r   r   r   r   r   compilerM   r   rI   rT   re   rm   r   r   r   r   r   r   r   r   r  r-  r:  r9   r3   r1   <module>rB     s   " " " " " "  				  ( ( ( ( ( ( ( (             $ $ $ $ $ $ $ $
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 tBCCdfggbcc0
RZqrr
        8   	 	 	 	   $" " " "L L L L
      K" K" K" K"\   >( ( ( ( (4   
   A! A! A! A!Ha a a a&G G G G G Gr3   