
    |\jk$                       d Z ddlmZ ddlZddlZddlZddlZddlmZ ddl	Z	ddl
Z
ddlZddlmZ ddlmZ ddlmZ ddlmZmZmZmZ dd	lmZmZ  ed
          ZddZddddZddZedk    r e e                      dS )zGControlled offline inventory/extraction for existing private documents.    )annotationsN)Path)datetime)ZoneInfo)probe_original)candidate_rowsextract_pagesnormalize_search_textsection_similarity)apply_schemaassert_schemazEurope/Zurichprefixstrreturnc                0    | t          j        d          z   S )N   )secrets	token_hexr   s    H/home/agent/.hermes/assets/Gesundheit/scripts/document_local_pipeline.pyopaquer      s    G%b))))    Fretry_faileddatabaser   r   booldict[str, int]c               D   t          j        |           }t           j        |_        d dD             }	 |                    d           t          |           |                                 t          |           t          |                    d          	                                d                   |d<   |                    dt          |          f          
                                }|D ]}t          j        t                                        d	          }t          |d
                   }t          |d         p|d                   }|j        dk    s|j        |                    d|t%          d          |j        dv r|j        ndt'          |d         p:t)          j        d|                                                                                     ||f           |                                 |                                 %|dxx         dz  cc<   	 t3          j        d          5 }	|j        dk    rdn|j        dk    rdnd}
t9          |	          d|
 z  }t)          j                    }t;          j        t;          j        |j                  d          5 }|                     d          5 }|!                    d           x}rA|"                    |           |#                    |           |!                    d           x}Ad d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |                                }|                    d!||f          	                                }|r|d"xx         dz  cc<   t'          |d#         pd$          }|$                                rd%d&|gd gf\  }}}}n(tK          ||j        t9          |	                    \  }}}}d d d            n# 1 swxY w Y   d' |D             }tM          tO          tP          |                    }|r|nd(}||d%k    rd)nd*xx         t          tS          |                    z  cc<   |d(xx         t          |           z  cc<   |                    d+           |                    d,||t%          d          ||rd-nd.||rt          |d                   nd tQ          |          ||||f           t          |                    d/|f          	                                d                   }d0*                    d1 tW          |d          D                       }d0*                    d2 tW          |d          D                       }|                    d3t%          d4          ||||||||f	           |                    d5|f          
                                }g }g }tW          tY          |||          d          D ]+\  } \  }!}"}#|rd6nd7}$d }%|D ]q}&t[          |!t'          |&d8                             }'|'dk    r)d9}$t          |&d:                   }%|d;xx         dz  cc<    n|'d<k    rd=}$t          |&d:                   }%r|                    d>t%          d?          ||| |!|"|#t)          j        |".                                                                                                          |$|%f
           |$d9k    r*|/                    |!           |/                    |            -t'          |d@         pd$          t'          |dA         pd$          dB}(ta          |||(|C          })|)D ]Y}*|                    dD|*d
         ||*dE         |*dF         |*dG         |*dH         |*dI         |*dJ         |*dK         |*dL         |f           Z|                    dMt'          |          f           tW          |d          D ]/\  } }"|"r(|                    dNt'          |          | | |"dOf           0|                    dP|||f           |                    dQ|)rdRndS|f           |                                 |dRxx         t          tS          |)                    z  cc<   n# tb          $ r |2                                 |dTxx         dz  cc<   |                    dU|t%          d          t'          |d         p:t)          j        dV|                                                                                     ||f           |                    dW||f           |                                 Y nw xY w|                                 # |                                 w xY wt          |                    dX          	                                d                   |dY<   ||                                 S # |                                 w xY w)ZNc                    i | ]}|d S )r    ).0keys     r   
<dictcomp>zprocess.<locals>.<dictcomp>   s,      ]  ]  ]#A  ]  ]  ]r   )
documents_totaloriginal_availabletext_layer_successocr_successfailedno_usable_textcandidates_availablepossible_duplicatesrepeated_sectionsuser_reviews_openzPRAGMA foreign_keys=ONzSELECT COUNT(*) FROM dokumenter   r$   zSELECT d.* FROM dokumente d LEFT JOIN document_processing p ON p.document_id=d.id
            WHERE p.document_id IS NULL OR (? AND p.extraction_status='failed') ORDER BY d.idseconds)timespecidlocal_original_path	dateipfad	availablezINSERT OR IGNORE INTO document_processing(document_id,intake_id,original_status,extraction_status,content_status,search_status,transfer_status,sha256,created_at,updated_at) VALUES(?,?,?,'not_started','pending','not_searchable','no_candidates',?,?,?)doc_>   blockedmissingunsupportedr5   
datei_hashzmissing:r%      zhealth-existing-doc-r   zapplication/pdfz.pdfz	image/pngz.pngz.jpgsourcerbwbi   zjSELECT document_id FROM document_processing WHERE sha256=? AND document_id<>? ORDER BY document_id LIMIT 1r+   extrahierte_inhalte 
text_layerzlegacy-preserved-v1c                ,    g | ]}t          |          S r    )r
   )r!   pages     r   
<listcomp>zprocess.<locals>.<listcomp>8   s!    JJJD1$77JJJr   r)   r&   r'   zBEGIN IMMEDIATEar  INSERT OR REPLACE INTO document_processing(document_id,intake_id,original_status,extraction_status,content_status,search_status,transfer_status,sha256,duplicate_document_id,page_count,engine,engine_version,created_at,updated_at) VALUES(?,COALESCE((SELECT intake_id FROM document_processing WHERE document_id=?),?),'available',?,'pending',?,'no_candidates',?,?,?,?,?,?,?)machine_searchablenot_searchablezQSELECT COALESCE(MAX(version),0)+1 FROM document_text_versions WHERE document_id=?z

c              3  ,   K   | ]\  }}d | d| V  dS z[SEITE z]
Nr    r!   itexts      r   	<genexpr>zprocess.<locals>.<genexpr>>   s7      )b)bVQt*@A*@*@$*@*@)b)b)b)b)b)br   c              3  ,   K   | ]\  }}d | d| V  dS rF   r    rG   s      r   rJ   zprocess.<locals>.<genexpr>>   s        }  }  [a  [\  ]a  @V  JK  @V  @V  PT  @V  @V  }  }  }  }  }  }r   zINSERT INTO document_text_versions(id,document_id,version,source_kind,original_text,normalized_text,engine,engine_version,created_at) VALUES(?,?,?,?,?,?,?,?,?)text_zmSELECT document_id,original_text FROM document_pages WHERE document_id<>? ORDER BY document_id DESC LIMIT 100newfirst_documentedoriginal_text	identicaldocument_idr,   gq=
ףp?
near_matchzINSERT INTO document_pages(id,document_id,text_version,page_number,original_text,normalized_text,confidence,section_hash,repetition_status,compared_document_id) VALUES(?,?,?,?,?,?,?,?,?,?)page_document_dateinstitution)rT   rU   )page_numberszINSERT OR IGNORE INTO document_candidates(id,document_id,candidate_type,value_text,unit,page_number,section_number,context_text,engine,confidence,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)candidate_type
value_textunitpage_numbersection_numbercontext_textengine
confidencez;DELETE FROM health_document_machine_fts WHERE document_id=?zvINSERT INTO health_document_machine_fts(document_id,page_number,section_number,content,review_label) VALUES(?,?,?,?,?)machine_unreviewedzsUPDATE dokumente SET extrahierte_inhalte=COALESCE(NULLIF(extrahierte_inhalte,''),?),processing_quality=? WHERE id=?zDUPDATE document_processing SET transfer_status=? WHERE document_id=?r*   no_candidatesr(   zINSERT OR IGNORE INTO document_processing(document_id,intake_id,original_status,extraction_status,content_status,search_status,transfer_status,sha256,created_at,updated_at) VALUES(?,?,'available','failed','pending','not_searchable','no_candidates',?,?,?)zfailed:z~UPDATE document_processing SET extraction_status='failed',last_error_code='extraction_failed',updated_at=? WHERE document_id=?zSELECT COUNT(*) FROM document_processing WHERE content_status IN ('pending','partial') OR transfer_status='candidates_available' OR duplicate_document_id IS NOT NULLr-   )3sqlite3connectRowrow_factoryexecuter   commitr   intfetchonefetchallr   nowTZ	isoformatr   status
descriptorr   r   hashlibsha256encode	hexdigestclosetempfileTemporaryDirectorymimer   osfdopendupopenreadupdatewritestripr	   summaplenr   join	enumeratezipr   casefoldappendr   	Exceptionrollback)+r   r   
connectioncountersrowsrowrj   doc_idprobetempsuffixcopieddigestr:   targetchunksha	duplicateexistingr]   versionpagesconfidences
normalizedusefulrm   text_versionfull_originalfull_normalizedpreviouscandidate_pagescandidate_page_numberspage_nooriginalnormr^   
repetitioncomparedold
similaritymetadata
candidatesitems+                                              r   processr      s   **J$[J ]  ]  "\  ]  ]  ]H; 3444Z   *"3"3"5"5"5}Z7P7P7P&)**<*<=]*^*^*g*g*i*ijk*l&m&m"#!! #abefrbsbsauw www  xB  xB 	 2	# 2	#CR  **I*>>Cs3t9~~ %:!;!Os;?OPPE|{**e.>.F""  $_  ag  hn  ou  hv  hv  GL  GS  Wz  Gz  Gz  w|  wC  wC  @I  JM  NQ  R^  N_  N[  cj  cq  rE  }C  rE  rE  rL  rL  rN  rN  cO  cO  cY  cY  c[  c[  J\  J\  ]`  ad  `e  f  f  f  gq  gx  gx  gz  gz  gz  {@  {F  {F  {H  {H  {H  IQ)***a/***,#08NOOO gSW%*Z1B%B%B66RWR\^iRiRiouF::&7v&7&77F">++F26%*:#;#;DAA dVFKKX\L]L] dag%+[[%9%99ec6==;O;O;OPVP\P\]bPcPcPcV[[%9%99ecd d d d d d d d d d d d d d d d d d d d d d d d d d d d d d((**C(00  2^  `c  dj  _k  l  l  u  u  w  wI D(+@"A"A"A1"D"A"A"A %:!;!ArBBH~~'' g,Wlnvmwy}x~J~)Ikk;HPUPZ[_`d[e[e;f;f:&{g g g g g g g g g g g g g g g KJEJJJ
SVWZ[^_iWjWjSkSkF  E  u[tztz  K[ms1E1E--=YYY]`aeflamam]n]nnYYY)***cf*oo=***""#4555""  $X  Z`  ag  hn  ou  hv  hv  w}  V\  ~r  ~R  ~R  br  sv  LU  w_  wz  {D  EF  {G  wH  wH  wH  [_  `c  di  `j  `j  kq  ry  z}  ~A	  YB	  C	  C	  C	 !3!3  5H  JP  IR  "S  "S  "\  "\  "^  "^  _`  "a  b  b$kk)b)byY^_`OaOa)b)b)bbbsys~s~  }  }  en  oy  z{  e|  e|  }  }  }  t}  t}cr""  $E  GM  NU  GV  GV  W]  ^j  kq  r  @O  PV  W^  _b  Fc  d  d  d#++  -\  ^d  ]f  g  g  p  p  r  r "2$::CCjYdDeDefg:h:h { {6G6Xd:(0Huu6HJRV' e e#5hs3CW?X?X#Y#Y
%q==[*#cR_N`JaJabjk~bbb  BC  cCbbb  DI  DI%s??|JSQTUbQcMdMdH&&  (f  hn  ov  hw  hw  x~  K  LS  T\  ]a  bl  mt  m{  |@  |I  |I  |K  |K  |R  |R  |T  |T  mU  mU  m_  m_  ma  ma  bl  mu  gv  w  w  w![00/2H2H2R2R2RTjTqTqryTzTzTz*-c/.B.Hb*I*IX[\_`m\n\trtXuXuvv)/&Wmnnn
& W WD&&  (e  gk  lp  gq  rx  y}  ~N  yO  PT  Ua  Pb  cg  hn  co  pt  uB  pC  DH  IY  DZ  [_  `n  [o  pt  u}  p~  C  DP  Q  RU  fV  W  W  W  W""#`beflbmbmaoppp$-j$;$; b bLGD  bZ//  1i  kn  ou  kv  kv  w~  F  GK  L`  ja  b  b  b""  $Y  [h  io  pv  Zw  x  x  x""#i  FP  le  lB  lB  Ve  fl  km  n  n  n  oy  o@  o@  oB  oB  oB/000CZ8H8H4I4II0000 M	 M	 M	##%%%hx&8&8&8!&;&8&8&8""  $d  fl  ms  tz  m{  m{  |  @C  DP  @Q  @L  U\  Uc  dv  nt  dv  dv  d}  d}  d  d  U@  U@  UJ  UJ  UL  UL  |M  |M  NQ  RU  eV  W  W  W  Xb  Xj  Xj  kk  mp  qw  lx  Xy  Xy  Xy  zD	  zK	  zK	  zM	  zM	  zM	  zM	  zM	M	 [[]]]]U[[]]]]&)**<*<  >e  +f  +f  +o  +o  +q  +q  rs  +t  'u  'u$%Zs   G1h	 %b1:A1P+L=AL&	L=&L**L=-L*.L=1P=MPMB4P9b1P			b1P		R#b10f1Cffffh	 f33Ah	 	hrg   c                 X   t          j                    } |                     ddt                     |                     dd           |                                 }t          t          j        t          |j	        
                                |j                  d                     d	S )
Nz--dbT)requiredtypez--retry-failed
store_true)actionr   )	sort_keysr   )argparseArgumentParseradd_argumentr   
parse_argsprintjsondumpsr   dbresolver   )parserargss     r   mainr   ]   s    "$$FV%8%8SW%8%X%X%XY_YlYlm}  FRYl  ZS  ZS  ZS  Y_  Yj  Yj  Yl  Yl  TX	$*WTW__..D<MNNNY]
^
^
^___ghghr   __main__)r   r   r   r   )r   r   r   r   r   r   )r   rg   )__doc__
__future__r   r   ro   r   rw   pathlibr   r   ra   rt   r   zoneinfor   dashboard_v5.document_originalsr   dashboard_v5.document_reviewr   r	   r
   r   dashboard_v5.sprint6i_a_schemar   r   rk   r   r   r   __name__
SystemExitr    r   r   <module>r      sg   M M " " " " " "    				                      : : : : : : q q q q q q q q q q q q F F F F F F F FXo* * * * 5: ?  ?  ?  ?  ?  ? Di i i i
 zDDFF!3!33r   