i#~dZddlmZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl m Z ddl mZddlZddlZej$dk\r dd l mZdd l mZndd lmZmZd d lmZd dlmZd dlmZd dlmZdZdZgdZgdZeezZ GddZ!Gdde!Z"GddZ#Gdde#Z$GddZ%Gdd Z&Gd!d"eZ'Gd#d$Z(y)%z6Contains the core functions to parse tables from PDFs.) annotationsN) itemgetter)Any)Iterable)Iterator) ) TypedDict)Unpack)r r )ImageConversionBackend)build_file_path_in_temp_dir)get_index_closest_point)get_textline_coords leftrightmiddle)topbottomcenterc"eZdZdZdZdZdZy) TextAlignmentaRepresents a list of textlines sharing an alignment on a coordinate. The alignment can be left/right/middle or top/bottom/center. (PDF coordinate space) Parameters ---------- coord : float coordinate of the initial text edge. Depending on the alignment it could be a vertical or horizontal coordinate. textline : obj the original textline to start the alignment align : str Name of the alignment (e.g. "left", "top", etc) Attributes ---------- coord : float The coordinate aligned averaged out across textlines. It can be along the x or y axis. textlines : array Array of textlines that demonstrate this alignment. align : str Name of the alignment (e.g. "left", "top", etc) c0||_|g|_||_yN)coord textlinesalignselfrtextliner s ?/home/agent/.local/lib/python3.12/site-packages/camelot/core.py__init__zTextAlignment.__init__Gs " cdjtd|jddjdd}d|jdt |jd|d S) Nz | c"|jSr)get_textxs r$z(TextAlignment.__repr__..Ns !**,r& z)joinmaprreplacerlen)r" text_insides r$__repr__zTextAlignment.__repr__Lsijj &r(: ; '$  tzzl$s4>>/B.CD*m5 2 r&c|jt|jz|ztt|jdzz |_|jj |y)z%  ! #C    h'r&N)__name__ __module__ __qualname____doc__r%r5r9r&r$rr,s4  (r&rc0eZdZdZfdZdZddZxZS)TextEdgeafDefines a text edge coordinates relative to a left-bottom origin. (PDF coordinate space) An edge is an alignment bounded over a segment. Parameters ---------- coord : float coordinate of the text edge. Can be x or y. y0 : float y-coordinate of bottommost point. y1 : float y-coordinate of topmost point. align : string, optional (default: 'left') {'left', 'right', 'middle'} Attributes ---------- is_valid: bool A text edge is valid if it intersects with at least TEXTEDGE_REQUIRED_ELEMENTS horizontal text rows. czt|||||j|_|j|_d|_y)NF)superr%y0y1is_valid)r"rr#r __class__s r$r%zTextEdge.__init__xs1 %0++++ r&c t|jd}t|jd}t|jd}d|d|d|d|jd|j d S)Nr-z )roundrrCrDr rE)r"r+rCrDs r$r5zTextEdge.__repr__~sd $**a  477A  477A 1#T"T"WTZZL VW X r&ctj|j|j|rH|j|||j|_t |j t kDrd|_yyy)zUpdate text edge coordinates. Update the text edge's x and bottom y coordinates and sets the is_valid attribute. abs_tolTN)mathiscloserCr9r3rTEXTEDGE_REQUIRED_ELEMENTSrE)r"r+r#edge_tols r$ update_coordszTextEdge.update_coordss^ <<h ?  * *8Q 7kkDG4>>"%?? $ @ @r&2)r:r;r<r=r%r5rR __classcell__rFs@r$r@r@_s0   %r&r@c2eZdZdZdZedZdZdZy)TextAlignmentsz9Defines a dict of text edges across reference alignments.c>i|_|D]}g|j|<yr)_text_alignments)r"alignment_namesalignment_names r$r%zTextAlignments.__init__s) "- 7N46D ! !. 1 7r&ct|||Sr)r)rr#r s r$_create_new_text_alignmentz)TextAlignments._create_new_text_alignmentsUHe44r&ctSr)NotImplementedr" alignmentrr#s r$_update_alignmentz TextAlignments._update_alignmentsr&crt|}|jjD]\}}||}t||d}d}|d}nG||j}|dz |cxkr|dzkrnn|j ||||n ||kr|dz}n|}|k|j |||} |j|| y)z1Update an existing text edge in the current dict.c|jSr)rr*s r$r,z3TextAlignments._register_textline..s QWWr&)fnNrg?r )rrZitemsrrrcr^insert) r"r#coords alignment_idalignment_arrayr idx_closest idx_insert coord_closest new_alignments r$_register_textlinez!TextAlignments._register_textlines$X.-1-B-B-H-H-J B )L/<(E2+<K J" / < B B 3;<<**' 4eX#U*!,qJ!,J% $ ? ?8\!  &&z=A9 Br&N) r:r;r<r=r% staticmethodr^rcrpr>r&r$rXrXs)C7 55Br&rXcfeZdZdZdfd ZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZxZS) TextEdgeszDefines a dict text edges on the PDF page. The dict contains the left, right and middle text edges found on the PDF page. The dict has three keys based on the alignments, and each key's value is a list of camelot.core.TextEdge objects. c:t|t||_yr)rBr%HORIZONTAL_ALIGNMENTSrQ)r"rQrFs r$r%zTextEdges.__init__s ./  r&ct|||Sr)r@r!s r$r^z$TextEdges._create_new_text_alignmentsx//r&cf|j|||}|j|j|y)z(Add a new text edge to the current dict.N)r^rZr8)r"rr#r tes r$addz TextEdges.adds/  , ,UHe D e$++B/r&c>|j|||jyr)rRrQras r$rczTextEdges._update_alignmentsx?r&c|D]>}t|jjdkDs.|j|@y)zz)TextEdges.get_relevant..s$&(R[[BLL!s..rc3^K|]%}|jrt|j'ywrrrs r$rz)TextEdges.get_relevant..s(;;BLL!+-rc3^K|]%}|jrt|j'ywrrrs r$rz)TextEdges.get_relevant..s(;;BLL!rrrr keyrc|jSr)rErxs r$r,z(TextEdges.get_relevant..s bkkr&)sumrZmaxrgrlistfilter)r"intersections_sumrelevant_aligns r$ get_relevantzTextEdges.get_relevants,0,A,A&,I//8 //9 &.446JqMJ1M )4+@+@+P Q  r&c|jd|j|}|j|||j|}|Dcic]}|j ||d}}|Scc}w)a  Return a dict of interesting table areas on the PDF page. The table areas are calculated using relevant text edges. Parameters ---------- textlines : list List of text line objects that are relevant for determining table areas. relevant_textedges : list List of relevant text edge objects used to identify table areas. Returns ------- dict A dictionary with padded table areas as keys and None as values. c4|j |jfSr)rCrrs r$r,z+TextEdges.get_table_areas..s/Ar&rN)sort_initialize_table_areas"_extend_table_areas_with_textlines"_calculate_average_textline_height_pad)r"rrelevant_textedges table_areasaverage_textline_heightareapadded_table_areass r$get_table_areaszTextEdges.get_table_areass& $AB223EF  // YG#'"I"I)"TGR ?CDIId3 4d :  "!  s A*ci}|D]H}|s2d||j|j|j|jf<7|j||J|S)aZ Initialize table areas based on relevant text edges. Parameters ---------- relevant_textedges : list List of relevant text edge objects used to initialize table areas. Returns ------- dict A dictionary of table areas initialized from relevant text edges. N)rrCrD_update_table_areas)r"rrrxs r$rz!TextEdges._initialize_table_areas#sV $ :BBF RXXruubhh>?((b9  : r&cd}|D]+}|j|dk\s|j|dks)|}n|2d||j|j|j|jf<y|j||dt |j|dt |d|jt |d|jf}d||<y)aA Update table areas by checking for overlaps with new text edges. Parameters ---------- table_areas : dict Current table areas to be updated. te : object The new text edge object to check for overlaps. Returns ------- None Nr rrr-)rDrCrpopminr)r"rrxfoundr updated_areas r$rzTextEdges._update_table_areas:s DuuQBEET!W$4   =>BK255"((BEE: ; OOE "aBEE58$E!Hbhh'E!Hbee$ L )-K %r&cl|D]}d}|D]+}|j|dk\s|j|dks)|}n|8|j|t|j|dt|j|dt |d|j t |d|jf}d||<y)aT Extend table areas based on text lines that overlap vertically. Parameters ---------- table_areas : dict Current table areas to be extended. textlines : list List of text line objects relevant for extending table areas. Returns ------- None Nr rrr-)rCrDrrx0rx1)r"rrr}rrrs r$rz,TextEdges._extend_table_areas_with_textlines\s 1BE# 55DG#a(8 E    &uQx(uQx(a"%%(a"%%(   -1 L)! 1r&c\td|D}|r|tt|z SdS)a Calculate the average height of text lines. Parameters ---------- textlines : list List of text line objects. Returns ------- float The average height of the text lines. c3NK|]}|j|jz ywr)rDrC)rr}s r$rz?TextEdges._calculate_average_textline_height..s!DB"%%"%%-!Ds#%r)rr7r3)r"rsum_textline_heights r$rz,TextEdges._calculate_average_textline_height}s1"!D)!DD>G"U3y>%::NQNr&cl|dtz }|dtz }|dtz}|d|dzz}||||fS)a\ Pad a given area by a constant value. Parameters ---------- area : tuple The area to be padded defined as (x0, y0, x1, y1). average_row_height : float The average height of rows to use for padding. Returns ------- tuple The padded area. rr r-r)TABLE_AREA_PADDING)r"raverage_row_heightrrCrrDs r$rzTextEdges._padsT !W) ) !W) ) !W) ) !W)A- -BBr&rS)r:r;r<r=r%r^ryrcr~rrrrrrrrUrVs@r$rsrssI!00 @,  <"@. -D1BO" r&rsceZdZdZdZdZedZejdZed dZ ed dZ edZ y ) CellaDefines a cell in a table. With coordinates relative to a left-bottom origin. (PDF coordinate space) Parameters ---------- x1 : float x-coordinate of left-bottom point. y1 : float y-coordinate of left-bottom point. x2 : float x-coordinate of right-top point. y2 : float y-coordinate of right-top point. Attributes ---------- lb : tuple Tuple representing left-bottom coordinates. lt : tuple Tuple representing left-top coordinates. rb : tuple Tuple representing right-bottom coordinates. rt : tuple Tuple representing right-top coordinates. left : bool Whether or not cell is bounded on the left. right : bool Whether or not cell is bounded on the right. top : bool Whether or not cell is bounded on the top. bottom : bool Whether or not cell is bounded on the bottom. text : string Text assigned to cell. c||_||_||_||_||f|_||f|_||f|_||f|_d|_d|_ d|_ d|_ d|_ y)NFr/) rrDx2y2lbltrbrtrrrr_textr"rrDrrs r$r%z Cell.__init__smr(r(r(r(    r&c t|j}t|j}t|j}t|j}d|d|d|d|d S)Nz >r&N)returnbool) r:r;r<r=r%r5propertyrsetterrrrr>r&r$rrsz%N 9 [[..////??r&rceZdZdZdZdZdZedZedZ dZ dZ dd Z d Z d Zd Zd ZdZdZddZdZdZdZdZdZdZdZdZy)TableaDefines a table with coordinates relative to a left-bottom origin. (PDF coordinate space) Parameters ---------- cols : list List of tuples representing column x-coordinates in increasing order. rows : list List of tuples representing row y-coordinates in decreasing order. Attributes ---------- df : :class:`pandas.DataFrame` shape : tuple Shape of the table. accuracy : float Accuracy with which text was assigned to the cell. whitespace : float Percentage of whitespace in the table. filename : str Path of the original PDF order : int Table number on PDF page. page : int PDF page number. c||_||_|Dcgc]*}|Dcgc]}t|d|d|d|dc},c}}|_t j |_d|_d|_d|_ d|_ d|_ d|_ d|_ d|_d|_d|_d|_d|_d|_ycc}wcc}}w)Nrr )rr)colsrowsrcellspd DataFramedfshapeaccuracy whitespacefilenameorderpageflavorpdf_size_bboxparse parse_details_image _image_path)r"rrrcs r$r%zTable.__init__s  LPQqTBtAaD!A$!ad3BQ ,,.         ! CQs C!C CCcPd|jjd|jdS)zReturn a string representation of the class . Returns ------- [type] [description] >**+74::,a@@r&c|j|jk(r|j|jkry|j|jkryy)zReturn True if the two pages are less than the current page . Parameters ---------- other : [type] [description] Returns ------- [type] [description] TN)rr)r"others r$__lt__z Table.__lt__:s? 99 "zzEKK' 99uzz ! "r&cg}|jD]9}|j|Dcgc]}|jjc};|Scc}w)z1Returns two-dimensional list of strings in table.)rr8rr|)r"drowcells r$dataz Table.dataMsH :: :C HHC8Ddiioo'8 9 :9s!A ct|jdt|jd|j|jd}|S)zxReturns a parsing report. with % accuracy, % whitespace, table number on page and page number. r-)rrrr)rJrrrr)r"reports r$parsing_reportzTable.parsing_reportUs<dmmQ/3ZZII    r&cf|j|jjttjj |j d|_td}|j|j |jtj|j|_|jS)zCompute pdf image and cache it.z.pngT) use_fallback) rrrospathbasenamerr convertcv2imread)r"backends r$ get_pdf_imagezTable.get_pdf_imagees ;; '#>GG$$T]]3V$ 1dC t/?/?@**T%5%56DK{{r&cn|jD]%}|D]}dx|_x|_x|_|_ '|S)zSet all table edges to True.T)rrrrr)r"rrs r$ set_all_edgeszTable.set_all_edgesqsK:: GC GBFF FDJFDK G G r&cN|j|||j|||S)aSet the edges of the joint. Set a cell's edges to True depending on whether the cell's coordinates overlap with the line's coordinates within a tolerance. Parameters ---------- vertical : list List of detected vertical lines. horizontal : list List of detected horizontal lines. joint_tol : int, optional Tolerance for determining proximity, by default 2 )_set_vertical_edges_set_horizontal_edges)r"vertical horizontal joint_tols r$ set_edgeszTable.set_edgesxs)   95 "":y9 r&cht|D]$\}}tj||d|s"|cSy)NrrL) enumeraterNrO)r"rirrirs r$_find_close_pointzTable._find_close_points8f% DAq||E1Q4; r&c,|D]}|j|j|d|}|&|j|j|d|}|t|j}|j|j|d|}|j |||y)Nrr r)rrr3r_update_vertical_edges)r"rrvstartendrs r$rzTable._set_vertical_edgess 7A**499adIFE}((AaD)DC{$))n&&tyy!A$ BA  ' 'sA 6 7r&cv|Gt|jdz }|dk\r)t||D]}d|j||_yy|dk(r)t||D]}d|j|d_yt||D]3}d|j||_d|j||dz _5yNr rT)r3rrangerrrr"r r indexjs r$r zTable._update_vertical_edgess = NQ&Ezuc*6A15DJJqM%(.6aZ5#& -(, 1 a % -5#& 6,0 1 e$)15 1 eai(. 6r&c,|D]}|j|j|d|}|&|j|j|d|}|t|j}|j|j|d|}|j |||y)Nrr-r )rrr3r_update_horizontal_edges)r"rrhr r rs r$rzTable._set_horizontal_edgess 9A**499adIFE}((AaD)DC{$))n&&tyy!A$ BA  ) )%a 8 9r&cv|Gt|jdz }|dk\r)t||D]}d|j||_yy|dk(r)t||D]}d|jd|_yt||D]3}d|j||_d|j|dz |_5yr)r3rrrrrrs r$rzTable._update_horizontal_edgess = NQ&Ezuc*7A26DJJu%a(/7aZ5#& ,'+ 1 a $ ,5#& 7+/ 5!!$(26 519%a(/ 7r&ct|j}t|j|dk(sdk(r|St|j|k7st fd|jDr t dt |D]3}d|j|d_d|j|dz _5t D]3}d|jd|_ d|j|dz |_ 5|S)z Sets table border edges to True.rc3:K|]}t|k7ywrr3)rrnum_colss r$rz#Table.set_border..s. %(CH . szQInconsistent cells structure: cells should match the dimensions of rows and cols.Tr ) r3rrrany ValueErrorrrrrr)r"num_rows row_index col_indexrs @r$ set_borderzTable.set_borderstyy>tyy> q=HMK tzz?h &#. ,0JJ. + c  x =I,0DJJy !! $ )8I+/DJJqM) $ (9=DJJx!| $Y / 6 > r&Nct||S|D].}|dk(r|j|dk(s|j0|S)a Copies over text in empty spanning cells. Parameters ---------- copy_text : list of str, optional (default: None) Select one or more of the following strings: {'h', 'v'} to specify the direction in which text should be copied over when a cell spans multiple rows or columns. Returns ------- camelot.core.Table The updated table with copied text in spanning cells. rr )_copy_horizontal_text_copy_vertical_text)r" copy_text directions r$copy_spanning_textzTable.copy_spanning_textsM  K" +IC**,c!((*  +  r&ctt|jD]}tt|j|D]}|j||jj dk(s1|j||j sN|j||j rk|j||dz j|j||_y)a Copies text horizontally in empty spanning cells. This method iterates through the cells and fills empty cells that span horizontally with the text from the left adjacent cell. Returns ------- None r/r N)rr3rrr|rrr"rrs r$r"zTable._copy_horizontal_textss4::' FA3tzz!}-. FJJqM!$))//1R7 1 a(.. JJqM!,11,0JJqM!a%,@,E,EDJJqM!$)  F Fr&ctt|jD]}tt|j|D]}|j||jj dk(s1|j||j sN|j||j rk|j|dz |j|j||_y)z Copies text vertically in empty spanning cells. This method iterates through the cells and fills empty cells that span vertically with the text from the top adjacent cell. Returns ------- None r/r N)rr3rrr|rrr(s r$r#zTable._copy_vertical_textss4::' FA3tzz!}-. FJJqM!$))//1R7 1 a(.. JJqM!,00,0JJq1u,=a,@,E,EDJJqM!$)  F Fr&c nddddd}|j||jj|fi|y)zWrite Table(s) to a comma-separated values (csv) file. For kwargs, check :meth:`pandas.DataFrame.to_csv`. Parameters ---------- path : str Output filepath. utf-8Fr )encodingrheaderquotingN)updaterto_csv)r"rkwargskws r$r0z Table.to_csv(s5"EUq Q &t"r"r&c ddi}|j||jjdi|}t|d5}|j |dddy#1swYyxYw)zWrite Table(s) to a JSON file. For kwargs, check :meth:`pandas.DataFrame.to_json`. Parameters ---------- path : str Output filepath. orientrecordswNr>)r/rto_jsonopenwrite)r"rr1r2 json_stringfs r$r7z Table.to_json7s] " &%dggoo++ $_ ! GGK  ! ! !s AA"c ddi}d|jd|j}|j|tj|}|j j |fd|i|y)zWrite Table(s) to an Excel file. For kwargs, check :meth:`pandas.DataFrame.to_excel`. Parameters ---------- path : str Output filepath. r,r+page--table- sheet_nameN)rrr/r ExcelWriterrto_excel)r"rr1r2r?writers r$rAzTable.to_excelHs`' "TYYKwtzzl;  &%=J="=r&c |jjdi|}t|dd5}|j|dddy#1swYyxYw)zWrite Table(s) to an HTML file. For kwargs, check :meth:`pandas.DataFrame.to_html`. Parameters ---------- path : str Output filepath. r6r+r,Nr>)rto_htmlr8r9)r"rr1 html_stringr;s r$rEz Table.to_htmlYsJ&dggoo// $g . !! GGK  ! ! ! AAc |jjdi|}t|dd5}|j|dddy#1swYyxYw)zWrite Table(s) to a Markdown file. For kwargs, check :meth:`pandas.DataFrame.to_markdown`. Parameters ---------- path : str Output filepath. r6r+rDNr>)r to_markdownr8r9)r"rr1 md_stringr;s r$rIzTable.to_markdownhsL(DGG''1&1 $g . ! GGI    rGc ddd}|j|tj|}d|jd|j}|j j ||fi||j|jy)zWrite Table(s) to sqlite database. For kwargs, check :meth:`pandas.DataFrame.to_sql`. Parameters ---------- path : str Output filepath. r2F) if_existsrr=r>N) r/sqlite3connectrrrto_sqlcommitclose)r"rr1r2conn table_names r$ to_sqlitezTable.to_sqlitewsm%u 5 &t$TYYKwtzzl; z4.2.  r&)r-r)r:r;r<r=r%r5rrrrrrrrrr rrr r&r"r#r0r7rArErIrTr>r&r$rrs> (A&   ( 7 6 9 7:6F(F( #!">" ! r&rc:eZdZUdZded<ded<ded<ded<y) _Kwz.Helper class to define file related arguments.zos.PathLike[Any] | strrstrdirnamerootextN)r:r;r<r=__annotations__r>r&r$rVrVs8  L I Hr&rVcxeZdZdZddZdZdZdZddZddZ e dZ e dd Z ddd Zdd Zddd Zy ) TableListzDefines a list of camelot.core.Table objects. Each table can be accessed using its index. Attributes ---------- n : int Number of tables in the list. c||_yr_tables)r"tabless r$r%zTableList.__init__s (. r&cPd|jjd|jdS)Nrz n=rI)rFr:nrs r$r5zTableList.__repr__s&4>>**+3tvvha88r&c,t|jSr)r3r`rs r$__len__zTableList.__len__s4<<  r&c |j|Srr_)r"idxs r$ __getitem__zTableList.__getitem__s||C  r&c,t|jSr)iterr`rs r$__iter__zTableList.__iter__sDLL!!r&ct|Sr)nextrs r$__next__zTableList.__next__s Dzr&c t|d|S)Nto_)getattr)tabler;s r$ _format_funczTableList._format_funcsuA3i((r&ct|S)z!The number of tables in the list.rrs r$rcz TableList.ns4yr&Nc |d}|d}|d}|jD]\}|d|jd|j|}tjj ||}|j ||} | |^y)NrXrYrZ-page-r>)r`rrrrr0rs) r"r;r1rXrYrZrrrfilepath to_formats r$ _write_filezTableList._write_files#f~Um\\ Evejj\ cUKHww||GX6H))%3I h   r&c |d}|d}|d}|d}tjjtjj||dz}t j |dd5}|j D]q}|d |jd |j|} tjj|| } |j| tjj| s dddy#1swYyxYw) NrrXrYrZ.zipr6T allowZip64rvr>) rrr0rXzipfileZipFiler`rrr9r) r"r1rrXrYrZzipnamezrrrrws r$ _compress_dirzTableList._compress_dirsf~#f~Um'',,rwwt4d;fD __Wcd ; >q >"V6%**WU[[M#O77<<:"''"2"28"<= > > > >s -BC88Dctjj|}tjj|}tjj |\}}|rt j }||||d}|dvr*|jdd|i||r|jdi|y y |dk(r#tjj||} tj| } |jD];} d| jd| j} | jj!| | =| j#|rtjjtjj||dz} t%j&| d d 5}|j)| tjj| d d d y y |d k(rtjj||} |jD]} | j+| |rtjjtjj||dz} t%j&| d d 5}|j)| tjj| d d d y y y #1swYy xYw#1swYy xYw)aBExport the list of tables to specified file format. Parameters ---------- path : str Output filepath. f : str File format. Can be csv, excel, html, json, markdown or sqlite. compress : bool Whether or not to add files to a ZIP archive. )rrXrYrZ)csvhtmljsonmarkdownr;excelr=r>)r?r{r6Tr|Nsqliter>)rrrXrsplitexttempfilemkdtempryrr0rr@r`rrrrArQr~rr9rT)r"rr;compressrXrrYrZr1rwrBrrr?rrs r$exportzTableList.exportsP''//$'77##D)GG$$X. c &&(G#cR 3 3 D   +q +F +""",V, '\ww||GX6H^^H-F A$UZZL }E !!&Z!@ A LLN'',,rwwt'r&r$r]r]s] /9!!"))  >+Br&r]))r= __future__rrNrrMsysrr~operatorrtypingrrrrpandasr version_infor r typing_extensionsbackendsr utilsrrrrPrruVERTICAL_ALIGNMENTSALL_ALIGNMENTSrr@rXrsrrrVr]r>r&r$rs<"   w 3,.*& 41&)<<0(0(f3%}3%l/B/Bd\ \ ~S?S?lLL^  ) hBhBr&