Si7ddlmZddlmZddlmZddlmZddlmZmZmZddl m Z ddl Z ddl Z ddl Z ddlZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZddlmZmZmZGddeZGdde eZ!y))absolute_import)division)print_function)unicode_literals)ProxyGeneratorMaxTriesExceededException DOSException) BeautifulSoupN)Timeout)TimeoutException)By_SearchScholarIterator) AuthorParser)PublicationParser)AuthorPublicationSource ProxyModec"eZdZiZfdZxZS) Singletonc~||jvr tt||i||j|<|j|SN) _instancessuperr__call__)clsargskwargs __class__s L/home/agent/.friday_env/lib/python3.12/site-packages/scholarly/_navigator.pyrzSingleton.__call__sD cnn $"' 3"@$#KCI#KCNN3 ~~c"")__name__ __module__ __qualname__rr __classcell__r s@r!rrsJ##r"rc eZdZdZfdZdefdZdefdZd!de d e fd Z d"d Z d#d e d ede fdZ deddfdZdefdZd"defdZdefdZde defdZde defdZ d#de dedefdZde defdZd$de dede dedef dZde dedefd ZxZS)% Navigatorz1A class used to navigate pages on google scholar.c@tt| tjd|_d|_d|_t|_ t|_ |jj|_ |jj|_ d|_y)N scholarlyF)rr)__init__logging getLoggerlogger_TIMEOUT _max_retriesrpm1pm2 get_session _session1 _session2got_403)selfr s r!r-zNavigator.__init__#sv i')'' 4  "#!#--/--/ r"enablec|jj|rtjytjy)z0Enable or disable the logger for google scholar.N)r0setLevelr.INFOCRITICAL)r9r:s r! set_loggerzNavigator.set_logger2s' fgllL':J:JLr"timeoutc|dk\r||_yy)z+Set timeout period in seconds for scholarlyrN)r1)r9r@s r! set_timeoutzNavigator.set_timeout7s a<#DM r"Npg1pg2c@|||_|||_nMt|_|jj}|s"|jj d||_|jj |_|jj |_y)NzWFreeProxy as a secondary proxy is not working. Using the primary proxy for all requests) r3r4r FreeProxiesr0infor5r6r7)r9rCrD proxy_workss r! use_proxyzNavigator.use_proxy<s ?DH ?DH%'DH((..0K   "LM--/--/r"c d|_|r"|jjdi||_y|jjdi||_y)NF)r8r3 _new_sessionr6r4r7)r9premiumrs r!rLzNavigator._new_sessionMsB 2TXX22t@f$r} dtC| jDd| jF} |jj| |d|jzkr0|jjd ||jz}Yd} ~ *|jjd!Yd} ~ d} ~ wtH$rf} d"tC| jDd| jF} |jj| |jjd#Yd} ~ d} ~ wwxYw#tH$r|jjd&YwxYw)(aPReturn the data from a webpage :param pagerequest: the page url :type pagerequest: str :param premium: whether or not to use the premium proxy right away :type premium: bool :returns: the text from a webpage :rtype: {str} :raises: MaxTriesExceededException, DOSException z Getting %sNrz citations?FT<r)r@zSession proxy config is {}iz+Got a 404 error. Attempting with same proxyzGot a captcha request.iz!Got an access denied error (403).zNo other connections possible.z*Retrying immediately with another session.xz5Will retry after %.2f seconds (with another session).)rMz0We can use another connection... let's try that.i.zGot a redirect.locationzAResponse code %d. Retrying...z6Will retry after %.2f seconds (with the same session).zTimeout Exception z while fetching page: z4Increasing timeout and retrying within same session.zGiving up this session.z Exception zRetrying with a new session.http) num_tries old_timeout old_proxyzRNo other secondary connections possible. Using the primary proxy for all requests.z!Cannot Fetch from Google Scholar.)(r0rGr4r7r3r6 proxy_moder SCRAPERAPIrBr1r2randomuniformtimesleepgetdebugformat_proxies_requests_has_captchatext status_code_handle_captcha2 has_proxyr8LUMINATIrLhas_redirect_locationheadersr r r typer#r Exceptionget_next_proxy _get_pager ) r9rNrMresptriespmsessionr@w has_captchaeerrs r!rpzNavigator._get_pageUs {3 K ''BnnGGBnnGG ==I00 0   R  d'''A ANN1Q' 1 {{;{@e#KK%%&B&I&I"++&VW"88C ##s*;99$%%, KK%%&STQJE KK$$%=> 11+>G%%,KK$$%HI<<> (()IJ#|| KK,,-YZ!}}Y5G5GI]I]4^^$*NN2t$< $ 0 01hjk l $ 1 ))'):'+   (()[\%%,1K1KKK%%&78"&,,z":KKK$$&3484D4DF. QJE #%#4#4V]ikititixixyBFjG#4$H Md'''Z>>+t4 4+,OP PA  ||~KK$$%EFr40AKK$$%]_`aJJqM & -. z1Navigator._requests_has_captcha..saS{d*r"cd|dvS)Nzclass="rrK)crfs r!rz1Navigator._requests_has_captcha..s1~-r") _has_captcha)r9rfs `r!rezNavigator._requests_has_captchas  * -  r"ch|r |jn |j|jfdfdS)zTests whether the current webdriver page contains a captcha. :returns: whether or not the site contains a captcha :rtype: {bool} cxtjjtj|dkDSNr)len_get_webdriver find_elementsrID)rrss r!rz2Navigator._webdriver_has_captcha..s+s2,,.<.s,s2,,.<>zJDK     s'A)) A65A6c#K|j|}t|} |jdd}|jj dt ||D]}|j |d}d}|j||z}|rad|jvrS|jj d|d d d }tjd |d }|j|}n|jj dyw)zAGenerator that returns Author objects from the author search pagergsc_1usrzFound %d authorszgs_btnPR gs_in_ib gs_btn_half z!gs_btn_lsb gs_btn_srt gsc_pgn_pnx)class_disabledzLoading next page of authorsonclickunicode_escaperzNo more author pagesN) rrfind_allr0rGr get_authorrattrscodecs getdecoder) r9rsoup author_parserrowsrowcls1cls2 next_buttons r!search_authorszNavigator.search_authorss~~c"$T* == 3D KK  /T ; 4#..s33 43D6D))49)5Kz1B1BB   !?@!),R39f''(89#>qA~~c*   !78sDD filledc|j|}t|}|j|jdddtj }|r|j |}|S)aFSearch by scholar query and return a single Publication object :param url: the url to be searched at :type url: str :param filled: Whether publication should be filled, defaults to False :type filled: bool, optional :returns: a publication object :rtype: {Publication} rgs_orr)rrget_publicationrrPUBLICATION_SEARCH_SNIPPETfill)r9rrrpublication_parserpubs r!search_publicationzNavigator.search_publicationsa~~c".t4 00ug1Nq1QSdSSA $))#.C r"ct||S)zReturns a Publication Generator given a url :param url: the url where publications can be found. :type url: str :returns: An iterator of Publications :rtype: {_SearchScholarIterator} r)r9rs r!search_publicationszNavigator.search_publications s&dC00r"rsortbypublication_limitct|}|j|}|r|j|||}|S|j|dg||}|S)aSearch by author ID and return a Author object :param id: the Google Scholar id of a particular author :type url: str :param filled: If the returned Author object should be filled :type filled: bool, optional :param sortby: if the object is an author, select the order of the citations in the author page. Either by 'citedby' or 'year'. Defaults to 'citedby'. :type sortby: string :param publication_limit: Select the max number of publications you want you want to fill for the author. Defaults to no limit. :type publication_limit: int :returns: an Author object :rtype: {Author} )rrbasics)sectionsrr)rrr)r9rrrrrrs r!search_author_idzNavigator.search_author_id*sc%T* &&r* $$SK\$]C  $$SH:f`q$rC r" fromauthorc|j|}|jdd}|r|jjdg}|D]J}|j |j j |j djddddL|gk(r\|durX t|j|}|j|jj}d|d <|j ||S|S#t$rg}Y|SwxYw) zGenerate instiution object from author search page. if no results are found and `fromuthor` is True, then use the first author from the search to get institution/organization name. h3 gsc_inst_reszFound institutionhrefzorg=r) OrganizationrTr)rrr0rGappendarfsplitnextrrr organizationrn) r9rrrrrrauthauthorgs r!search_organizationzNavigator.search_organization?s ~~c"}}T>2  KK  0 1 ^C JJ #%%-:M:MfVW:XYZ:[\ ] ^ 2:*, D//45//8EE(, % 7# s    sAC55 DDr)T)F)Fcitedbyr)r#r$r%__doc__r-boolr?intrBrrIrLstrrpr|rerrr rrrrrrrrlistrr&r's@r!r)r) s6; MM $3$ 0^0.0"=iQSiQ4iQCiQX(((  T    d   6 6 S ] #.+0c#'4E$1s1/E13cjmv|*sr"r)) metaclass)" __future__rrrr_proxy_generatorrr r bs4r rr.r]r_requests.exceptionsr httpxr selenium.webdriver.common.byrrrrrr data_typesrrrrmrobjectr)rKr"r!rsZ&%'UU  '"+6'1<<##v)vr"