
    &th2j                        d dl Z d dlZd dlmZmZmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d d	lmZ d dlZd d
lmZ d dlmZ d dlmZ d dl m!Z" d dl#m$Z% d dl#m&Z&m'Z' d dl(m)Z) dZ*dZ+dZ,g dZ- e            Z.de._/        de._0        da1eed         df         Z2eee3         ed         e3df         Z4eed         df         Z5ee3e6f         Z7e3Z8e3Z9e3Z:e3Z;e6Z<e3Z=eZ>e3Z?e'Z@ee3ee3         f         ZAee3         ZBeZCedee3         f         ZDee3         ZE	 e jF        jG        dk    sJ e jF        jH        dk    sJ n# eI$ r  eJd          w xY w G d deK          ZL G d d          ZM G d  deM          ZN G d! deM          ZO G d" d#ejP                  ZQd.d$e9fd%ZRd/d'ZS G d( d)ejT                  ZU G d* d+eU          ZV G d, d-eU          ZWdS )0    N)urlparse
urlunparseurljoin)ThreadPoolExecutor)TimeoutError)partial)SetUnionListMutableMappingOptional)PyQuery)	UserAgent)Cleaner)etree)HtmlElement)tostring)
fromstring)search)findallResult)html_to_unicodezutf-8zhttps://example.org/zuMozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8)nextmoreolderTElementr   HTML      z#Requests-HTML requires Python 3.6+!c                       e Zd Zd ZdS )
MaxRetriesc                     || _         d S N)message)selfr$   s     a/var/www/html/mycamper/aliexpress-site/backend/venv/lib/python3.11/site-packages/requests_html.py__init__zMaxRetries.__init__A   s        N)__name__
__module____qualname__r'    r(   r&   r!   r!   ?   s#            r(   r!   c                   B   e Zd ZdZddddedededdfdZede	fd	            Z
edefd
            Zej        deddfd            Ze
j        deddfd            Z
edefd            Zej        deddfd            Zedefd            Zedefd            Zedefd            Zedefd            Zd&ddddddedededededefdZdddddededededef
dZdedefd Z dede!fd!Z"ede#fd"            Z$d# Z%ede#fd$            Z&edefd%            Z'dS )'
BaseParseraJ  A basic HTML/Element Parser, for Humans.

    :param element: The element from which to base the parsing upon.
    :param default_encoding: Which encoding to default to.
    :param html: HTML from which to base the parsing upon (optional).
    :param url: The URL from which the HTML originated, used for ``absolute_links``.

    N)default_encodinghtmlr/   r0   urlreturnc                    || _         || _        d| _        || _        d | _        t          |t                    r|                    t                    n|| _	        d | _
        d | _        d S )NT)elementr1   skip_anchorsr/   	_encoding
isinstancestrencodeDEFAULT_ENCODING_html_lxml_pq)r%   r4   r/   r0   r1   s        r&   r'   zBaseParser.__init__O   sb      06@s6K6KUT[[!1222QU

r(   c                     | j         r| j         S t          j        | j        d                                                              | j                  S )zyBytes representation of the HTML content.
        (`learn more <http://www.diveintopython3.net/strings.html>`_).
        unicodeencoding)r;   r   r   r4   stripr9   rA   r%   s    r&   raw_htmlzBaseParser.raw_htmlY   sL    
 : 	b:>$,CCCIIKKRRSWS`aaar(   c                     | j         r!| j                            | j        d          S t	          j        | j        d                                          S )zzUnicode representation of the HTML content
        (`learn more <http://www.diveintopython3.net/strings.html>`_).
        replaceerrorsr?   r@   )r;   rD   decoderA   r   r   r4   rB   rC   s    r&   r0   zBaseParser.htmlc   sO    
 : 	L=''i'HHH>$,CCCIIKKKr(   c                 D    |                     | j                  | _        d S r#   )r9   rA   r;   r%   r0   s     r&   r0   zBaseParser.htmlm   s    [[//


r(   c                     || _         dS )zProperty setter for self.html.N)r;   rK   s     r&   rD   zBaseParser.raw_htmlq   s     


r(   c                    | j         r| j         S | j        rdt          | j        | j                  d         | _         	 | j                            | j        d           n# t          $ r | j        | _         Y nw xY w| j         r| j         n| j        S )zzThe encoding string to be used, extracted from the HTML and
        :class:`HTMLResponse <HTMLResponse>` headers.
        r   rF   rG   )r6   r;   r   r/   rD   rI   rA   UnicodeDecodeErrorrC   s    r&   rA   zBaseParser.encodingv   s    
 > 	">! : 	7,T-BDJOOPQRDN7$$T]9$EEEE% 7 7 7!%!67 "&Jt~~T5JJs   !A A76A7encc                     || _         dS )z"Property setter for self.encoding.N)r6   )r%   rO   s     r&   rA   zBaseParser.encoding   s     r(   c                 P    | j         t          | j                  | _         | j         S )z`PyQuery <https://pythonhosted.org/pyquery/>`_ representation
        of the :class:`Element <Element>` or :class:`HTML <HTML>`.
        )r=   r   lxmlrC   s    r&   pqzBaseParser.pq   s$    
 8ty))DHxr(   c                     | j         V	 t          | j        d          | _         n9# t          $ r, t          j                            | j                  | _         Y nw xY w| j         S )zs`lxml <http://lxml.de>`_ representation of the
        :class:`Element <Element>` or :class:`HTML <HTML>`.
        Nzhtml.parser)features)r<   
soup_parser0   
ValueErrorrR   r   rD   rC   s    r&   rR   zBaseParser.lxml   sp    
 :A'	MJJJ

 A A A!Y11$-@@


A zs   % 3AAc                 4    | j                                         S )z\The text content of the
        :class:`Element <Element>` or :class:`HTML <HTML>`.
        )rS   textrC   s    r&   rY   zBaseParser.text   s    
 w||~~r(   c                 4    | j                                         S )zsThe full text content (including links) of the
        :class:`Element <Element>` or :class:`HTML <HTML>`.
        )rR   text_contentrC   s    r&   	full_textzBaseParser.full_text   s    
 y%%'''r(   *F)
containingcleanfirstr6   selectorr^   r_   r`   r6   c                6   	 t          |t                    r|g}|p j        		 fd                     |          D             }|r_|                                }g }|D ]2t          fd|D                       r|                               3|                                 |ra|                                }g }|D ]Ht          t          
                    j                            _        |                               It          ||          S )aC  Given a CSS Selector, returns a list of
        :class:`Element <Element>` objects or a single one.

        :param selector: CSS Selector to use.
        :param clean: Whether or not to sanitize the found HTML of ``<script>`` and ``<style>`` tags.
        :param containing: If specified, only return elements that contain the provided text.
        :param first: Whether or not to return just the first result.
        :param _encoding: The encoding format.

        Example CSS Selectors:

        - ``a``
        - ``a.someClass``
        - ``a#someID``
        - ``a[target=_blank]``

        See W3School's `CSS Selectors Reference
        <https://www.w3schools.com/cssref/css_selectors.asp>`_
        for more details.

        If ``first`` is ``True``, only returns the first
        :class:`Element <Element>` found.
        c                 >    g | ]}t          |j                   S r4   r1   r/   )r   r1   ).0foundrA   r%   s     r&   
<listcomp>z#BaseParser.find.<locals>.<listcomp>   s:     
 
 
 Etx(KKK
 
 
r(   c                 j    g | ]/}|                                 j                                         v 0S r,   )lowerr\   )rf   cr4   s     r&   rh   z#BaseParser.find.<locals>.<listcomp>   s5    SSS1		W%6%<%<%>%>>SSSr(   )r7   r8   rA   rS   copyanyappendreverselxml_html_tostringcleaner
clean_htmlrR   rD   _get_first_or_list)
r%   ra   r^   r_   r`   r6   elementselements_copyr4   rA   s
   `       @@r&   findzBaseParser.find   sJ   4 j#&& 	&$J-
 
 
 
 
**
 
 

  	$MMOOMH( - -SSSS
SSSTT -OOG,,,  	)$MMOOMH( ) )#5g6H6H6V6V#W#W ((((!(E222r(   )r_   r`   r6   c                >     j                             |          } fd|D             }|ra|                                }g }|D ]H}t          t                              |j                             |_        |                    |           It          ||          S )a  Given an XPath selector, returns a list of
        :class:`Element <Element>` objects or a single one.

        :param selector: XPath Selector to use.
        :param clean: Whether or not to sanitize the found HTML of ``<script>`` and ``<style>`` tags.
        :param first: Whether or not to return just the first result.
        :param _encoding: The encoding format.

        If a sub-selector is specified (e.g. ``//a/@href``), a simple
        list of results is returned.

        See W3School's `XPath Examples
        <https://www.w3schools.com/xml/xpath_examples.asp>`_
        for more details.

        If ``first`` is ``True``, only returns the first
        :class:`Element <Element>` found.
        c                     g | ]I}t          |t          j                  st          |j        pj                   nt          |          JS rd   )r7   r   _ElementUnicodeResultr   r1   rA   r8   )rf   	selectionr6   r%   s     r&   rh   z$BaseParser.xpath.<locals>.<listcomp>  sf     
 
 
  i)DEEZGI48iF`SWS`aaaaKNy>>
 
 
r(   )	rR   xpathrl   rp   rq   rr   rD   rn   rs   )	r%   ra   r_   r`   r6   selectedrt   ru   r4   s	   `   `    r&   r{   zBaseParser.xpath   s    & 9??8,,
 
 
 
 
 &
 
 
  	)$MMOOMH( ) )#5g6H6H6V6V#W#W ((((!(E222r(   templatec                 ,    t          || j                  S )zSearch the :class:`Element <Element>` for the given Parse template.

        :param template: The Parse template to use.
        )parse_searchr0   r%   r}   s     r&   r   zBaseParser.search  s     Hdi000r(   c                 @    d t          || j                  D             S )zSearch the :class:`Element <Element>` (multiple times) for the given parse
        template.

        :param template: The Parse template to use.
        c                     g | ]}|S r,   r,   )rf   rs     r&   rh   z)BaseParser.search_all.<locals>.<listcomp>   s    888a888r(   )r   r0   r   s     r&   
search_allzBaseParser.search_all  s$     9878TY778888r(   c                 <      fd}t           |                      S )u)   All found links on page, in as–is form.c               3   
  K                        d          D ]i} 	 | j        d                                         }|r5|                    d          rj        s|                    d          s|V  Z# t
          $ r Y fw xY wd S )Nahref#)zjavascript:zmailto:)rv   attrsrB   
startswithr5   KeyError)linkr   r%   s     r&   genzBaseParser.links.<locals>.gen&  s      		#  :f-3355D #T__S%9%9 #d>O #Y]YhYh  jD  ZE  ZE #"


   D s   AA33
B ?B setr%   r   s   ` r&   linkszBaseParser.links"  s0    	 	 	 	 	 3355zzr(   c                 $   t          |                                          }|d         st          | j        |          S |d         sIt          | j                  j        |d<   d |                                D             }t          |          S |S )zMakes a given link absolute.netlocschemec              3      K   | ]}|V  d S r#   r,   rf   vs     r&   	<genexpr>z,BaseParser._make_absolute.<locals>.<genexpr>A  s"      11Aa111111r(   )r   _asdictr   base_urlr   valuesr   )r%   r   parseds      r&   _make_absolutezBaseParser._make_absolute2  s     $'')) h 	04=$/// h 	&'66=F8 21111Ff%%% r(   c                 <      fd}t           |                      S )zAll found links on page, in absolute form
        (`learn more <https://www.navegabem.com/absolute-or-relative-links.html>`_).
        c               3   N   K   j         D ]}                     |           V  d S r#   )r   r   )r   r%   s    r&   r   z&BaseParser.absolute_links.<locals>.genN  s?      
 0 0))$//////0 0r(   r   r   s   ` r&   absolute_linkszBaseParser.absolute_linksH  s0    	0 	0 	0 	0 	0 3355zzr(   c                    |                      dd          }|r1|j                            dd                                          }|r|S t	          | j                                                  }d                    |d                             d          dd	                   dz   |d<   d
 |	                                D             }t          |          }|S )z}The base URL for the page. Supports the ``<base>`` tag
        (`learn more <https://www.w3schools.com/tags/tag_base.asp>`_).baseT)r`   r    /pathNc              3      K   | ]}|V  d S r#   r,   r   s     r&   r   z&BaseParser.base_url.<locals>.<genexpr>g  s"      --!------r(   )rv   r   getrB   r   r1   r   joinsplitr   r   )r%   r   resultr   r1   s        r&   r   zBaseParser.base_urlT  s     yyty,, 	Z^^FB//5577F  $(##++-- &."6"6s";";CRC"@AACGv .-V]]__---  
r(   )r]   )(r)   r*   r+   __doc___DefaultEncoding_HTML_URLr'   property_RawHTMLrD   	_BaseHTMLr0   setterr8   bytes	_EncodingrA   r   rS   r   rR   _TextrY   r\   _Containingbool_Findrv   _XPathr{   r   r   _Resultr   _Linksr   r   r   r   r,   r(   r&   r.   r.   E   s         IM\`   5E TY gk pt     b( b b b Xb Li L L L XL 
[0 0 0 0 0 [0 _U t    _ K) K K K XK& _C D    _ G    X 
k 
 
 
 X
 e    X (5 ( ( ( X(63TY^ns  GK 63 63 63S 63{ 63RV 63gk 63  AD 63  PU 63 63 63 63p 5:ae $3 $3 $3c $3T $3$ $3[^ $3jp $3 $3 $3 $3L1s 1v 1 1 1 193 97 9 9 9 9 v    X  , 	 	 	 	 X	 $    X  r(   r.   c                   f     e Zd ZdZg dZdddededdf fdZdefd	Z	e
defd
            Z xZS )r   zAn element of HTML.

    :param element: The element from which to base the parsing upon.
    :param url: The URL from which the HTML originated, used for ``absolute_links``.
    :param default_encoding: Which encoding to default to.
    )
r4   r1   r5   r/   r6   r;   r<   r=   _attrssessionN)r/   r1   r/   r2   c                    t          t          |                               |||           || _        |j        | _        |j        | _        d | _        d S )Nre   )superr   r'   r4   tag
sourcelinelinenor   )r%   r4   r1   r/   	__class__s       r&   r'   zElement.__init__z  sN    gt%%g3Qa%bbb;(r(   c                       fd j         D             }d                    t           j        j                  d                    |                    S )Nc           	      l    g | ]0}d                      |t          j        |                             1S )z{}={})formatreprr   )rf   attrr%   s     r&   rh   z$Element.__repr__.<locals>.<listcomp>  s6    UUU$d4:d+;&<&<==UUUr(   z<Element {} {}> )r   r   r   r4   r   r   )r%   r   s   ` r&   __repr__zElement.__repr__  sK    UUUU$*UUU ''T\-=(>(>PPPr(   c                     | j         jd | j                                        D             | _         dD ]?}|| j         v r4t          | j         |                                                   | j         |<   @| j         S )zReturns a dictionary of the attributes of the :class:`Element <Element>`
        (`learn more <https://www.w3schools.com/tags/ref_attributes.asp>`_).
        Nc                     i | ]\  }}||	S r,   r,   )rf   kr   s      r&   
<dictcomp>z!Element.attrs.<locals>.<dictcomp>  s    AAADAq1aAAAr(   )classrel)r   r4   itemstupler   )r%   r   s     r&   r   zElement.attrs  s    
 ;AADL,>,>,@,@AAADK ) I I4;&&(-dk$.?.E.E.G.G(H(HDK%{r(   )r)   r*   r+   r   	__slots__r   r   r'   r8   r   r   _Attrsr   __classcell__r   s   @r&   r   r   m  s           I
 TX    @P \`      Q# Q Q Q Q v    X    r(   c                   J    e Zd ZdZdeeddded         deded	ed
e	ddf fdZ
defdZdefde	dedefdZd Zd Zd Zd Zd Zdddededededee         deeef         de	fdZd'd#edededed$e	deeef         de	fd%Zd'd#edededed$e	deeef         de	fd&Z xZS )(r   a   An HTML document, ready for parsing.

    :param url: The URL from which the HTML originated, used for ``absolute_links``.
    :param html: HTML from which to base the parsing upon (optional).
    :param default_encoding: Which encoding to default to.
    NF)r   r1   r/   async_r   HTMLSessionAsyncHTMLSessionr1   r0   r/   r   r2   c                   t          |t                    r|                    t                    }t	          t
          |                                t          |          d          p t          d| d          d          |||           |p|rt                      pt                      | _
        d | _        t          | _        d S )Nr0   z<html>z</html>)r4   r0   r1   r/   )r7   r8   r9   r:   r   r   r'   r   r   r   r   pageDEFAULT_NEXT_SYMBOLnext_symbol)r%   r   r1   r0   r/   r   r   s         r&   r'   zHTML.__init__  s     dC   	1;;/00DdD""!GDMM&))T-LW5Kd5K5K5K-L-LV-T-T- 	# 	
 	
 	
 P&"?-=-?-?P;==	.r(   c                     d| j         dS )Nz
<HTML url=>)r1   rC   s    r&   r   zHTML.__repr__  s    )DH))))r(   fetchr   c                       fd} |            }|r                      |          }ndS |r j                            |          S |S )zAttempts to find the next page, if there is one. If ``fetch``
        is ``True`` (default), returns :class:`HTML <HTML>` object of
        next page. If ``fetch`` is ``False``, simply returns the next URL.

        c                                          d          } | D ]}|j                            d          r}d|j                            dg           v r|j        d         c S |j                            dg           D ]}d|v r|j        d         c c S d|j        d         v r|j        d         c S 	 | d         j        d         S # t          $ r Y d S w xY w)	Nr   )r^   r   r   r   r   r   r   )rv   r   r   
IndexError)
candidates	candidate_classr   r%   s      r&   get_nextzHTML.next.<locals>.get_next  s   3;??J' 7 7	?&&v.. 7!4!4UB!?!???(v6666 #,/"5"5gr"B"B ; ;!V++#,?6#:::::: , !888(v6666!"~+F33   tts   6C	 	
CCN)r   r   r   )r%   r   r   r   _HTML__nextr1   s   ` `   r&   r   z	HTML.next  st    	 	 	 	 	 	.  	%%f--CC4 	<##C(((Jr(   c              #   ~   K   | }	 |V  	 |                     d| j                  j        }n# t          $ r Y d S w xY w9NTr   r   )r   r   r0   AttributeError)r%   r   s     r&   __iter__zHTML.__iter__  sb      	JJJyyt9IyJJO!   		s   !- 
;;c                 D    |                      d| j                  j        S r   )r   r   r0   rC   s    r&   __next__zHTML.__next__  s    yyt1AyBBGGr(   c                     | S r#   r,   rC   s    r&   	__aiter__zHTML.__aiter__  s    r(   c                    K   	 |                      d| j                  }|sd S | j                            |           d {V }|j        S )NTFr   )r   r   r   r   r0   )r%   r1   responses      r&   	__anext__zHTML.__anext__  s_      	!))%T5E)FFC !\--c22222222H= r(   c                 :    | j                             |           d S r#   )r   rn   )r%   r   s     r&   add_next_symbolzHTML.add_next_symbol  s    ,,,,,r(   )scriptr   sleepwaitcontenttimeout	keep_pagec       	         t  K   	 | j                                          d{V }
t          j        |           d{V  |r0|
                    |dt          |dz            i           d{V  n7|
                    d| j         dt          |dz            i           d{V  d}|r|
                    |           d{V }|rMt          |          D ]<}|
j	        
                    d           d{V  t          j        |           d{V  =nt          j        |           d{V  |r |
j	                            d           d{V  |
                                 d{V }|	s|
                                 d{V  d}
|||
fS # t          $ r  |
                                 d{V  d}
Y dS w xY w)zQ Handle page creation and js rendering. Internal use for render/arender methods. Nr   i  )optionszdata:text/html,PageDown)browsernewPageasyncior   gotointr0   evaluaterange	_keyboarddownupr   closer   )r%   r1   r   
scrolldownr   r   reloadr   r   r  r   r   _s                r&   _async_renderzHTML._async_render  sz     #	--////////D -%%%%%%%%%  iiiiWt^9L9L-MiNNNNNNNNNNii =$) = =	SVW^aeWeSfSfGgihhhhhhhhhF 5#}}V44444444 +z** / /A.--j999999999!-........../ mE********* 4n''
333333333 !LLNN******G jjll"""""""FD(( 	 	 	**,,D44	s   FF &F76F7   皙?r   T       @retriesr  c	                    | j         j        | _        d}	| j        t          k    rd}t	          |          D ]_}
|	s[	 | j         j                            |                     | j        |||| j        ||||	  	                  \  }	}}O# t          $ r Y [w xY w |	st          d          t          | j        |	                    t                    t                    }| j                            |j                   || _        |S )a  Reloads the response in Chromium, and replaces HTML content
        with an updated version, with JavaScript executed.

        :param retries: The number of times to retry loading the page in Chromium.
        :param script: JavaScript to execute upon page load (optional).
        :param wait: The number of seconds to wait before loading the page, preventing timeouts (optional).
        :param scrolldown: Integer, if provided, of how many times to page down.
        :param sleep: Integer, if provided, of how many long to sleep after initial render.
        :param reload: If ``False``, content will not be loaded from the browser, but will be provided from memory.
        :param keep_page: If ``True`` will allow you to interact with the browser page through ``r.html.page``.

        If ``scrolldown`` is specified, the page will scrolldown the specified
        number of times, after sleeping the specified amount of time
        (e.g. ``scrolldown=10, sleep=1``).

        If just ``sleep`` is provided, the rendering will wait *n* seconds, before
        returning.

        If ``script`` is specified, it will execute the provided JavaScript at
        runtime. Example:

        .. code-block:: python

            script = """
                () => {
                    return {
                        width: document.documentElement.clientWidth,
                        height: document.documentElement.clientHeight,
                        deviceScaleFactor: window.devicePixelRatio,
                    }
                }
            """

        Returns the return value of the executed  ``script``, if any is provided:

        .. code-block:: python

            >>> r.html.render(script=script)
            {'width': 800, 'height': 600, 'deviceScaleFactor': 1}

        Warning: the first time you run this method, it will download
        Chromium into your home directory (``~/.pyppeteer``).
        NF	r1   r   r   r   r   r  r  r   r  1Unable to render the page. Try increasing timeoutr1   r0   r/   )r   r  r1   DEFAULT_URLr  looprun_until_completer  r0   	TypeErrorr!   r   r9   r:   __dict__updater   )r%   r  r   r   r  r   r  r   r  r   ir   r   r0   s                 r&   renderzHTML.render  sS   Z |+ 8{""F w 	 	A ,0L,=,P,PQUQcQchlhpy  HM  TX  bf  bk  tz  GQ  [b  nwQc  Rx  Rx  -y  -y)GVTT    D  	RPQQQw~~6F'G'GZjkkkT]+++	s   A	B
BBc	                   K   | j         j         d{V | _        d}	| j        t          k    rd}t	          |          D ]H}
|	sD	 |                     | j        |||| j        ||||	  	         d{V \  }	}}8# t          $ r Y Dw xY w |	st          d          t          | j        |	
                    t                    t                    }| j                            |j                   || _        |S )z1 Async version of render. Takes same parameters. NFr  r  r  )r   r  r1   r  r  r  r0   r  r!   r   r9   r:   r   r!  r   )r%   r  r   r   r  r   r  r   r  r   r  r   r   r0   s                 r&   arenderzHTML.arenderd  sd      "\1111111 8{""Fw 	 	A 262D2DZ`hmtx  CG  CL  U[  hr  |C  OX2D  3Y  3Y  -Y  -Y  -Y  -Y  -Y  -Y)GVTT    D  	RPQQQw~~6F'G'GZjkkkT]+++	s   2A55
BB)r  Nr  Fr   Tr  F)r)   r*   r+   r   r  r:   r
   r8   r   r   r'   r   r   _NextSymbol_Nextr   r   r   r   r   r   r	  floatr   r  r#  r%  r   r   s   @r&   r   r     s         OS_j  Rb  sx / / /5)J#K /Y\ /rw /  LO /  lp /  }A / / / / / /"*# * * * * "'CV ' '$ '[ '[` ' ' ' 'R	 	 	H H H  ! ! !- - - >B % % %# %s %VY %af %  zB  CF  zG %  RW  X]  _b  Xb  Rc %  pt % % % %NE Ec Es E Egj Ex| E  OT  UZ  \_  U_  O` E  sw E E E EN S c  nq   @D   V[  \a  cf  \f  Vg   z~        r(   c                   |     e Zd ZdZded         ddf fdZedefd            Ze	ded         fd            Z
 xZS )	HTMLResponsezAn HTML-enabled :class:`requests.Response <requests.Response>` object.
    Effectively the same, but with an intelligent ``.html`` property added.
    r   r   r2   Nc                 r    t          t          |                                            d | _        || _        d S r#   )r   r*  r'   r;   r   )r%   r   r   s     r&   r'   zHTMLResponse.__init__  s1    lD!!**,,,
r(   c                 v    | j         s,t          | j        | j        | j        | j                  | _         | j         S )N)r   r1   r0   r/   )r;   r   r   r1   r   rA   rC   s    r&   r0   zHTMLResponse.html  s7    z 	udlt|fjfstttDJzr(   c                 \     | |          }|j                             |j                    |S )N)r   )r   r!  )clsr   r   html_rs       r&   _from_responsezHTMLResponse._from_response  s2    W%%%x0111r(   )r)   r*   r+   r   r
   r'   r   r   r0   classmethodr0  r   r   s   @r&   r*  r*    s         &G H T      
 d    X u5V/W    [    r(   r*  r2   c                 \    t           s| rt                      a | rt           |          nt          S )zReturns an apparently legit user-agent, if not requested one of a specific
    style. Defaults to a Chrome-style User-Agent.
    )	useragentr   DEFAULT_USER_AGENT)styles    r&   
user_agentr6    s2    
   5  KK	$<9U*<<r(   Fc                 >    |r	 | d         S # t           $ r Y d S w xY w| S )Nr   )r   )lr`   s     r&   rs   rs     sA     	Q4K 	 	 	44	 s    
c                   \     e Zd ZdZdddgfdededef fdZdefd	Ze	d
             Z
 xZS )BaseSessionzd A consumable session, for cookie persistence and connection pooling,
    amongst other things.
    Tz--no-sandboxmock_browserverifybrowser_argsc                     t                                                       |rt                      | j        d<   | j        d                             | j                   || _        || _        d S )Nz
User-Agentr   )	r   r'   r6  headershooksrn   response_hookr<  _BaseSession__browser_args)r%   r;  r<  r=  r   s       r&   r'   zBaseSession.__init__  sf      	6)3DL&
:%%d&8999*r(   r2   c                 ^    |j         st          |_         t                              ||           S )z= Change response enconding and replace it by a HTMLResponse. )rA   r:   r*  r0  )r%   r   kwargss      r&   rA  zBaseSession.response_hook  s+      	1 0H**8T:::r(   c                    K   t          | d          s-t          j        | j         d| j                   d {V | _        | j        S )N_browserT)ignoreHTTPSErrorsheadlessargs)hasattr	pyppeteerlaunchr<  rB  rF  rC   s    r&   r  zBaseSession.browser  sV      tZ(( 	@"+"2EU`dkok~"""DM}r(   )r)   r*   r+   r   r   listr'   r*  rA  r   r  r   r   s   @r&   r:  r:    s          .24(6'7+ +d +D + $+ + + + + +;< ; ; ; ;   X    r(   r:  c                   B     e Zd Z fdZe fd            Z fdZ xZS )r   c                 H     t          t          |           j        di | d S )Nr,   )r   r   r'   )r%   rD  r   s     r&   r'   zHTMLSession.__init__  s,    )k4  )33F33333r(   c                    t          | d          spt          j                    | _        | j                                        rt          d          | j                            t                      j                  | _	        | j	        S )NrF  zSCannot use HTMLSession within an existing event loop. Use AsyncHTMLSession instead.)
rJ  r  get_event_loopr  
is_runningRuntimeErrorr  r   r  rF  r%   r   s    r&   r  zHTMLSession.browser  sq    tZ(( 	J.00DIy##%% z"#xyyy I88IIDM}r(   c                     t          | d          r1| j                            | j                                                   t                                                       dS z* If a browser was created close it first. rF  N)rJ  r  r  rF  r  r   rT  s    r&   r  zHTMLSession.close  sO    4$$ 	@I(()<)<)>)>???r(   )r)   r*   r+   r'   r   r  r  r   r   s   @r&   r   r     sv        4 4 4 4 4     X        r(   r   c                   H     e Zd ZdZ	 	 d	def fdZ fdZ fdZd Z xZ	S )
r   z An async consumable session. NTr;  c                      t                      j        |i | |pt          j                    | _        t          |          | _        dS )a(   Set or create an event loop and a thread pool.

            :param loop: Asyncio loop to use.
            :param workers: Amount of threads to use for executing async calls.
                If not pass it will default to the number of processors on the
                machine, multiplied by 5. )max_workersN)r   r'   r  rQ  r  r   thread_pool)r%   r  workersr;  rI  rD  r   s         r&   r'   zAsyncHTMLSession.__init__  sO     	$)&)))4G244	-'BBBr(   c                     t          t                      j        g|R i |}| j                            | j        |          S )z7 Partial original request func and run it in a thread. )r   r   requestr  run_in_executorrZ  )r%   rI  rD  funcr   s       r&   r]  zAsyncHTMLSession.request  sA    uww888888y(()94@@@r(   c                    K   t          | d          r| j                                         d{V  t                                                       dS rV  )rJ  rF  r  r   rT  s    r&   r  zAsyncHTMLSession.close  sT      4$$ 	(-%%'''''''''r(   c                     d |D             }| j                             t          j        |                    \  }}d |D             S )z Pass in all the coroutines you want to run, it will wrap each one
            in a task, run it and wait for the result. Return a list with all
            results, this is returned in the same order coros are passed in. c                 F    g | ]}t          j         |                      S r,   )r  ensure_future)rf   coros     r&   rh   z(AsyncHTMLSession.run.<locals>.<listcomp>  s6     
 
 
.2G!$$&&))
 
 
r(   c                 6    g | ]}|                                 S r,   )r   )rf   ts     r&   rh   z(AsyncHTMLSession.run.<locals>.<listcomp>  s     )))q

)))r(   )r  r  r  r   )r%   corostasksdoner  s        r&   runzAsyncHTMLSession.run  sY    
 
6;
 
 
 )..w|E/B/BCCa))D))))r(   )NNT)
r)   r*   r+   r   r   r'   r]  r  rj  r   r   s   @r&   r   r     s        ((*.&*C C#C C C C C CA A A A A
    * * * * * * *r(   r   r#   )F)Xsysr  urllib.parser   r   r   concurrent.futuresr   concurrent.futures._baser   	functoolsr   typingr	   r
   r   r   r   rK  requestspyqueryr   fake_useragentr   lxml.html.cleanr   rR   r   	lxml.htmlr   r   rp   lxml.html.soupparserr   rV   parser   r   r   r   w3lib.encodingr   r:   r  r4  r   rq   
javascriptr5  r3  r   r8   r   r   r   r   r   
_UserAgentr   r   r   r   _LXMLr   _Searchr   r   r   r'  r&  version_infomajorminorAssertionErrorrS  	Exceptionr!   r.   r   r   Responser*  r6  rs   Sessionr:  r   r   r,   r(   r&   <module>r     so   



  6 6 6 6 6 6 6 6 6 6 1 1 1 1 1 1 1 1 1 1 1 1       = = = = = = = = = = = = = =            $ $ $ $ $ $ # # # # # #        ! ! ! ! ! ! 4 4 4 4 4 4 9 9 9 9 9 9 ( ( ( ( ( ( ! ! ! ! ! ! ! ! * * * * * * $ M /// 
')) 	 	d9oy()	tCy$y/3	9	:
X(
)c5j	
 
	
CcN#	S	fd3i 3i>!Q&&&&!A%%%%% > > >
,<
=
==>       e e e e e e e eP	% % % % %j % % %Pi i i i i: i i iX    8$   0= =j = = = =       ("   B    +   *#* #* #* #* #*{ #* #* #* #* #*s   ($E E