ÿØÿà JFIF      ÿÛ „ 	 ( %!1!%)+//.383,7(-.+



-%%-////---/-.+/--+------/------/--0+--/-/-----.-----ÿÀ  ¥2" ÿÄ               ÿÄ J  	     ! 1AQ"aq2‘#BR‚¡ÁÑ3br’¢±Âð$CSƒ²á4c“%DsÓñÿÄ              ÿÄ *        !1AQa‘"2q3±ð#b¡ÿÚ   ? ¼QxJQaÍuò¸Zö Úü8,ÐÚú
"SSn<rçù–´âE—^ªBÖ9À\†¸ÔÁT­ÃÛ5
ëd´³Í#Ý;Þ38œî ¶H£M:wÎ3…³…âpÔF&‚FK¸9„â4àGEõªfÿ ‘ñ(ßw­pŽF|È¥ù®häðÍÑ¶¹‘[ÒinÙW¶ùñY˜Q{›K"išÒ[Ú8žë\F¹@-?v"ÔU”,ìöžkÿ {I‡£šÍ?e
ríV
..............................................................................................................................................................................
.............................................................................                                                  
                                                                                                                                                                                     ÿØÿà JFIF      ÿÛ „ 	 ( %!1!%)+//.383,7(-.+



-%%-////---/-.+/--+------/------/--0+--/-/-----.-----ÿÀ  ¥2" ÿÄ               ÿÄ J  	     ! 1AQ"aq2‘#BR‚¡ÁÑ3br’¢±Âð$CSƒ²á4c“%DsÓñÿÄ              ÿÄ *        !1AQa‘"2q3±ð#b¡ÿÚ   ? ¼QxJQaÍuò¸Zö Úü8,ÐÚú
"SSn<rçù–´âE—^ªBÖ9À\†¸ÔÁT­ÃÛ5
ëd´³Í#Ý;Þ38œî ¶H£M:wÎ3…³…âpÔF&‚FK¸9„â4àGEõªfÿ ‘ñ(ßw­pŽF|È¥ù®häðÍÑ¶¹‘[ÒinÙW¶ùñY˜Q{›K"išÒ[Ú8žë\F¹@-?v"ÔU”,ìöžkÿ {I‡£šÍ?e
ríV
..............................................................................................................................................................................
.............................................................................                                                  
                                                                                                                                                                                     U
    e5d$                     @   s\   d Z ddlZddlZddlZdgZeddZG dd dZG dd dZ	G d	d
 d
Z
dS )a%   robotparser.py

    Copyright (C) 2000  Bastian Kleineidam

    You can choose between two licenses when using this package:
    1) GNU GPLv2
    2) PSF license for Python 2.2

    The robots.txt Exclusion Protocol is implemented as specified in
    http://www.robotstxt.org/norobots-rfc.txt
    NRobotFileParserRequestRatezrequests secondsc                   @   sr   e Zd ZdZdddZdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd ZdS )r   zs This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

     c                 C   s2   g | _ g | _d | _d| _d| _| | d| _d S )NFr   )entriessitemapsdefault_entrydisallow_all	allow_allset_urllast_checkedselfurl r   */usr/lib64/python3.8/urllib/robotparser.py__init__   s    
zRobotFileParser.__init__c                 C   s   | j S )zReturns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        )r   r   r   r   r   mtime%   s    zRobotFileParser.mtimec                 C   s   ddl }|  | _dS )zYSets the time the robots.txt file was last fetched to the
        current time.

        r   N)timer   )r   r   r   r   r   modified.   s    zRobotFileParser.modifiedc                 C   s&   || _ tj|dd \| _| _dS )z,Sets the URL referring to a robots.txt file.      N)r   urllibparseurlparseZhostpathr   r   r   r   r
   6   s    zRobotFileParser.set_urlc              
   C   s   zt j| j}W nR t jjk
rd } z0|jdkr:d| _n|jdkrT|jdk rTd| _W 5 d}~X Y nX |	 }| 
|d  dS )z4Reads the robots.txt URL and feeds it to the parser.)i  i  Ti  i  Nzutf-8)r   ZrequestZurlopenr   errorZ	HTTPErrorcoder   r	   readr   decode
splitlines)r   ferrrawr   r   r   r   ;   s    
zRobotFileParser.readc                 C   s,   d|j kr| jd kr(|| _n| j| d S N*)
useragentsr   r   append)r   entryr   r   r   
_add_entryH   s    

zRobotFileParser._add_entryc                 C   sP  d}t  }|   |D ]}|sP|dkr4t  }d}n|dkrP| | t  }d}|d}|dkrn|d| }| }|s|q|dd}t|dkr|d   |d< tj	
|d  |d< |d dkr|dkr| | t  }|j|d  d}q|d dkr.|dkr6|jt|d d	 d}q|d d
krb|dkr6|jt|d d d}q|d dkr|dkr6|d   rt|d |_d}q|d dkr|dkr6|d d}t|dkr|d   r|d   rtt|d t|d |_d}q|d dkr| j|d  q|dkrL| | dS )zParse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        r   r      #N:z
user-agentZdisallowFZallowTzcrawl-delayzrequest-rate/Zsitemap)Entryr   r)   findstripsplitlenlowerr   r   unquoter&   r'   	rulelinesRuleLineisdigitintdelayr   req_rater   )r   linesstater(   lineiZnumbersr   r   r   r   Q   sj    








 
zRobotFileParser.parsec                 C   s   | j r
dS | jrdS | jsdS tjtj|}tjdd|j|j	|j
|jf}tj|}|sfd}| jD ]}||rl||  S ql| jr| j|S dS )z=using the parsed robots.txt decide if useragent can fetch urlFTr   r-   )r   r	   r   r   r   r   r4   
urlunparser   ZparamsZqueryZfragmentquoter   
applies_to	allowancer   )r   	useragentr   Z
parsed_urlr(   r   r   r   	can_fetch   s*      

zRobotFileParser.can_fetchc                 C   s>   |   sd S | jD ]}||r|j  S q| jr:| jjS d S N)r   r   rA   r9   r   r   rC   r(   r   r   r   crawl_delay   s    

zRobotFileParser.crawl_delayc                 C   s>   |   sd S | jD ]}||r|j  S q| jr:| jjS d S rE   )r   r   rA   r:   r   rF   r   r   r   request_rate   s    

zRobotFileParser.request_ratec                 C   s   | j s
d S | j S rE   )r   r   r   r   r   	site_maps   s    zRobotFileParser.site_mapsc                 C   s,   | j }| jd k	r|| jg }dtt|S )Nz

)r   r   joinmapstr)r   r   r   r   r   __str__   s    
zRobotFileParser.__str__N)r   )__name__
__module____qualname____doc__r   r   r   r
   r   r)   r   rD   rG   rH   rI   rM   r   r   r   r   r      s   
			I

c                   @   s(   e Zd ZdZdd Zdd Zdd ZdS )	r6   zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
       (allowance==False) followed by a path.c                 C   s<   |dkr|sd}t jt j|}t j|| _|| _d S )Nr   T)r   r   r?   r   r@   r   rB   )r   r   rB   r   r   r   r      s
    zRuleLine.__init__c                 C   s   | j dkp|| j S r$   )r   
startswith)r   filenamer   r   r   rA      s    zRuleLine.applies_toc                 C   s   | j r
dndd | j S )NZAllowZDisallowz: )rB   r   r   r   r   r   rM      s    zRuleLine.__str__N)rN   rO   rP   rQ   r   rA   rM   r   r   r   r   r6      s   r6   c                   @   s0   e Zd ZdZdd Zdd Zdd Zdd	 Zd
S )r.   z?An entry has one or more user-agents and zero or more rulelinesc                 C   s   g | _ g | _d | _d | _d S rE   )r&   r5   r9   r:   r   r   r   r   r      s    zEntry.__init__c                 C   s   g }| j D ]}|d|  q
| jd k	r<|d| j  | jd k	rf| j}|d|j d|j  |tt| j	 d
|S )NzUser-agent: zCrawl-delay: zRequest-rate: r-   
)r&   r'   r9   r:   ZrequestsZsecondsextendrK   rL   r5   rJ   )r   ZretagentZrater   r   r   rM      s    


zEntry.__str__c                 C   sF   | dd  }| jD ](}|dkr* dS | }||kr dS qdS )z2check if this entry applies to the specified agentr-   r   r%   TF)r1   r3   r&   )r   rC   rV   r   r   r   rA      s    
zEntry.applies_toc                 C   s$   | j D ]}||r|j  S qdS )zZPreconditions:
        - our agent applies to this entry
        - filename is URL decodedT)r5   rA   rB   )r   rS   r=   r   r   r   rB   
  s    

zEntry.allowanceN)rN   rO   rP   rQ   r   rM   rA   rB   r   r   r   r   r.      s
   r.   )rQ   collectionsZurllib.parser   Zurllib.request__all__
namedtupler   r   r6   r.   r   r   r   r   <module>   s    B