Operation   : Linux premium131.web-hosting.com 4.18.0-553.44.1.lve.el8.x86_64 #1 SMP Thu Mar 13 14:29:12 UTC 2025 x86_64
Software     : Apache
Server IP    : 162.0.232.56 | Your IP: 216.73.216.111
Domains      : 1034 Domain(s)
Permission   : [ 0755 ]

o

�=?h�$�@s\dZd
dl
Z
d
dlZd
dlZdg
Ze
�dd�ZGdd�d�ZGdd�d�Z	Gd	d
�d
�Z
dS)a%
 robotparser.py

    Copyright (C) 2000  Bastian Kleineidam

    You can choose between two licenses when using this package:
    1) GNU GPLv2
    2) PSF license for Python 2.2

    The robots.txt Exclusion Protocol is implemented as specified in
    http://www.robotstxt.org/norobots-rfc.txt
�N�RobotFileParser�RequestRatezrequests secondsc@sreZ
dZd
Zddd�
Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dd�Zdd�Zdd�Z
dd�Zdd�ZdS)rzs This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

    �cCs2g|_g|_
d|_d
|_d
|_|�|
�

d|_dS)NFr
)�entries�sitemaps�
default_entry�disallow_all�	allow_all�set_url�last_checked��self�url�r�9/opt/alt/python310/lib64/python3.10/urllib/robotparser.py�__init__s








zRobotFileParser.__init__c


Cs|jS)
z�Returns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        )
r�
r
rrr�mtime%szRobotFileParser.mtimec
Csd
dl}
|
��|_
dS)zYSets the time the robots.txt file was last fetched to the
        current time.

        r
N)�timer)r
rrrr�modified.s
zRobotFileParser.modifiedcCs&|
|_t
j�|
�
d
d�\|_|_dS)z,Sets the URL referring to a robots.txt file.�
�N)r�urllib�parse�urlparseZhost�pathrrrrr
6s 
zRobotFileParser.set_urlc

Cs�z	tj
�|j�
}
Wn@tjjyI
}
z2|jd
vrd|_n|jdkr0|jdkr>d|_WYd}~dSWYd}~dSWYd}~dSWYd}~dSd}~w
w|
�	�}|�
|�d�
���

dS)z4Reads the robots.txt URL and feeds it to the parser.)i�
i�
Ti�
i�
Nzutf-8)
rZrequestZurlopenr�errorZ	HTTPError�coderr	�readr�decode�
splitlines)r
�
f�err�rawrrrr;s






��
��
zRobotFileParser.readcCs2d
|
jvr|j
dur|
|_
dSdS|j�|
�

dS�N�
*)�
useragentsrr�append)r
�entryrrr�
_add_entryHs




�zRobotFileParser._add_entrycCsJd
}t�}|�
�
|
D�
]
}|s(|dkrt�}d
}n|dkr(|�|�

t�}d
}|�d�
}|d
kr7|d|�}|��}|s>q|�dd�}t|�
dk�
r|d
����|d
<tj	�
|d���
|d<|d
dkr~|dkrs|�|�

t�}|j�|d�

d}q|d
dkr�|d
kr�|j
�t|dd	��

d}q|d
d
kr�|d
kr�|j
�t|dd��

d}q|d
dkr�|d
kr�|d����r�t|d�
|_d}q|d
d
k�
r|d
k�
r|d�d�
}t|�
dk�
r|d
�����
r|d�����
rtt|d
�
t|d�
�|_d}q|d
dk�
r|j�|d�

q|dk�
r#|�|�

dSdS)z�Parse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        r
r��
#N�
:z
user-agentZdisallowFZallowTzcrawl-delayzrequest-rate�
/Zsitemap)�Entryrr)�find�strip�split�len�lowerrr�unquoter&r'�	rulelines�RuleLine�isdigit�int�delayr�req_rater)r
�lines�stater(�line�
iZnumbersrrrrQsv

































�



�



�



 
�
�
�


�zRobotFileParser.parsecCs�|jrd
S|j
r
dS|jsd
Stj�tj�|�
�
}tj�dd|j|j	|j
|jf�
}tj�|�
}|s3d}|j
D]}|�|
�
rD|�|�

Sq6|jrN|j�|�
SdS)z=using the parsed robots.txt decide if useragent can fetch urlFTrr-)rr	rrrrr4�
urlunparser�paramsZqueryZfragment�quoter�
applies_to�	allowancer)r
�	useragentrZ
parsed_urlr(rrr�	can_fetch�s(





�






�
zRobotFileParser.can_fetchcC�>|��sdS|j
D]}|�|
�
r|j
Sq	|jr|jjSdS�
N)rrrBr9r�r
rDr(rrr�crawl_delay��







�

zRobotFileParser.crawl_delaycCrFrG)rrrBr:rrHrrr�request_rate�rJzRobotFileParser.request_ratec


Cs|jsdS|jSrG)
rrrrr�	site_maps�s


zRobotFileParser.site_mapsc
Cs,|j}
|j
du
r|
|j
g
}
d
�tt|
��
S)Nz

)rr�join�map�str)r
rrrr�__str__�s




zRobotFileParser.__str__N)
r)�__name__�
__module__�__qualname__�__doc__rrrr
rr)rrErIrKrLrPrrrrrs

		
	I

c@s(eZ
dZd
Zdd�Zdd�Zdd�ZdS)	r6zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
       (allowance==False) followed by a path.cCs<|
d
kr|sd}tj
�tj
�|
�
�
}
tj
�|
�
|_||_dS)NrT)rrr?rrArrC)r
rrCrrrr�s





zRuleLine.__init__cCs|jd
kp
|
�
|j�
Sr$)r�
startswith)r
�filenamerrrrB�s
zRuleLine.applies_toc

Cs|jrd
n
dd|j
S)NZAllowZDisallowz: )rCrrrrrrP�s
zRuleLine.__str__N)rQrRrSrTrrBrPrrrrr6�s

r6c@s0eZ
dZd
Zdd�Zdd�Zdd�Zdd	�Zd
S)r.z?An entry has one or more user-agents and zero or more rulelinesc

Csg|_g|_
d|_d|_dSrG)r&r5r9r:rrrrr�s




zEntry.__init__c
Cs�g}
|jD]
}|
�
d
|���

q|jdu
r|
�
d|j���

|jdu
r3|j}|
�
d|j�d|j���

|
�tt|j	��

d�
|
�
S)NzUser-agent: z
Crawl-delay: zRequest-rate: r-�

)r&r'r9r:ZrequestsZseconds�extendrNrOr5rM)r
Zret�agentZraterrrrP�s













z
Entry.__str__cCsF|
�d
�
d�
�}
|jD]}|dkr
dS|�
�}||
vr 
dSqdS)z2check if this entry applies to the specified agentr-r
r%TF)r1r3r&)r
rDrYrrrrB�s





�zEntry.applies_tocCs$|jD]}|�
|
�
r|j
Sqd
S)zZPreconditions:
        - our agent applies to this entry
        - filename is URL decodedT)r5rBrC)r
rVr=rrrrC

s





�zEntry.allowanceN)rQrRrSrTrrPrBrCrrrrr.�s


r.)rT�collectionsZurllib.parserZurllib.request�__all__�
namedtuplerrr6r.rrrr�<module>
s

B