Operation   : Linux premium131.web-hosting.com 4.18.0-553.44.1.lve.el8.x86_64 #1 SMP Thu Mar 13 14:29:12 UTC 2025 x86_64
Software     : Apache
Server IP    : 162.0.232.56 | Your IP: 216.73.216.111
Domains      : 1034 Domain(s)
Permission   : [ 0755 ]

�
��f�c@sddZd
dl
Zd
dlZdg
ZGdd�d�ZGdd�d�ZGdd�d�ZdS(	u<
 robotparser.py

    Copyright (C) 2000  Bastian Kleineidam

    You can choose between two licenses when using this package:
    1) GNU GPLv2
    2) PSF license for Python 2.2

    The robots.txt Exclusion Protocol is implemented as specified in
    http://info.webcrawler.com/mak/projects/robots/norobots-rfc.html
iNuRobotFileParserc

Bs�|EeZ
dZd
Zddd�
Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dd�Zdd�ZdS(uRobotFileParserus This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

    ucCs>g|_d|_d|_d|_|j|
�

d
|_dS(NiF(uentriesuNoneu
default_entryuFalseudisallow_allu	allow_alluset_urlulast_checked(uselfuurl((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu__init__s
	
	
	
	


uRobotFileParser.__init__c


Cs|jS(
u�Returns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        (
ulast_checked(
uself((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyumtimesuRobotFileParser.mtimec
Csd
dl}
|
j�|_
dS(uYSets the time the robots.txt file was last fetched to the
        current time.

        iN(utimeulast_checked(uselfutime((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyumodified(s
uRobotFileParser.modifiedcCs5|
|_t
jj|
�
d
d�\|_|_dS(u,Sets the URL referring to a robots.txt file.i
iN(uurluurllibuparseuurlparseuhostupath(uselfuurl((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyuset_url0s	
uRobotFileParser.set_urlc
Cs�ytj
j|j�
}
Wnatjjk
r|
}
z;|jdkrOd|_n|jdkrjd|_	nWYdd}~Xn)X|
j
�}|j|jd�
j
��

dS(u4Reads the robots.txt URL and feeds it to the parser.i�
i�
i�
Nuutf-8(i�
i�
T(uurlliburequestuurlopenuurluerroru	HTTPErrorucodeuTrueudisallow_allu	allow_allureaduparseudecodeu
splitlines(uselfu
fuerruraw((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyuread5s






uRobotFileParser.readcCsAd
|
jkr-|j
dkr=|
|_
q=n|jj|
�

dS(Nu
*(u
useragentsu
default_entryuNoneuentriesuappend(uselfuentry((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu
_add_entryBs
uRobotFileParser._add_entrycCsd
}t�}x�
|
D]�
}|sn|dkr@t�}d
}qn|dkrn|j
|�

t�}d
}qnn|jd�
}|d
kr�|d|�}n|j�}|s�qn|jdd�}t|�
dkr|d
j�j�|d
<tjj	|dj��
|d<|d
dkrd
|dkrG
|j
|�

t�}n|j
j|d�

d}q�
|d
dkr�
|d
kr�
|jjt
|dd
��

d}q�
q�
|d
d	kr�
|d
kr�
|jjt
|dd��

d}q�
q�
qqW|dkr|j
|�

ndS(u�Parse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        ii
iu
#Nu
:u
user-agentudisallowuallowFT(uEntryu
_add_entryufindustripusplitulenuloweruurllibuparseuunquoteu
useragentsuappendu	rulelinesuRuleLineuFalseuTrue(uselfulinesustateuentryulineu
i((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyuparseKsJ

	



	
	



	









 






	









uRobotFileParser.parsecCs�|jr
dS|jrdStjjtjj|�
�
}tjjd
d
|j	|j
|j|jf�
}tjj
|�
}|s�d}nx-|jD]"}|j|
�
r�|j|�
Sq�W|jr�|jj|�
SdS(u=using the parsed robots.txt decide if useragent can fetch urluu
/FT(udisallow_alluFalseu	allow_alluTrueuurllibuparseuurlparseuunquoteu
urlunparseupathuparamsuqueryufragmentuquoteuentriesu
applies_tou	allowanceu
default_entry(uselfu	useragentuurlu
parsed_urluentry((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu	can_fetch~s 	

	





	


	
uRobotFileParser.can_fetchc

Csd
jdd�|j
D�
�
S(Nuc
Ss g|]}
t|
�
d�qS(
u

(
ustr(u.0uentry((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu
<listcomp>�s	u+RobotFileParser.__str__.<locals>.<listcomp>(ujoinuentries(
uself((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu__str__�s
uRobotFileParser.__str__N(
u__name__u
__module__u__qualname__u__doc__u__init__umtimeumodifieduset_urlureadu
_add_entryuparseu	can_fetchu__str__(
u
__locals__((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyuRobotFileParsers	
	3c

Bs>|EeZ
dZd
Zdd�Zdd�Zdd�ZdS(	uRuleLineuoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
       (allowance==False) followed by a path.cCs\|
d
kr|rd}nt
jjt
jj|
�
�
}
t
jj|
�
|_||_dS(NuT(uTrueuurllibuparseu
urlunparseuurlparseuquoteupathu	allowance(uselfupathu	allowance((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu__init__�s

	


uRuleLine.__init__cCs|jd
kp|
j
|j�
S(Nu
*(upathu
startswith(uselfufilename((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu
applies_to�s
uRuleLine.applies_toc

Cs|jrd
pdd|j
S(NuAllowuDisallowu: (u	allowanceupath(
uself((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu__str__�s
uRuleLine.__str__N(u__name__u
__module__u__qualname__u__doc__u__init__u
applies_tou__str__(
u
__locals__((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyuRuleLine�s
uRuleLinec

BsJ|EeZ
dZd
Zdd�Zdd�Zdd�Zdd	�Zd
S(uEntryu?An entry has one or more user-agents and zero or more rulelinesc

Csg|_g|_
dS(
N(u
useragentsu	rulelines(
uself((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu__init__�s
	
uEntry.__init__c
Csjg}
x'|jD]}|
j
d
|dg�

qWx*|jD]}|
j
t|�
dg�

q:Wdj|
�
S(NuUser-agent: u

u(u
useragentsuextendu	rulelinesustrujoin(uselfuretuagentuline((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu__str__�s





u
Entry.__str__cCs]|
jd
�
dj
�}
x=|jD]2}|dkr9dS|j
�}||
kr#dSq#WdS(u2check if this entry applies to the specified agentu
/iu
*TF(usplituloweru
useragentsuTrueuFalse(uselfu	useragentuagent((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu
applies_to�s





uEntry.applies_tocCs.x'|jD]}|j
|
�
r
|jSq
Wd
S(uZPreconditions:
        - our agent applies to this entry
        - filename is URL decodedT(u	rulelinesu
applies_tou	allowanceuTrue(uselfufilenameuline((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu	allowance�s


uEntry.allowanceN(u__name__u
__module__u__qualname__u__doc__u__init__u__str__u
applies_tou	allowance(
u
__locals__((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyuEntry�s



uEntry(u__doc__uurllib.parseuurllibuurllib.requestu__all__uRobotFileParseruRuleLineuEntry(((u7/opt/alt/python33/lib64/python3.3/urllib/robotparser.pyu<module>s
	�