Operation   : Linux premium131.web-hosting.com 4.18.0-553.44.1.lve.el8.x86_64 #1 SMP Thu Mar 13 14:29:12 UTC 2025 x86_64
Software     : Apache
Server IP    : 162.0.232.56 | Your IP: 216.73.216.111
Domains      : 1034 Domain(s)
Permission   : [ 0755 ]



��Yf4�@sddZd
dl
Zd
dlZdg
ZGdd�d�ZGdd�d�ZGdd�d�ZdS)	a%
 robotparser.py

    Copyright (C) 2000  Bastian Kleineidam

    You can choose between two licenses when using this package:
    1) GNU GPLv2
    2) PSF license for Python 2.2

    The robots.txt Exclusion Protocol is implemented as specified in
    http://www.robotstxt.org/norobots-rfc.txt
�N�RobotFileParserc@s�eZ
dZd
Zddd�
Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dd�Zdd�ZdS)rzs This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

    �cCs>g|_d|_
d
|_d
|_|j|
�

d|_dS)NFr
)�entries�
default_entry�disallow_all�	allow_all�set_url�last_checked)�self�url�r�7/opt/alt/python35/lib64/python3.5/urllib/robotparser.py�__init__s
	
	
	
	


zRobotFileParser.__init__c


Cs|jS)
z�Returns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        )
r	)
r
rrr
�mtimeszRobotFileParser.mtimec
Csd
dl}
|
j�|_
dS)zYSets the time the robots.txt file was last fetched to the
        current time.

        r
N)�timer	)r
rrrr
�modified(s
zRobotFileParser.modifiedcCs5|
|_t
jj|
�
d
d�\|_|_dS)z,Sets the URL referring to a robots.txt file.�
�N)r�urllib�parse�urlparse�host�path)r
rrrr
r0s	
zRobotFileParser.set_urlc
Cs�ytj
j|j�
}
Wnmtjjk
r�
}
zG|jdkrOd|_n'|jdkrv|jdkrvd|_WYdd}~Xn)X|
j	�}|j
|jd�
j��

dS)	z4Reads the robots.txt URL and feeds it to the parser.�
�
Ti�
i�
Nzutf-8)rr)
rZrequestZurlopenr�errorZ	HTTPError�coderr�readr�decode�
splitlines)r
�
f�err�rawrrr
r5s






zRobotFileParser.readcCs>d
|
jkr*|j
dkr:|
|_
n|jj|
�

dS)N�
*)�
useragentsrr�append)r
�entryrrr
�
_add_entryBs
zRobotFileParser._add_entrycCs
d
}t�}|j
�
x�
|
D]�
}|sr|dkrJt�}d
}n(|dkrr|j|�

t�}d
}|jd�
}|d
kr�|d|�}|j�}|s�q |jdd�}t|�
dkr |d
j�j�|d
<tj	j
|dj��
|d<|d
dkr_
|dkrB
|j|�

t�}|jj|d�

d}q |d
dkr�
|d
kr�
|j
jt|dd	��

d}q |d
d
kr |d
kr |j
jt|dd��

d}q W|dkr�
|j|�

dS)z�Parse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        r
r��
#N�
:z
user-agentZdisallowFZallowT)�Entryrr'�find�strip�split�len�lowerrr�unquoter$r%�	rulelines�RuleLine)r
�lines�stater&�line�
irrr
rKsL

	





	
	



	









 




	

	



	






zRobotFileParser.parsecCs�|jr
d
S|j
rdS|js'd
Stjjtjj|�
�
}tjjdd|j|j	|j
|jf�
}tjj|�
}|s�d}x-|j
D]"}|j|
�
r�|j|�
Sq�W|jr�|jj|�
SdS)z=using the parsed robots.txt decide if useragent can fetch urlFTr�
/)rrr	rrrr1�
urlunparserZparamsZqueryZfragment�quoter�
applies_to�	allowancer)r
�	useragentrZ
parsed_urlr&rrr
�	can_fetchs$	

	
	








	
zRobotFileParser.can_fetchc

Csd
jdd�|j
D�
�
S)Nrc
Ss g|]}
t|
�
d�qS)
�

)
�str)�.0r&rrr
�
<listcomp>�s	z+RobotFileParser.__str__.<locals>.<listcomp>)�joinr)
r
rrr
�__str__�s
zRobotFileParser.__str__N)
�__name__�
__module__�__qualname__�__doc__rrrrrr'rr>rDrrrr
rs	
	4c@s:eZ
dZd
Zdd�Zdd�Zdd�ZdS)	r3zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
       (allowance==False) followed by a path.cCsY|
d
kr|rd}tj
jtj
j|
�
�
}
tj
j|
�
|_||_dS)NrT)rrr9rr:rr<)r
rr<rrr
r�s




zRuleLine.__init__cCs|jd
kp|
j
|j�
S)Nr#)r�
startswith)r
�filenamerrr
r;�s
zRuleLine.applies_toc

Cs|jrd
ndd|j
S)NZAllowZDisallowz: )r<r)
r
rrr
rD�s
zRuleLine.__str__N)rErFrGrHrr;rDrrrr
r3�s
r3c@sFeZ
dZd
Zdd�Zdd�Zdd�Zdd	�Zd
S)r+z?An entry has one or more user-agents and zero or more rulelinesc

Csg|_g|_
dS)
N)r$r2)
r
rrr
r�s
	
zEntry.__init__c
Csjg}
x'|jD]}|
j
d
|dg�

qWx*|jD]}|
j
t|�
dg�

q:Wdj|
�
S)NzUser-agent: r?r)r$�extendr2r@rC)r
Zret�agentr6rrr
rD�s





z
Entry.__str__cCs]|
jd
�
dj
�}
x=|jD]2}|dkr9dS|j
�}||
kr#dSq#WdS)z2check if this entry applies to the specified agentr8r
r#TF)r.r0r$)r
r=rLrrr
r;�s





zEntry.applies_tocCs.x'|jD]}|j
|
�
r
|jSq
Wd
S)zZPreconditions:
        - our agent applies to this entry
        - filename is URL decodedT)r2r;r<)r
rJr6rrr
r<�s


zEntry.allowanceN)rErFrGrHrrDr;r<rrrr
r+�s



r+)rHZurllib.parserZurllib.request�__all__rr3r+rrrr
�<module>s
	�