daf,vddlZddlZddlZddlZddlZddlmZddlm Z ddlm Z ddl m Z m Z GddZGdd eejZGd d eejZGd d eejZGddeejZGddeZGddeejZGddeejZGddeZGddeejZGddeejZGddeejZGddeejZGd d!eZGd"d#eejZGd$d%eejZGd&d'eejZGd(d)eejZ Gd*d+eejZ!Gd,d-eejZ"Gd.d/e Z#ejHejJd0Gd1d2ejZ&ejNGd3d4ejZ(e)d5k(rejTyy)6N)support) socket_helper)threading_helper)BaseHTTPRequestHandler HTTPServerc>eZdZdZdZgZgZdZdZdZ dZ dZ dZ y) BaseRobotTesttest_robotparserNctj|jj}tj j |_|jj|yN) ioStringIO robots_txt readlinesurllib robotparserRobotFileParserparserparse)selfliness 0/root/Python-3.12.4/Lib/test/test_robotparser.pysetUpzBaseRobotTest.setUpsE DOO,668((88:  % cPt|tr |\}}||fS|j|fSr ) isinstancetupleagentrurlrs rget_agent_and_urlzBaseRobotTest.get_agent_and_urls- c5 !JE3#: zz3rc|jD]]}|j|\}}|j||5|j|jj ||ddd_y#1swYjxYwN)r!r)goodr"subTest assertTruer can_fetchr s rtest_good_urlszBaseRobotTest.test_good_urlssw99 CC//4JE3#U3 C 5 5eS AB C C C C C ,A..A7 c|jD]]}|j|\}}|j||5|j|jj ||ddd_y#1swYjxYwr$)badr"r& assertFalserr(r s r test_bad_urlszBaseRobotTest.test_bad_urls$sy88 DC//4JE3#U3 D  !6!6uc!BC D D D D Dr*cl|j|jj|jyr ) assertEqualr site_mapsrs rtest_site_mapszBaseRobotTest.test_site_maps*s# ..0$..Ar) __name__ __module__ __qualname__rrr%r,r1rr"r)r.r3rrr r s7J E D CI!  C D Brr c eZdZdZddgZgdZy)UserAgentWildcardTestzUser-agent: * Disallow: /cyberworld/map/ # This is an infinite virtual URL space Disallow: /tmp/ # these will soon disappear Disallow: /foo.html / /test.html)/cyberworld/map/index.htmlz/tmp/xxx /foo.htmlNr4r5r6rr%r,r7rrr9r9.sJ  D ACrr9ceZdZdZgdZdgZy)CrawlDelayAndCustomAgentTestz# robots.txt for http://www.example.com/ User-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: )r:r;) cybermapperr<r<Nr>r7rrr@r@9s J ND ' (Crr@c&eZdZdZddgZdgZddgZy) SitemapTesta# robots.txt for http://www.example.com/ User-agent: * Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml Sitemap: http://www.google.com/hostednews/sitemap_index.xml Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space r:r;r<z7http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xmlz2http://www.google.com/hostednews/sitemap_index.xmlN)r4r5r6rr%r,r1r7rrrCrCJs+ J  D ' (CJEGIrrCceZdZdZgZgdZy)RejectAllRobotsTestz(# go away User-agent: * Disallow: / )r<r:/tmp/Nr>r7rrrErE[sJ D 6CrrEceZdZdZdZdZy)BaseRequestRateTestNc|j}|j|jzD]!}|j|\}}|j ||5|j |j ||j |j|}|j ||j|j|j|tjj|j |j|jj|j |j|jjddd$y#1swY0xYwr$)rr%r,r"r&r0 crawl_delay request_rateassertIsInstancerr RequestRaterequestsseconds)rrr!rparsed_request_rates rtest_request_ratez%BaseRequestRateTest.test_request_rateis*99txx' C//4JE3#U3   !3!3E!:Dr7rrrdrdsJ =D Crrdc eZdZdZdZdgZdgZy)AnotherInvalidRequestRateTestzeUser-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ Request-rate: whale/banana Googlebot/folder1/myfile.html/folder1/anotherfile.htmlNr4r5r6rrr%r,r7rrrfrfs J E " #D & 'CrrfceZdZdZdZdgZy)UserAgentOrderingTestzMUser-agent: Googlebot Disallow: / User-agent: Googlebot-Mobile Allow: / rgz/something.jpgN)r4r5r6rrr,r7rrrlrlsJ E  CrrlceZdZdZy)UserAgentGoogleMobileTestzGooglebot-MobileNr`r7rrrnrns Errnc eZdZdZdZdgZdgZy)GoogleURLOrderingTestzJUser-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ googlebotrhriNrjr7rrrprps J E " #D & 'CrrpceZdZdZdgZdgZy)DisallowQueryStringTestz2User-agent: * Disallow: /some/path?name=value /some/pathz/some/path?name=valueNr>r7rrrsrssJ >D " #CrrsceZdZdZdgZdgZy)UseFirstUserAgentWildcardTestzNUser-agent: * Disallow: /some/path User-agent: * Disallow: /another/path z /another/pathrtNr>r7rrrvrvsJ  D .CrrvceZdZdZdgZdgZy)EmptyQueryStringTestz>User-agent: * Allow: /some/path? Disallow: /another/path? z /some/path?z/another/path?Nr>r7rrrxrxsJ ?D  CrrxcZeZdZdZej j ddZdZddgZ dgZ y) DefaultEntryTestzOUser-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ rYr:r;r<N) r4r5r6rrrrMrKrJr%r,r7rrrzrzs:J %%11!R8LK  D ' (CrrzceZdZdZdZdZy)StringFormattingTestzUser-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: /some/path zxUser-agent: cybermapper Disallow: /some/path User-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/cb|jt|j|jyr )r0strrexpected_outputr2s rtest_string_formattingz+StringFormattingTest.test_string_formatting*s! T[[)4+?+?@rN)r4r5r6rrrr7rrr~r~s JOArr~ceZdZdZdZy) RobotHandlerc(|jddy)NizForbidden access) send_errorr2s rdo_GETzRobotHandler.do_GET0s /0rcyr r7)rformatargss r log_messagezRobotHandler.log_message3s rN)r4r5r6rrr7rrrr.s 1 rrz&Socket server requires working socket.c<eZdZdZdZej dZy)PasswordProtectedSiteTestCasec\|jtjjt t j dft|_tjd|jjddi|_ d|j_ |jjy)NrzHTTPServer serving poll_intervalg{Gz?)nametargetkwargsT) addCleanuprrequest urlcleanuprrHOSTrserver threadingThread serve_forevertdaemonstartr2s rrz#PasswordProtectedSiteTestCase.setUp=sw 112 -"4"4a!8,G !!%;;,,$D) +   rc|jj|jj|jj yr )rshutdownrjoin server_closer2s rtearDownz&PasswordProtectedSiteTestCase.tearDownMs/      "rcD|jj}dtjzdzt |dz}|dz}t j j}|j||j|j|jd|y)Nzhttp://:r|z /robots.txt*) rserver_addressrrrrrrset_urlreadr-r()raddrr! robots_urlrs rtestPasswordProtectedSitez7PasswordProtectedSiteTestCase.testPasswordProtectedSiteRs{{))-,,,s2Sa\A=( ##335s  ))#z:;rN)r4r5r6rrr reap_threadsrr7rrrr7s&  # ""<#rs+ &):BBDBM8+<+<B)=(2C2C)"G-!2!2G"7-):):7-8'):): .#68I8I.$-5- ]H,=,=   M8+<+<   (M83D3D ( M8+<+<  5 (M8+<+< ($mX->->$ M83D3D =(*;*; )*H,=,= )A=(*;*;A4 )  ,