← Back to branch summary

~pythonregexp2.7/python/issue2636-01+09-01-01

« back to all changes in this revision

Viewing changes to Lib/test/test_robotparser.py

  • Committer: Jeffrey C. "The TimeHorse" Jacobs
  • Date: 2008-09-22 00:02:12 UTC
  • mfrom: (39022.1.34 Regexp-2.7)
  • Revision ID: darklord@timehorse.com-20080922000212-7r0q4f4ugiq57jph
Merged in changes from the Atomic Grouping / Possessive Qualifiers branch.

Show diffs side-by-side

added added

removed removed

Lines of Context:
Lib/test/test_robotparser.py
134
134
 
135
135
RobotTest(7, doc, good, bad)
136
136
 
 
137
# From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364
 
138
 
 
139
# 8.
 
140
doc = """
 
141
User-agent: Googlebot
 
142
Allow: /folder1/myfile.html
 
143
Disallow: /folder1/
 
144
"""
 
145
 
 
146
good = ['/folder1/myfile.html']
 
147
bad = ['/folder1/anotherfile.html']
 
148
 
 
149
RobotTest(8, doc, good, bad, agent="Googlebot")
 
150
 
 
151
# 9.  This file is incorrect because "Googlebot" is a substring of
 
152
#     "Googlebot-Mobile", so test 10 works just like test 9.
 
153
doc = """
 
154
User-agent: Googlebot
 
155
Disallow: /
 
156
 
 
157
User-agent: Googlebot-Mobile
 
158
Allow: /
 
159
"""
 
160
 
 
161
good = []
 
162
bad = ['/something.jpg']
 
163
 
 
164
RobotTest(9, doc, good, bad, agent="Googlebot")
 
165
 
 
166
good = []
 
167
bad = ['/something.jpg']
 
168
 
 
169
RobotTest(10, doc, good, bad, agent="Googlebot-Mobile")
 
170
 
 
171
# 11.  Get the order correct.
 
172
doc = """
 
173
User-agent: Googlebot-Mobile
 
174
Allow: /
 
175
 
 
176
User-agent: Googlebot
 
177
Disallow: /
 
178
"""
 
179
 
 
180
good = []
 
181
bad = ['/something.jpg']
 
182
 
 
183
RobotTest(11, doc, good, bad, agent="Googlebot")
 
184
 
 
185
good = ['/something.jpg']
 
186
bad = []
 
187
 
 
188
RobotTest(12, doc, good, bad, agent="Googlebot-Mobile")
 
189
 
 
190
 
 
191
# 13.  Google also got the order wrong in #8.  You need to specify the
 
192
#      URLs from more specific to more general.
 
193
doc = """
 
194
User-agent: Googlebot
 
195
Allow: /folder1/myfile.html
 
196
Disallow: /folder1/
 
197
"""
 
198
 
 
199
good = ['/folder1/myfile.html']
 
200
bad = ['/folder1/anotherfile.html']
 
201
 
 
202
RobotTest(13, doc, good, bad, agent="googlebot")
 
203
 
 
204
 
 
205
 
137
206
class TestCase(unittest.TestCase):
138
207
    def runTest(self):
139
208
        test_support.requires('network')
149
218
    TestCase().run()
150
219
 
151
220
if __name__=='__main__':
152
 
    test_support.Verbose = 1
 
221
    test_support.verbose = 1
153
222
    test_main()