| 0 | |
#! /usr/bin/env python
|
| 1 | |
# -*- coding: utf-8 -*-
|
| 2 | |
|
| 3 | |
from grequests import get, map, imap
|
| 4 | |
from nose.tools import ok_
|
| 5 | |
|
| 6 | |
########### Constants ############
|
| 7 | |
urls = [
|
| 8 | |
'http://github.com',
|
| 9 | |
'http://www.google.com',
|
| 10 | |
'http://www.psf.org'
|
| 11 | |
]
|
| 12 | |
############# tests ##############
|
| 13 | |
def test_get():
|
| 14 | |
global urls
|
| 15 | |
to_fetch = (get(url) for url in urls)
|
| 16 | |
map(to_fetch)
|
| 17 | |
for fetched in to_fetch:
|
| 18 | |
ok_(fetched.ok, True)
|
| 19 | |
|
| 20 | |
def test_imap_with_size():
|
| 21 | |
global urls
|
| 22 | |
to_fetch = (get(url) for url in urls)
|
| 23 | |
imap(to_fetch, size = len(urls) - 1)
|
| 24 | |
for fetching in to_fetch:
|
| 25 | |
ok_(fetching.send(), True)
|
| 26 | |
|
| 27 | |
import os
|
| 28 | |
import time
|
| 29 | |
import unittest
|
| 30 | |
|
| 31 | |
import requests
|
| 32 | |
from requests.exceptions import Timeout
|
| 33 | |
import grequests
|
| 34 | |
|
| 35 | |
HTTPBIN_URL = os.environ.get('HTTPBIN_URL', 'http://httpbin.org/')
|
| 36 | |
|
| 37 | |
def httpbin(*suffix):
|
| 38 | |
"""Returns url for HTTPBIN resource."""
|
| 39 | |
return HTTPBIN_URL + '/'.join(suffix)
|
| 40 | |
|
| 41 | |
|
| 42 | |
N = 5
|
| 43 | |
URLS = [httpbin('get?p=%s' % i) for i in range(N)]
|
| 44 | |
|
| 45 | |
|
| 46 | |
class GrequestsCase(unittest.TestCase):
|
| 47 | |
|
| 48 | |
def test_map(self):
|
| 49 | |
reqs = [grequests.get(url) for url in URLS]
|
| 50 | |
resp = grequests.map(reqs, size=N)
|
| 51 | |
self.assertEqual([r.url for r in resp], URLS)
|
| 52 | |
|
| 53 | |
def test_imap(self):
|
| 54 | |
reqs = (grequests.get(url) for url in URLS)
|
| 55 | |
i = 0
|
| 56 | |
for i, r in enumerate(grequests.imap(reqs, size=N)):
|
| 57 | |
self.assertTrue(r.url in URLS)
|
| 58 | |
self.assertEqual(i, N - 1)
|
| 59 | |
|
| 60 | |
def test_hooks(self):
|
| 61 | |
result = {}
|
| 62 | |
|
| 63 | |
def hook(r, **kwargs):
|
| 64 | |
result[r.url] = True
|
| 65 | |
return r
|
| 66 | |
|
| 67 | |
reqs = [grequests.get(url, hooks={'response': [hook]}) for url in URLS]
|
| 68 | |
grequests.map(reqs, size=N)
|
| 69 | |
self.assertEqual(sorted(result.keys()), sorted(URLS))
|
| 70 | |
|
| 71 | |
def test_callback_kwarg(self):
|
| 72 | |
result = {'ok': False}
|
| 73 | |
|
| 74 | |
def callback(r, **kwargs):
|
| 75 | |
result['ok'] = True
|
| 76 | |
return r
|
| 77 | |
|
| 78 | |
self.get(URLS[0], callback=callback)
|
| 79 | |
self.assertTrue(result['ok'])
|
| 80 | |
|
| 81 | |
def test_session_and_cookies(self):
|
| 82 | |
c1 = {'k1': 'v1'}
|
| 83 | |
r = self.get(httpbin('cookies/set'), params=c1).json()
|
| 84 | |
self.assertEqual(r['cookies'], c1)
|
| 85 | |
s = requests.Session()
|
| 86 | |
r = self.get(httpbin('cookies/set'), session=s, params=c1).json()
|
| 87 | |
self.assertEqual(dict(s.cookies), c1)
|
| 88 | |
|
| 89 | |
# ensure all cookies saved
|
| 90 | |
c2 = {'k2': 'v2'}
|
| 91 | |
c1.update(c2)
|
| 92 | |
r = self.get(httpbin('cookies/set'), session=s, params=c2).json()
|
| 93 | |
self.assertEqual(dict(s.cookies), c1)
|
| 94 | |
|
| 95 | |
# ensure new session is created
|
| 96 | |
r = self.get(httpbin('cookies')).json()
|
| 97 | |
self.assertEqual(r['cookies'], {})
|
| 98 | |
|
| 99 | |
# cookies as param
|
| 100 | |
c3 = {'p1': '42'}
|
| 101 | |
r = self.get(httpbin('cookies'), cookies=c3).json()
|
| 102 | |
self.assertEqual(r['cookies'], c3)
|
| 103 | |
|
| 104 | |
def test_calling_request(self):
|
| 105 | |
reqs = [grequests.request('POST', httpbin('post'), data={'p': i})
|
| 106 | |
for i in range(N)]
|
| 107 | |
resp = grequests.map(reqs, size=N)
|
| 108 | |
self.assertEqual([int(r.json()['form']['p']) for r in resp], list(range(N)))
|
| 109 | |
|
| 110 | |
def test_stream_enabled(self):
|
| 111 | |
r = grequests.map([grequests.get(httpbin('stream/10'))],
|
| 112 | |
size=2, stream=True)[0]
|
| 113 | |
self.assertFalse(r._content_consumed)
|
| 114 | |
|
| 115 | |
def test_concurrency_with_delayed_url(self):
|
| 116 | |
t = time.time()
|
| 117 | |
n = 10
|
| 118 | |
reqs = [grequests.get(httpbin('delay/1')) for _ in range(n)]
|
| 119 | |
grequests.map(reqs, size=n)
|
| 120 | |
self.assertLess((time.time() - t), n)
|
| 121 | |
|
| 122 | |
def test_map_timeout_no_exception_handler(self):
|
| 123 | |
"""
|
| 124 | |
compliance with existing 0.2.0 behaviour
|
| 125 | |
"""
|
| 126 | |
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
|
| 127 | |
responses = grequests.map(reqs)
|
| 128 | |
self.assertIsNone(responses[0])
|
| 129 | |
self.assertTrue(responses[1].ok)
|
| 130 | |
self.assertEqual(len(responses), 2)
|
| 131 | |
|
| 132 | |
def test_map_timeout_exception_handler_no_return(self):
|
| 133 | |
"""
|
| 134 | |
ensure default behaviour for a handler that returns None
|
| 135 | |
"""
|
| 136 | |
def exception_handler(request, exception):
|
| 137 | |
pass
|
| 138 | |
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
|
| 139 | |
responses = grequests.map(reqs, exception_handler=exception_handler)
|
| 140 | |
self.assertIsNone(responses[0])
|
| 141 | |
self.assertTrue(responses[1].ok)
|
| 142 | |
self.assertEqual(len(responses), 2)
|
| 143 | |
|
| 144 | |
def test_map_timeout_exception_handler_returns_exception(self):
|
| 145 | |
"""
|
| 146 | |
ensure returned value from exception handler is stuffed in the map result
|
| 147 | |
"""
|
| 148 | |
def exception_handler(request, exception):
|
| 149 | |
return exception
|
| 150 | |
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
|
| 151 | |
responses = grequests.map(reqs, exception_handler=exception_handler)
|
| 152 | |
self.assertIsInstance(responses[0], Timeout)
|
| 153 | |
self.assertTrue(responses[1].ok)
|
| 154 | |
self.assertEqual(len(responses), 2)
|
| 155 | |
|
| 156 | |
def test_imap_timeout_no_exception_handler(self):
|
| 157 | |
"""
|
| 158 | |
compliance with existing 0.2.0 behaviour
|
| 159 | |
"""
|
| 160 | |
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)]
|
| 161 | |
out = []
|
| 162 | |
try:
|
| 163 | |
for r in grequests.imap(reqs):
|
| 164 | |
out.append(r)
|
| 165 | |
except Timeout:
|
| 166 | |
pass
|
| 167 | |
self.assertEquals(out, [])
|
| 168 | |
|
| 169 | |
def test_imap_timeout_exception_handler_no_return(self):
|
| 170 | |
"""
|
| 171 | |
ensure imap-default behaviour for a handler that returns None
|
| 172 | |
"""
|
| 173 | |
def exception_handler(request, exception):
|
| 174 | |
pass
|
| 175 | |
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)]
|
| 176 | |
out = []
|
| 177 | |
for r in grequests.imap(reqs, exception_handler=exception_handler):
|
| 178 | |
out.append(r)
|
| 179 | |
self.assertEquals(out, [])
|
| 180 | |
|
| 181 | |
|
| 182 | |
def test_imap_timeout_exception_handler_returns_value(self):
|
| 183 | |
"""
|
| 184 | |
ensure behaviour for a handler that returns a value
|
| 185 | |
"""
|
| 186 | |
def exception_handler(request, exception):
|
| 187 | |
return 'a value'
|
| 188 | |
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)]
|
| 189 | |
out = []
|
| 190 | |
for r in grequests.imap(reqs, exception_handler=exception_handler):
|
| 191 | |
out.append(r)
|
| 192 | |
self.assertEquals(out, ['a value'])
|
| 193 | |
|
| 194 | |
def test_map_timeout_exception(self):
|
| 195 | |
class ExceptionHandler:
|
| 196 | |
def __init__(self):
|
| 197 | |
self.counter = 0
|
| 198 | |
|
| 199 | |
def callback(self, request, exception):
|
| 200 | |
self.counter += 1
|
| 201 | |
eh = ExceptionHandler()
|
| 202 | |
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)]
|
| 203 | |
list(grequests.map(reqs, exception_handler=eh.callback))
|
| 204 | |
self.assertEqual(eh.counter, 1)
|
| 205 | |
|
| 206 | |
def test_imap_timeout_exception(self):
|
| 207 | |
class ExceptionHandler:
|
| 208 | |
def __init__(self):
|
| 209 | |
self.counter = 0
|
| 210 | |
|
| 211 | |
def callback(self, request, exception):
|
| 212 | |
self.counter += 1
|
| 213 | |
eh = ExceptionHandler()
|
| 214 | |
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)]
|
| 215 | |
list(grequests.imap(reqs, exception_handler=eh.callback))
|
| 216 | |
self.assertEqual(eh.counter, 1)
|
| 217 | |
|
| 218 | |
def get(self, url, **kwargs):
|
| 219 | |
return grequests.map([grequests.get(url, **kwargs)])[0]
|
| 220 | |
|
| 221 | |
|
| 222 | |
if __name__ == '__main__':
|
| 223 | |
unittest.main()
|