Import upstream version 0.6.0+git20220126.1.47de40a
Kali Janitor
1 year, 4 months ago
0 | .coverage | |
1 | MANIFEST | |
2 | coverage.xml | |
3 | nosetests.xml | |
4 | junit-report.xml | |
5 | pylint.txt | |
6 | toy.py | |
7 | violations.pyflakes.txt | |
8 | cover/ | |
9 | docs/_build | |
10 | grequests.egg-info/ | |
11 | *.py[cx] | |
12 | *.swp | |
13 | env/ |
0 | dist: xenial | |
1 | language: python | |
2 | ||
3 | ||
4 | python: | |
5 | - "2.7" | |
6 | - "3.6" | |
7 | - "3.7" | |
8 | ||
9 | ||
10 | install: | |
11 | - pip install -r requirements.txt | |
12 | - pip install pytest | |
13 | ||
14 | script: | |
15 | - pytest tests.py | |
16 | ||
17 | ||
18 | deploy: | |
19 | provider: pypi | |
20 | distributions: "sdist bdist_wheel" | |
21 | user: "spyoungtech" | |
22 | password: | |
23 | secure: "QtuuH0X/A/iQI23MxvqsnxUy63XD5awJHDkeQNmUDIGGQqIox2DTYKoc6x354I5wpqprtODQRYRqIsA9+2cpRcF49Ft50cvi3cmuoeozkID3ybQyLHCIcJ4CKt6X+h2LFbrgqyyBcny7tKQlYr4/nsjeQegPblnJ6OTljJgJyE0=" | |
24 | on: | |
25 | tags: true | |
26 | python: 3.6⏎ |
0 | GRequests is written and maintained by Kenneth Reitz and | |
0 | GRequests is authored by Kenneth Reitz, maintained by Spencer Young and | |
1 | 1 | various contributors: |
2 | 2 | |
3 | Development Lead | |
4 | ```````````````` | |
3 | Development Leads | |
4 | ````````````````` | |
5 | 5 | |
6 | - Spencer Phillip Young <[email protected]> | |
6 | 7 | - Kenneth Reitz <[email protected]> |
7 | 8 | |
8 | 9 | Patches and Suggestions |
9 | 10 | ``````````````````````` |
10 | - Kracekumar <[email protected]> | |
11 | - Spencer Young <[email protected]> | |
11 | ||
12 | Adam Tauber <[email protected]> | |
13 | Akshat Mahajan <[email protected]> | |
14 | Alexander Simeonov <[email protected]> | |
15 | Antonio A <[email protected]> | |
16 | Chris Drackett <[email protected]> | |
17 | Eugene Eeo <[email protected]> | |
18 | Frost Ming <[email protected]> | |
19 | Ian Cordasco <[email protected]> | |
20 | Joe Gordon <[email protected]> | |
21 | Luke Hutscal <[email protected]> | |
22 | Marc Abramowitz <[email protected]> | |
23 | Mathieu Lecarme <[email protected]> | |
24 | Michael Newman <[email protected]> | |
25 | Mircea Ulinic <[email protected]> | |
26 | Nate Lawson <[email protected]> | |
27 | Nathan Hoad <[email protected]> | |
28 | Roman Haritonov <[email protected]> | |
29 | Ryan T. Dean <[email protected]> | |
30 | Spencer Phillip Young <[email protected]> | |
31 | Spencer Young <[email protected]> | |
32 | Yuri Prezument <[email protected]> | |
33 | koobs <[email protected]> | |
34 | kracekumar <[email protected]> | |
35 | 崔庆才丨静觅 <[email protected]> |
0 | Metadata-Version: 2.1 | |
1 | Name: grequests | |
2 | Version: 0.6.0 | |
3 | Summary: Requests + Gevent | |
4 | Home-page: https://github.com/kennethreitz/grequests | |
5 | Author: Kenneth Reitz | |
6 | Author-email: [email protected] | |
7 | License: BSD | |
8 | Platform: any | |
9 | Classifier: Environment :: Web Environment | |
10 | Classifier: Intended Audience :: Developers | |
11 | Classifier: License :: OSI Approved :: BSD License | |
12 | Classifier: Operating System :: OS Independent | |
13 | Classifier: Programming Language :: Python | |
14 | Classifier: Programming Language :: Python :: 2.7 | |
15 | Classifier: Programming Language :: Python :: 3 | |
16 | Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content | |
17 | Classifier: Topic :: Software Development :: Libraries :: Python Modules | |
18 | License-File: LICENSE | |
19 | License-File: AUTHORS.rst | |
20 | ||
21 | ||
22 | GRequests allows you to use Requests with Gevent to make asynchronous HTTP | |
23 | Requests easily. | |
24 | ||
25 | Usage | |
26 | ----- | |
27 | ||
28 | Usage is simple:: | |
29 | ||
30 | import grequests | |
31 | ||
32 | urls = [ | |
33 | 'http://www.heroku.com', | |
34 | 'http://tablib.org', | |
35 | 'http://httpbin.org', | |
36 | 'http://python-requests.org', | |
37 | 'http://kennethreitz.com' | |
38 | ] | |
39 | ||
40 | Create a set of unsent Requests:: | |
41 | ||
42 | >>> rs = (grequests.get(u) for u in urls) | |
43 | ||
44 | Send them all at the same time:: | |
45 | ||
46 | >>> grequests.map(rs) | |
47 | [<Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>] | |
48 |
3 | 3 | GRequests allows you to use Requests with Gevent to make asynchronous HTTP |
4 | 4 | Requests easily. |
5 | 5 | |
6 | |version| |pyversions| | |
7 | ||
8 | ||
6 | 9 | **Note**: You should probably use `requests-threads <https://github.com/requests/requests-threads>`_ or `requests-futures <https://github.com/ross/requests-futures>`_ instead. |
10 | ||
11 | ||
12 | Installation | |
13 | ------------ | |
14 | ||
15 | Installation is easy with pip:: | |
16 | ||
17 | $ pip install grequests | |
18 | ✨🍰✨ | |
7 | 19 | |
8 | 20 | |
9 | 21 | Usage |
37 | 49 | >>> grequests.map(rs) |
38 | 50 | [<Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>, None, <Response [200]>] |
39 | 51 | |
40 | Optionally, in the event of a timeout or any other exception during the connection of | |
41 | the request, you can add an exception handler that will be called with the request and | |
52 | ||
53 | The HTTP verb methods in ``grequests`` (``grequests.get``, ``grequests.post``, etc) accept all the same keyword arguments as in the ``requests`` library. | |
54 | ||
55 | To handle timeouts or any other exception during the connection of | |
56 | the request, you can add an optional exception handler that will be called with the request and | |
42 | 57 | exception inside the main thread: |
43 | 58 | |
44 | 59 | .. code-block:: python |
45 | 60 | |
46 | 61 | >>> def exception_handler(request, exception): |
47 | ... print "Request failed" | |
62 | ... print("Request failed") | |
48 | 63 | |
49 | 64 | >>> reqs = [ |
50 | 65 | ... grequests.get('http://httpbin.org/delay/1', timeout=0.001), |
55 | 70 | Request failed |
56 | 71 | [None, None, <Response [500]>] |
57 | 72 | |
58 | For some speed/performance gains, you may also want to use `imap` instead of `map`. `imap` returns a generator of responses. Order of these responses does not map to the order of the requests you send out. The API for `imap` is equivalent to the API for `map`. | |
73 | For some speed/performance gains, you may also want to use ``imap`` instead of ``map``. ``imap`` returns a generator of responses. Order of these responses does not map to the order of the requests you send out. The API for ``imap`` is equivalent to the API for ``map``. You can also adjust the ``size`` argument to ``map`` or ``imap`` to increase the gevent pool size. | |
59 | 74 | |
60 | Installation | |
61 | ------------ | |
62 | 75 | |
63 | Installation is easy with pip:: | |
76 | .. code-block:: python | |
64 | 77 | |
65 | $ pip install grequests | |
66 | ✨🍰✨ | |
78 | for resp in grequests.imap(reqs, size=10): | |
79 | print(resp) | |
80 | ||
81 | ||
82 | ||
83 | NOTE: because ``grequests`` leverages ``gevent`` (which in turn uses monkeypatching for enabling concurrency), you will often need to make sure ``grequests`` is imported before other libraries, especially ``requests``, to avoid problems. See `grequests gevent issues <https://github.com/spyoungtech/grequests/issues?q=is%3Aissue+label%3A%22%3Ahear_no_evil%3A%3Asee_no_evil%3A%3Aspeak_no_evil%3A++gevent%22+>`_ for additional information. | |
84 | ||
85 | ||
86 | .. code-block:: python | |
87 | ||
88 | # GOOD | |
89 | import grequests | |
90 | import requests | |
91 | ||
92 | # BAD | |
93 | import requests | |
94 | import grequests | |
95 | ||
96 | ||
97 | ||
98 | ||
99 | ||
100 | ||
101 | ||
102 | .. |version| image:: https://img.shields.io/pypi/v/grequests.svg?colorB=blue | |
103 | :target: https://pypi.org/project/grequests/ | |
104 | ||
105 | .. |pyversions| image:: https://img.shields.io/pypi/pyversions/grequests.svg? | |
106 | :target: https://pypi.org/project/grequests/ | |
107 | ||
108 |
0 | Metadata-Version: 2.1 | |
1 | Name: grequests | |
2 | Version: 0.6.0 | |
3 | Summary: Requests + Gevent | |
4 | Home-page: https://github.com/kennethreitz/grequests | |
5 | Author: Kenneth Reitz | |
6 | Author-email: [email protected] | |
7 | License: BSD | |
8 | Platform: any | |
9 | Classifier: Environment :: Web Environment | |
10 | Classifier: Intended Audience :: Developers | |
11 | Classifier: License :: OSI Approved :: BSD License | |
12 | Classifier: Operating System :: OS Independent | |
13 | Classifier: Programming Language :: Python | |
14 | Classifier: Programming Language :: Python :: 2.7 | |
15 | Classifier: Programming Language :: Python :: 3 | |
16 | Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content | |
17 | Classifier: Topic :: Software Development :: Libraries :: Python Modules | |
18 | License-File: LICENSE | |
19 | License-File: AUTHORS.rst | |
20 | ||
21 | ||
22 | GRequests allows you to use Requests with Gevent to make asynchronous HTTP | |
23 | Requests easily. | |
24 | ||
25 | Usage | |
26 | ----- | |
27 | ||
28 | Usage is simple:: | |
29 | ||
30 | import grequests | |
31 | ||
32 | urls = [ | |
33 | 'http://www.heroku.com', | |
34 | 'http://tablib.org', | |
35 | 'http://httpbin.org', | |
36 | 'http://python-requests.org', | |
37 | 'http://kennethreitz.com' | |
38 | ] | |
39 | ||
40 | Create a set of unsent Requests:: | |
41 | ||
42 | >>> rs = (grequests.get(u) for u in urls) | |
43 | ||
44 | Send them all at the same time:: | |
45 | ||
46 | >>> grequests.map(rs) | |
47 | [<Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>] | |
48 |
0 | AUTHORS.rst | |
1 | LICENSE | |
2 | MANIFEST.in | |
3 | README.rst | |
4 | grequests.py | |
5 | setup.py | |
6 | grequests.egg-info/PKG-INFO | |
7 | grequests.egg-info/SOURCES.txt | |
8 | grequests.egg-info/dependency_links.txt | |
9 | grequests.egg-info/not-zip-safe | |
10 | grequests.egg-info/requires.txt | |
11 | grequests.egg-info/top_level.txt⏎ |
0 | grequests |
9 | 9 | """ |
10 | 10 | from functools import partial |
11 | 11 | import traceback |
12 | ||
12 | 13 | try: |
13 | 14 | import gevent |
14 | 15 | from gevent import monkey as curious_george |
20 | 21 | curious_george.patch_all(thread=False, select=False) |
21 | 22 | |
22 | 23 | from requests import Session |
23 | ||
24 | 24 | |
25 | 25 | __all__ = ( |
26 | 26 | 'map', 'imap', |
46 | 46 | self.session = kwargs.pop('session', None) |
47 | 47 | if self.session is None: |
48 | 48 | self.session = Session() |
49 | self._close = True | |
50 | else: | |
51 | self._close = False # don't close adapters after each request if the user provided the session | |
49 | 52 | |
50 | 53 | callback = kwargs.pop('callback', None) |
51 | 54 | if callback: |
72 | 75 | except Exception as e: |
73 | 76 | self.exception = e |
74 | 77 | self.traceback = traceback.format_exc() |
78 | finally: | |
79 | if self._close: | |
80 | # if we provided the session object, make sure we're cleaning up | |
81 | # because there's no sense in keeping it open at this point if it wont be reused | |
82 | self.session.close() | |
75 | 83 | return self |
76 | 84 | |
77 | 85 | |
122 | 130 | ret.append(request.response) |
123 | 131 | elif exception_handler and hasattr(request, 'exception'): |
124 | 132 | ret.append(exception_handler(request, request.exception)) |
133 | elif exception_handler and not hasattr(request, 'exception'): | |
134 | ret.append(exception_handler(request, None)) | |
125 | 135 | else: |
126 | 136 | ret.append(None) |
127 | 137 |
32 | 32 | |
33 | 33 | setup( |
34 | 34 | name='grequests', |
35 | version='0.4.0', | |
35 | version='0.6.0', | |
36 | 36 | url='https://github.com/kennethreitz/grequests', |
37 | 37 | license='BSD', |
38 | 38 | author='Kenneth Reitz', |
0 | #! /usr/bin/env python | |
1 | # -*- coding: utf-8 -*- | |
2 | ||
3 | from grequests import get, map, imap | |
4 | from nose.tools import ok_ | |
5 | ||
6 | ########### Constants ############ | |
7 | urls = [ | |
8 | 'http://github.com', | |
9 | 'http://www.google.com', | |
10 | 'http://www.psf.org' | |
11 | ] | |
12 | ############# tests ############## | |
13 | def test_get(): | |
14 | global urls | |
15 | to_fetch = (get(url) for url in urls) | |
16 | map(to_fetch) | |
17 | for fetched in to_fetch: | |
18 | ok_(fetched.ok, True) | |
19 | ||
20 | def test_imap_with_size(): | |
21 | global urls | |
22 | to_fetch = (get(url) for url in urls) | |
23 | imap(to_fetch, size = len(urls) - 1) | |
24 | for fetching in to_fetch: | |
25 | ok_(fetching.send(), True) | |
26 | ||
27 | import os | |
28 | import time | |
29 | import unittest | |
30 | ||
31 | import requests | |
32 | from requests.exceptions import Timeout | |
33 | import grequests | |
34 | ||
35 | HTTPBIN_URL = os.environ.get('HTTPBIN_URL', 'http://httpbin.org/') | |
36 | ||
37 | def httpbin(*suffix): | |
38 | """Returns url for HTTPBIN resource.""" | |
39 | return HTTPBIN_URL + '/'.join(suffix) | |
40 | ||
41 | ||
42 | N = 5 | |
43 | URLS = [httpbin('get?p=%s' % i) for i in range(N)] | |
44 | ||
45 | ||
46 | class GrequestsCase(unittest.TestCase): | |
47 | ||
48 | def test_map(self): | |
49 | reqs = [grequests.get(url) for url in URLS] | |
50 | resp = grequests.map(reqs, size=N) | |
51 | self.assertEqual([r.url for r in resp], URLS) | |
52 | ||
53 | def test_imap(self): | |
54 | reqs = (grequests.get(url) for url in URLS) | |
55 | i = 0 | |
56 | for i, r in enumerate(grequests.imap(reqs, size=N)): | |
57 | self.assertTrue(r.url in URLS) | |
58 | self.assertEqual(i, N - 1) | |
59 | ||
60 | def test_hooks(self): | |
61 | result = {} | |
62 | ||
63 | def hook(r, **kwargs): | |
64 | result[r.url] = True | |
65 | return r | |
66 | ||
67 | reqs = [grequests.get(url, hooks={'response': [hook]}) for url in URLS] | |
68 | grequests.map(reqs, size=N) | |
69 | self.assertEqual(sorted(result.keys()), sorted(URLS)) | |
70 | ||
71 | def test_callback_kwarg(self): | |
72 | result = {'ok': False} | |
73 | ||
74 | def callback(r, **kwargs): | |
75 | result['ok'] = True | |
76 | return r | |
77 | ||
78 | self.get(URLS[0], callback=callback) | |
79 | self.assertTrue(result['ok']) | |
80 | ||
81 | def test_session_and_cookies(self): | |
82 | c1 = {'k1': 'v1'} | |
83 | r = self.get(httpbin('cookies/set'), params=c1).json() | |
84 | self.assertEqual(r['cookies'], c1) | |
85 | s = requests.Session() | |
86 | r = self.get(httpbin('cookies/set'), session=s, params=c1).json() | |
87 | self.assertEqual(dict(s.cookies), c1) | |
88 | ||
89 | # ensure all cookies saved | |
90 | c2 = {'k2': 'v2'} | |
91 | c1.update(c2) | |
92 | r = self.get(httpbin('cookies/set'), session=s, params=c2).json() | |
93 | self.assertEqual(dict(s.cookies), c1) | |
94 | ||
95 | # ensure new session is created | |
96 | r = self.get(httpbin('cookies')).json() | |
97 | self.assertEqual(r['cookies'], {}) | |
98 | ||
99 | # cookies as param | |
100 | c3 = {'p1': '42'} | |
101 | r = self.get(httpbin('cookies'), cookies=c3).json() | |
102 | self.assertEqual(r['cookies'], c3) | |
103 | ||
104 | def test_calling_request(self): | |
105 | reqs = [grequests.request('POST', httpbin('post'), data={'p': i}) | |
106 | for i in range(N)] | |
107 | resp = grequests.map(reqs, size=N) | |
108 | self.assertEqual([int(r.json()['form']['p']) for r in resp], list(range(N))) | |
109 | ||
110 | def test_stream_enabled(self): | |
111 | r = grequests.map([grequests.get(httpbin('stream/10'))], | |
112 | size=2, stream=True)[0] | |
113 | self.assertFalse(r._content_consumed) | |
114 | ||
115 | def test_concurrency_with_delayed_url(self): | |
116 | t = time.time() | |
117 | n = 10 | |
118 | reqs = [grequests.get(httpbin('delay/1')) for _ in range(n)] | |
119 | grequests.map(reqs, size=n) | |
120 | self.assertLess((time.time() - t), n) | |
121 | ||
122 | def test_map_timeout_no_exception_handler(self): | |
123 | """ | |
124 | compliance with existing 0.2.0 behaviour | |
125 | """ | |
126 | reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))] | |
127 | responses = grequests.map(reqs) | |
128 | self.assertIsNone(responses[0]) | |
129 | self.assertTrue(responses[1].ok) | |
130 | self.assertEqual(len(responses), 2) | |
131 | ||
132 | def test_map_timeout_exception_handler_no_return(self): | |
133 | """ | |
134 | ensure default behaviour for a handler that returns None | |
135 | """ | |
136 | def exception_handler(request, exception): | |
137 | pass | |
138 | reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))] | |
139 | responses = grequests.map(reqs, exception_handler=exception_handler) | |
140 | self.assertIsNone(responses[0]) | |
141 | self.assertTrue(responses[1].ok) | |
142 | self.assertEqual(len(responses), 2) | |
143 | ||
144 | def test_map_timeout_exception_handler_returns_exception(self): | |
145 | """ | |
146 | ensure returned value from exception handler is stuffed in the map result | |
147 | """ | |
148 | def exception_handler(request, exception): | |
149 | return exception | |
150 | reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))] | |
151 | responses = grequests.map(reqs, exception_handler=exception_handler) | |
152 | self.assertIsInstance(responses[0], Timeout) | |
153 | self.assertTrue(responses[1].ok) | |
154 | self.assertEqual(len(responses), 2) | |
155 | ||
156 | def test_imap_timeout_no_exception_handler(self): | |
157 | """ | |
158 | compliance with existing 0.2.0 behaviour | |
159 | """ | |
160 | reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)] | |
161 | out = [] | |
162 | try: | |
163 | for r in grequests.imap(reqs): | |
164 | out.append(r) | |
165 | except Timeout: | |
166 | pass | |
167 | self.assertEquals(out, []) | |
168 | ||
169 | def test_imap_timeout_exception_handler_no_return(self): | |
170 | """ | |
171 | ensure imap-default behaviour for a handler that returns None | |
172 | """ | |
173 | def exception_handler(request, exception): | |
174 | pass | |
175 | reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)] | |
176 | out = [] | |
177 | for r in grequests.imap(reqs, exception_handler=exception_handler): | |
178 | out.append(r) | |
179 | self.assertEquals(out, []) | |
180 | ||
181 | ||
182 | def test_imap_timeout_exception_handler_returns_value(self): | |
183 | """ | |
184 | ensure behaviour for a handler that returns a value | |
185 | """ | |
186 | def exception_handler(request, exception): | |
187 | return 'a value' | |
188 | reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)] | |
189 | out = [] | |
190 | for r in grequests.imap(reqs, exception_handler=exception_handler): | |
191 | out.append(r) | |
192 | self.assertEquals(out, ['a value']) | |
193 | ||
194 | def test_map_timeout_exception(self): | |
195 | class ExceptionHandler: | |
196 | def __init__(self): | |
197 | self.counter = 0 | |
198 | ||
199 | def callback(self, request, exception): | |
200 | self.counter += 1 | |
201 | eh = ExceptionHandler() | |
202 | reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)] | |
203 | list(grequests.map(reqs, exception_handler=eh.callback)) | |
204 | self.assertEqual(eh.counter, 1) | |
205 | ||
206 | def test_imap_timeout_exception(self): | |
207 | class ExceptionHandler: | |
208 | def __init__(self): | |
209 | self.counter = 0 | |
210 | ||
211 | def callback(self, request, exception): | |
212 | self.counter += 1 | |
213 | eh = ExceptionHandler() | |
214 | reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)] | |
215 | list(grequests.imap(reqs, exception_handler=eh.callback)) | |
216 | self.assertEqual(eh.counter, 1) | |
217 | ||
218 | def get(self, url, **kwargs): | |
219 | return grequests.map([grequests.get(url, **kwargs)])[0] | |
220 | ||
221 | ||
222 | if __name__ == '__main__': | |
223 | unittest.main() |