diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 624f4a6..0000000 --- a/.gitignore +++ /dev/null @@ -1,14 +0,0 @@ -.coverage -MANIFEST -coverage.xml -nosetests.xml -junit-report.xml -pylint.txt -toy.py -violations.pyflakes.txt -cover/ -docs/_build -grequests.egg-info/ -*.py[cx] -*.swp -env/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c4f010d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,27 +0,0 @@ -dist: xenial -language: python - - -python: - - "2.7" - - "3.6" - - "3.7" - - -install: - - pip install -r requirements.txt - - pip install pytest - -script: - - pytest tests.py - - -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - user: "spyoungtech" - password: - secure: "QtuuH0X/A/iQI23MxvqsnxUy63XD5awJHDkeQNmUDIGGQqIox2DTYKoc6x354I5wpqprtODQRYRqIsA9+2cpRcF49Ft50cvi3cmuoeozkID3ybQyLHCIcJ4CKt6X+h2LFbrgqyyBcny7tKQlYr4/nsjeQegPblnJ6OTljJgJyE0=" - on: - tags: true - python: 3.6 \ No newline at end of file diff --git a/AUTHORS.rst b/AUTHORS.rst index b93507b..dc2e98c 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,12 +1,36 @@ -GRequests is written and maintained by Kenneth Reitz and +GRequests is authored by Kenneth Reitz, maintained by Spencer Young and various contributors: -Development Lead -```````````````` +Development Leads +````````````````` +- Spencer Phillip Young - Kenneth Reitz Patches and Suggestions ``````````````````````` -- Kracekumar -- Spencer Young + +Adam Tauber +Akshat Mahajan +Alexander Simeonov +Antonio A +Chris Drackett +Eugene Eeo +Frost Ming +Ian Cordasco +Joe Gordon +Luke Hutscal +Marc Abramowitz +Mathieu Lecarme +Michael Newman +Mircea Ulinic +Nate Lawson +Nathan Hoad +Roman Haritonov +Ryan T. Dean +Spencer Phillip Young +Spencer Young +Yuri Prezument +koobs +kracekumar +崔庆才丨静觅 diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..9d5d250 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE +include README.rst diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..1181a77 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,51 @@ +Metadata-Version: 2.1 +Name: grequests +Version: 0.6.0 +Summary: Requests + Gevent +Home-page: https://github.com/kennethreitz/grequests +Author: Kenneth Reitz +Author-email: me@kennethreitz.com +License: BSD +Platform: any +Classifier: Environment :: Web Environment +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content +Classifier: Topic :: Software Development :: Libraries :: Python Modules +License-File: LICENSE +License-File: AUTHORS.rst + + +GRequests allows you to use Requests with Gevent to make asynchronous HTTP +Requests easily. + +Usage +----- + +Usage is simple:: + + import grequests + + urls = [ + 'http://www.heroku.com', + 'http://tablib.org', + 'http://httpbin.org', + 'http://python-requests.org', + 'http://kennethreitz.com' + ] + +Create a set of unsent Requests:: + + >>> rs = (grequests.get(u) for u in urls) + +Send them all at the same time:: + + >>> grequests.map(rs) + [, , , , ] + + + diff --git a/README.rst b/README.rst index 3ea3e86..20db323 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,19 @@ GRequests allows you to use Requests with Gevent to make asynchronous HTTP Requests easily. +|version| |pyversions| + + **Note**: You should probably use `requests-threads `_ or `requests-futures `_ instead. + + +Installation +------------ + +Installation is easy with pip:: + + $ pip install grequests + ✨🍰✨ Usage @@ -38,14 +50,17 @@ >>> grequests.map(rs) [, , , , None, ] -Optionally, in the event of a timeout or any other exception during the connection of -the request, you can add an exception handler that will be called with the request and + +The HTTP verb methods in ``grequests`` (``grequests.get``, ``grequests.post``, etc) accept all the same keyword arguments as in the ``requests`` library. + +To handle timeouts or any other exception during the connection of +the request, you can add an optional exception handler that will be called with the request and exception inside the main thread: .. code-block:: python >>> def exception_handler(request, exception): - ... print "Request failed" + ... print("Request failed") >>> reqs = [ ... grequests.get('http://httpbin.org/delay/1', timeout=0.001), @@ -56,12 +71,39 @@ Request failed [None, None, ] -For some speed/performance gains, you may also want to use `imap` instead of `map`. `imap` returns a generator of responses. Order of these responses does not map to the order of the requests you send out. The API for `imap` is equivalent to the API for `map`. +For some speed/performance gains, you may also want to use ``imap`` instead of ``map``. ``imap`` returns a generator of responses. Order of these responses does not map to the order of the requests you send out. The API for ``imap`` is equivalent to the API for ``map``. You can also adjust the ``size`` argument to ``map`` or ``imap`` to increase the gevent pool size. -Installation ------------- -Installation is easy with pip:: +.. code-block:: python - $ pip install grequests - ✨🍰✨ + for resp in grequests.imap(reqs, size=10): + print(resp) + + + +NOTE: because ``grequests`` leverages ``gevent`` (which in turn uses monkeypatching for enabling concurrency), you will often need to make sure ``grequests`` is imported before other libraries, especially ``requests``, to avoid problems. See `grequests gevent issues `_ for additional information. + + +.. code-block:: python + + # GOOD + import grequests + import requests + + # BAD + import requests + import grequests + + + + + + + +.. |version| image:: https://img.shields.io/pypi/v/grequests.svg?colorB=blue + :target: https://pypi.org/project/grequests/ + +.. |pyversions| image:: https://img.shields.io/pypi/pyversions/grequests.svg? + :target: https://pypi.org/project/grequests/ + + diff --git a/grequests.egg-info/PKG-INFO b/grequests.egg-info/PKG-INFO new file mode 100644 index 0000000..1181a77 --- /dev/null +++ b/grequests.egg-info/PKG-INFO @@ -0,0 +1,51 @@ +Metadata-Version: 2.1 +Name: grequests +Version: 0.6.0 +Summary: Requests + Gevent +Home-page: https://github.com/kennethreitz/grequests +Author: Kenneth Reitz +Author-email: me@kennethreitz.com +License: BSD +Platform: any +Classifier: Environment :: Web Environment +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content +Classifier: Topic :: Software Development :: Libraries :: Python Modules +License-File: LICENSE +License-File: AUTHORS.rst + + +GRequests allows you to use Requests with Gevent to make asynchronous HTTP +Requests easily. + +Usage +----- + +Usage is simple:: + + import grequests + + urls = [ + 'http://www.heroku.com', + 'http://tablib.org', + 'http://httpbin.org', + 'http://python-requests.org', + 'http://kennethreitz.com' + ] + +Create a set of unsent Requests:: + + >>> rs = (grequests.get(u) for u in urls) + +Send them all at the same time:: + + >>> grequests.map(rs) + [, , , , ] + + + diff --git a/grequests.egg-info/SOURCES.txt b/grequests.egg-info/SOURCES.txt new file mode 100644 index 0000000..aee273d --- /dev/null +++ b/grequests.egg-info/SOURCES.txt @@ -0,0 +1,12 @@ +AUTHORS.rst +LICENSE +MANIFEST.in +README.rst +grequests.py +setup.py +grequests.egg-info/PKG-INFO +grequests.egg-info/SOURCES.txt +grequests.egg-info/dependency_links.txt +grequests.egg-info/not-zip-safe +grequests.egg-info/requires.txt +grequests.egg-info/top_level.txt \ No newline at end of file diff --git a/grequests.egg-info/dependency_links.txt b/grequests.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/grequests.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/grequests.egg-info/not-zip-safe b/grequests.egg-info/not-zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/grequests.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/grequests.egg-info/requires.txt b/grequests.egg-info/requires.txt new file mode 100644 index 0000000..2c4f918 --- /dev/null +++ b/grequests.egg-info/requires.txt @@ -0,0 +1,2 @@ +gevent +requests diff --git a/grequests.egg-info/top_level.txt b/grequests.egg-info/top_level.txt new file mode 100644 index 0000000..2276f04 --- /dev/null +++ b/grequests.egg-info/top_level.txt @@ -0,0 +1 @@ +grequests diff --git a/grequests.py b/grequests.py index 582cffb..f4ec5c7 100755 --- a/grequests.py +++ b/grequests.py @@ -10,6 +10,7 @@ """ from functools import partial import traceback + try: import gevent from gevent import monkey as curious_george @@ -21,7 +22,6 @@ curious_george.patch_all(thread=False, select=False) from requests import Session - __all__ = ( 'map', 'imap', @@ -47,6 +47,9 @@ self.session = kwargs.pop('session', None) if self.session is None: self.session = Session() + self._close = True + else: + self._close = False # don't close adapters after each request if the user provided the session callback = kwargs.pop('callback', None) if callback: @@ -73,6 +76,11 @@ except Exception as e: self.exception = e self.traceback = traceback.format_exc() + finally: + if self._close: + # if we provided the session object, make sure we're cleaning up + # because there's no sense in keeping it open at this point if it wont be reused + self.session.close() return self @@ -123,6 +131,8 @@ ret.append(request.response) elif exception_handler and hasattr(request, 'exception'): ret.append(exception_handler(request, request.exception)) + elif exception_handler and not hasattr(request, 'exception'): + ret.append(exception_handler(request, None)) else: ret.append(None) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index ebe8eeb..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -requests -gevent -nose diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..8bfd5a1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[egg_info] +tag_build = +tag_date = 0 + diff --git a/setup.py b/setup.py index defc7cf..9122abd 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( name='grequests', - version='0.4.0', + version='0.6.0', url='https://github.com/kennethreitz/grequests', license='BSD', author='Kenneth Reitz', diff --git a/tests.py b/tests.py deleted file mode 100644 index 2fda406..0000000 --- a/tests.py +++ /dev/null @@ -1,224 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -from grequests import get, map, imap -from nose.tools import ok_ - -########### Constants ############ -urls = [ - 'http://github.com', - 'http://www.google.com', - 'http://www.psf.org' - ] -############# tests ############## -def test_get(): - global urls - to_fetch = (get(url) for url in urls) - map(to_fetch) - for fetched in to_fetch: - ok_(fetched.ok, True) - -def test_imap_with_size(): - global urls - to_fetch = (get(url) for url in urls) - imap(to_fetch, size = len(urls) - 1) - for fetching in to_fetch: - ok_(fetching.send(), True) - -import os -import time -import unittest - -import requests -from requests.exceptions import Timeout -import grequests - -HTTPBIN_URL = os.environ.get('HTTPBIN_URL', 'http://httpbin.org/') - -def httpbin(*suffix): - """Returns url for HTTPBIN resource.""" - return HTTPBIN_URL + '/'.join(suffix) - - -N = 5 -URLS = [httpbin('get?p=%s' % i) for i in range(N)] - - -class GrequestsCase(unittest.TestCase): - - def test_map(self): - reqs = [grequests.get(url) for url in URLS] - resp = grequests.map(reqs, size=N) - self.assertEqual([r.url for r in resp], URLS) - - def test_imap(self): - reqs = (grequests.get(url) for url in URLS) - i = 0 - for i, r in enumerate(grequests.imap(reqs, size=N)): - self.assertTrue(r.url in URLS) - self.assertEqual(i, N - 1) - - def test_hooks(self): - result = {} - - def hook(r, **kwargs): - result[r.url] = True - return r - - reqs = [grequests.get(url, hooks={'response': [hook]}) for url in URLS] - grequests.map(reqs, size=N) - self.assertEqual(sorted(result.keys()), sorted(URLS)) - - def test_callback_kwarg(self): - result = {'ok': False} - - def callback(r, **kwargs): - result['ok'] = True - return r - - self.get(URLS[0], callback=callback) - self.assertTrue(result['ok']) - - def test_session_and_cookies(self): - c1 = {'k1': 'v1'} - r = self.get(httpbin('cookies/set'), params=c1).json() - self.assertEqual(r['cookies'], c1) - s = requests.Session() - r = self.get(httpbin('cookies/set'), session=s, params=c1).json() - self.assertEqual(dict(s.cookies), c1) - - # ensure all cookies saved - c2 = {'k2': 'v2'} - c1.update(c2) - r = self.get(httpbin('cookies/set'), session=s, params=c2).json() - self.assertEqual(dict(s.cookies), c1) - - # ensure new session is created - r = self.get(httpbin('cookies')).json() - self.assertEqual(r['cookies'], {}) - - # cookies as param - c3 = {'p1': '42'} - r = self.get(httpbin('cookies'), cookies=c3).json() - self.assertEqual(r['cookies'], c3) - - def test_calling_request(self): - reqs = [grequests.request('POST', httpbin('post'), data={'p': i}) - for i in range(N)] - resp = grequests.map(reqs, size=N) - self.assertEqual([int(r.json()['form']['p']) for r in resp], list(range(N))) - - def test_stream_enabled(self): - r = grequests.map([grequests.get(httpbin('stream/10'))], - size=2, stream=True)[0] - self.assertFalse(r._content_consumed) - - def test_concurrency_with_delayed_url(self): - t = time.time() - n = 10 - reqs = [grequests.get(httpbin('delay/1')) for _ in range(n)] - grequests.map(reqs, size=n) - self.assertLess((time.time() - t), n) - - def test_map_timeout_no_exception_handler(self): - """ - compliance with existing 0.2.0 behaviour - """ - reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))] - responses = grequests.map(reqs) - self.assertIsNone(responses[0]) - self.assertTrue(responses[1].ok) - self.assertEqual(len(responses), 2) - - def test_map_timeout_exception_handler_no_return(self): - """ - ensure default behaviour for a handler that returns None - """ - def exception_handler(request, exception): - pass - reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))] - responses = grequests.map(reqs, exception_handler=exception_handler) - self.assertIsNone(responses[0]) - self.assertTrue(responses[1].ok) - self.assertEqual(len(responses), 2) - - def test_map_timeout_exception_handler_returns_exception(self): - """ - ensure returned value from exception handler is stuffed in the map result - """ - def exception_handler(request, exception): - return exception - reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))] - responses = grequests.map(reqs, exception_handler=exception_handler) - self.assertIsInstance(responses[0], Timeout) - self.assertTrue(responses[1].ok) - self.assertEqual(len(responses), 2) - - def test_imap_timeout_no_exception_handler(self): - """ - compliance with existing 0.2.0 behaviour - """ - reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)] - out = [] - try: - for r in grequests.imap(reqs): - out.append(r) - except Timeout: - pass - self.assertEquals(out, []) - - def test_imap_timeout_exception_handler_no_return(self): - """ - ensure imap-default behaviour for a handler that returns None - """ - def exception_handler(request, exception): - pass - reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)] - out = [] - for r in grequests.imap(reqs, exception_handler=exception_handler): - out.append(r) - self.assertEquals(out, []) - - - def test_imap_timeout_exception_handler_returns_value(self): - """ - ensure behaviour for a handler that returns a value - """ - def exception_handler(request, exception): - return 'a value' - reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)] - out = [] - for r in grequests.imap(reqs, exception_handler=exception_handler): - out.append(r) - self.assertEquals(out, ['a value']) - - def test_map_timeout_exception(self): - class ExceptionHandler: - def __init__(self): - self.counter = 0 - - def callback(self, request, exception): - self.counter += 1 - eh = ExceptionHandler() - reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)] - list(grequests.map(reqs, exception_handler=eh.callback)) - self.assertEqual(eh.counter, 1) - - def test_imap_timeout_exception(self): - class ExceptionHandler: - def __init__(self): - self.counter = 0 - - def callback(self, request, exception): - self.counter += 1 - eh = ExceptionHandler() - reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)] - list(grequests.imap(reqs, exception_handler=eh.callback)) - self.assertEqual(eh.counter, 1) - - def get(self, url, **kwargs): - return grequests.map([grequests.get(url, **kwargs)])[0] - - -if __name__ == '__main__': - unittest.main()