Commit 5d1c933be62bef98ec68d51fc502f90948205de3 - python-grequests

New upstream version 0.4.0 Sophie Brun 4 years ago

9 changed file(s) with 574 addition(s) and 0 deletion(s). Raw diff Collapse all Expand all

+14

-0

.gitignore less more

	0	.coverage
	1	MANIFEST
	2	coverage.xml
	3	nosetests.xml
	4	junit-report.xml
	5	pylint.txt
	6	toy.py
	7	violations.pyflakes.txt
	8	cover/
	9	docs/_build
	10	grequests.egg-info/
	11	*.py[cx]
	12	*.swp
	13	env/

+27

-0

.travis.yml less more

	0	dist: xenial
	1	language: python
	2
	3
	4	python:
	5	- "2.7"
	6	- "3.6"
	7	- "3.7"
	8
	9
	10	install:
	11	- pip install -r requirements.txt
	12	- pip install pytest
	13
	14	script:
	15	- pytest tests.py
	16
	17
	18	deploy:
	19	provider: pypi
	20	distributions: "sdist bdist_wheel"
	21	user: "spyoungtech"
	22	password:
	23	secure: "QtuuH0X/A/iQI23MxvqsnxUy63XD5awJHDkeQNmUDIGGQqIox2DTYKoc6x354I5wpqprtODQRYRqIsA9+2cpRcF49Ft50cvi3cmuoeozkID3ybQyLHCIcJ4CKt6X+h2LFbrgqyyBcny7tKQlYr4/nsjeQegPblnJ6OTljJgJyE0="
	24	on:
	25	tags: true
	26	python: 3.6⏎

+12

-0

AUTHORS.rst less more

	0	GRequests is written and maintained by Kenneth Reitz and
	1	various contributors:
	2
	3	Development Lead
	4	````````````````
	5
	6	- Kenneth Reitz <[email protected]>
	7
	8	Patches and Suggestions
	9	```````````````````````
	10	- Kracekumar <[email protected]>
	11	- Spencer Young <[email protected]>

-0

LICENSE less more

	0	Copyright (c) 2012, Kenneth Reitz
	1	All rights reserved.
	2
	3	Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
	4
	5	Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
	6	Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
	7	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.⏎

+67

-0

README.rst less more

	0	GRequests: Asynchronous Requests
	1	===============================
	2
	3	GRequests allows you to use Requests with Gevent to make asynchronous HTTP
	4	Requests easily.
	5
	6	Note: You should probably use `requests-threads <https://github.com/requests/requests-threads>`_ or `requests-futures <https://github.com/ross/requests-futures>`_ instead.
	7
	8
	9	Usage
	10	-----
	11
	12	Usage is simple:
	13
	14	.. code-block:: python
	15
	16	import grequests
	17
	18	urls = [
	19	'http://www.heroku.com',
	20	'http://python-tablib.org',
	21	'http://httpbin.org',
	22	'http://python-requests.org',
	23	'http://fakedomain/',
	24	'http://kennethreitz.com'
	25	]
	26
	27	Create a set of unsent Requests:
	28
	29	.. code-block:: python
	30
	31	>>> rs = (grequests.get(u) for u in urls)
	32
	33	Send them all at the same time:
	34
	35	.. code-block:: python
	36
	37	>>> grequests.map(rs)
	38	[<Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>, None, <Response [200]>]
	39
	40	Optionally, in the event of a timeout or any other exception during the connection of
	41	the request, you can add an exception handler that will be called with the request and
	42	exception inside the main thread:
	43
	44	.. code-block:: python
	45
	46	>>> def exception_handler(request, exception):
	47	... print "Request failed"
	48
	49	>>> reqs = [
	50	... grequests.get('http://httpbin.org/delay/1', timeout=0.001),
	51	... grequests.get('http://fakedomain/'),
	52	... grequests.get('http://httpbin.org/status/500')]
	53	>>> grequests.map(reqs, exception_handler=exception_handler)
	54	Request failed
	55	Request failed
	56	[None, None, <Response [500]>]
	57
	58	For some speed/performance gains, you may also want to use `imap` instead of `map`. `imap` returns a generator of responses. Order of these responses does not map to the order of the requests you send out. The API for `imap` is equivalent to the API for `map`.
	59
	60	Installation
	61	------------
	62
	63	Installation is easy with pip::
	64
	65	$ pip install grequests
	66	✨🍰✨

+155

-0

grequests.py less more

	0	# -- coding: utf-8 --
	1
	2	"""
	3	grequests
	4	~~~~~~~~~
	5
	6	This module contains an asynchronous replica of ``requests.api``, powered
	7	by gevent. All API methods return a ``Request`` instance (as opposed to
	8	``Response``). A list of requests can be sent with ``map()``.
	9	"""
	10	from functools import partial
	11	import traceback
	12	try:
	13	import gevent
	14	from gevent import monkey as curious_george
	15	from gevent.pool import Pool
	16	except ImportError:
	17	raise RuntimeError('Gevent is required for grequests.')
	18
	19	# Monkey-patch.
	20	curious_george.patch_all(thread=False, select=False)
	21
	22	from requests import Session
	23
	24
	25	__all__ = (
	26	'map', 'imap',
	27	'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request'
	28	)
	29
	30
	31	class AsyncRequest(object):
	32	""" Asynchronous request.
	33
	34	Accept same parameters as ``Session.request`` and some additional:
	35
	36	:param session: Session which will do request
	37	:param callback: Callback called on response.
	38	Same as passing ``hooks={'response': callback}``
	39	"""
	40	def __init__(self, method, url, **kwargs):
	41	#: Request method
	42	self.method = method
	43	#: URL to request
	44	self.url = url
	45	#: Associated ``Session``
	46	self.session = kwargs.pop('session', None)
	47	if self.session is None:
	48	self.session = Session()
	49
	50	callback = kwargs.pop('callback', None)
	51	if callback:
	52	kwargs['hooks'] = {'response': callback}
	53
	54	#: The rest arguments for ``Session.request``
	55	self.kwargs = kwargs
	56	#: Resulting ``Response``
	57	self.response = None
	58
	59	def send(self, **kwargs):
	60	"""
	61	Prepares request based on parameter passed to constructor and optional ``kwargs```.
	62	Then sends request and saves response to :attr:`response`
	63
	64	:returns: ``Response``
	65	"""
	66	merged_kwargs = {}
	67	merged_kwargs.update(self.kwargs)
	68	merged_kwargs.update(kwargs)
	69	try:
	70	self.response = self.session.request(self.method,
	71	self.url, **merged_kwargs)
	72	except Exception as e:
	73	self.exception = e
	74	self.traceback = traceback.format_exc()
	75	return self
	76
	77
	78	def send(r, pool=None, stream=False):
	79	"""Sends the request object using the specified pool. If a pool isn't
	80	specified this method blocks. Pools are useful because you can specify size
	81	and can hence limit concurrency."""
	82	if pool is not None:
	83	return pool.spawn(r.send, stream=stream)
	84
	85	return gevent.spawn(r.send, stream=stream)
	86
	87
	88	# Shortcuts for creating AsyncRequest with appropriate HTTP method
	89	get = partial(AsyncRequest, 'GET')
	90	options = partial(AsyncRequest, 'OPTIONS')
	91	head = partial(AsyncRequest, 'HEAD')
	92	post = partial(AsyncRequest, 'POST')
	93	put = partial(AsyncRequest, 'PUT')
	94	patch = partial(AsyncRequest, 'PATCH')
	95	delete = partial(AsyncRequest, 'DELETE')
	96
	97	# synonym
	98	def request(method, url, **kwargs):
	99	return AsyncRequest(method, url, **kwargs)
	100
	101
	102	def map(requests, stream=False, size=None, exception_handler=None, gtimeout=None):
	103	"""Concurrently converts a list of Requests to Responses.
	104
	105	:param requests: a collection of Request objects.
	106	:param stream: If True, the content will not be downloaded immediately.
	107	:param size: Specifies the number of requests to make at a time. If None, no throttling occurs.
	108	:param exception_handler: Callback function, called when exception occured. Params: Request, Exception
	109	:param gtimeout: Gevent joinall timeout in seconds. (Note: unrelated to requests timeout)
	110	"""
	111
	112	requests = list(requests)
	113
	114	pool = Pool(size) if size else None
	115	jobs = [send(r, pool, stream=stream) for r in requests]
	116	gevent.joinall(jobs, timeout=gtimeout)
	117
	118	ret = []
	119
	120	for request in requests:
	121	if request.response is not None:
	122	ret.append(request.response)
	123	elif exception_handler and hasattr(request, 'exception'):
	124	ret.append(exception_handler(request, request.exception))
	125	else:
	126	ret.append(None)
	127
	128	return ret
	129
	130
	131	def imap(requests, stream=False, size=2, exception_handler=None):
	132	"""Concurrently converts a generator object of Requests to
	133	a generator of Responses.
	134
	135	:param requests: a generator of Request objects.
	136	:param stream: If True, the content will not be downloaded immediately.
	137	:param size: Specifies the number of requests to make at a time. default is 2
	138	:param exception_handler: Callback function, called when exception occurred. Params: Request, Exception
	139	"""
	140
	141	pool = Pool(size)
	142
	143	def send(r):
	144	return r.send(stream=stream)
	145
	146	for request in pool.imap_unordered(send, requests):
	147	if request.response is not None:
	148	yield request.response
	149	elif exception_handler:
	150	ex_result = exception_handler(request, request.exception)
	151	if ex_result is not None:
	152	yield ex_result
	153
	154	pool.join()

-0

requirements.txt less more

	0	requests
	1	gevent
	2	nose

+64

-0

setup.py less more

	0	# -- coding: utf-8 --
	1	"""
	2	GRequests allows you to use Requests with Gevent to make asynchronous HTTP
	3	Requests easily.
	4
	5	Usage
	6	-----
	7
	8	Usage is simple::
	9
	10	import grequests
	11
	12	urls = [
	13	'http://www.heroku.com',
	14	'http://tablib.org',
	15	'http://httpbin.org',
	16	'http://python-requests.org',
	17	'http://kennethreitz.com'
	18	]
	19
	20	Create a set of unsent Requests::
	21
	22	>>> rs = (grequests.get(u) for u in urls)
	23
	24	Send them all at the same time::
	25
	26	>>> grequests.map(rs)
	27	[<Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>]
	28
	29	"""
	30
	31	from setuptools import setup
	32
	33	setup(
	34	name='grequests',
	35	version='0.4.0',
	36	url='https://github.com/kennethreitz/grequests',
	37	license='BSD',
	38	author='Kenneth Reitz',
	39	author_email='[email protected]',
	40	description='Requests + Gevent',
	41	long_description=__doc__,
	42	install_requires=[
	43	'gevent',
	44	'requests'
	45	],
	46	tests_require = ['nose'],
	47	test_suite = 'nose.collector',
	48	py_modules=['grequests'],
	49	zip_safe=False,
	50	include_package_data=True,
	51	platforms='any',
	52	classifiers=[
	53	'Environment :: Web Environment',
	54	'Intended Audience :: Developers',
	55	'License :: OSI Approved :: BSD License',
	56	'Operating System :: OS Independent',
	57	'Programming Language :: Python',
	58	'Programming Language :: Python :: 2.7',
	59	'Programming Language :: Python :: 3',
	60	'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
	61	'Topic :: Software Development :: Libraries :: Python Modules'
	62	]
	63	)

+224

-0

tests.py less more

	0	#! /usr/bin/env python
	1	# -- coding: utf-8 --
	2
	3	from grequests import get, map, imap
	4	from nose.tools import ok_
	5
	6	########### Constants ############
	7	urls = [
	8	'http://github.com',
	9	'http://www.google.com',
	10	'http://www.psf.org'
	11	]
	12	############# tests ##############
	13	def test_get():
	14	global urls
	15	to_fetch = (get(url) for url in urls)
	16	map(to_fetch)
	17	for fetched in to_fetch:
	18	ok_(fetched.ok, True)
	19
	20	def test_imap_with_size():
	21	global urls
	22	to_fetch = (get(url) for url in urls)
	23	imap(to_fetch, size = len(urls) - 1)
	24	for fetching in to_fetch:
	25	ok_(fetching.send(), True)
	26
	27	import os
	28	import time
	29	import unittest
	30
	31	import requests
	32	from requests.exceptions import Timeout
	33	import grequests
	34
	35	HTTPBIN_URL = os.environ.get('HTTPBIN_URL', 'http://httpbin.org/')
	36
	37	def httpbin(*suffix):
	38	"""Returns url for HTTPBIN resource."""
	39	return HTTPBIN_URL + '/'.join(suffix)
	40
	41
	42	N = 5
	43	URLS = [httpbin('get?p=%s' % i) for i in range(N)]
	44
	45
	46	class GrequestsCase(unittest.TestCase):
	47
	48	def test_map(self):
	49	reqs = [grequests.get(url) for url in URLS]
	50	resp = grequests.map(reqs, size=N)
	51	self.assertEqual([r.url for r in resp], URLS)
	52
	53	def test_imap(self):
	54	reqs = (grequests.get(url) for url in URLS)
	55	i = 0
	56	for i, r in enumerate(grequests.imap(reqs, size=N)):
	57	self.assertTrue(r.url in URLS)
	58	self.assertEqual(i, N - 1)
	59
	60	def test_hooks(self):
	61	result = {}
	62
	63	def hook(r, **kwargs):
	64	result[r.url] = True
	65	return r
	66
	67	reqs = [grequests.get(url, hooks={'response': [hook]}) for url in URLS]
	68	grequests.map(reqs, size=N)
	69	self.assertEqual(sorted(result.keys()), sorted(URLS))
	70
	71	def test_callback_kwarg(self):
	72	result = {'ok': False}
	73
	74	def callback(r, **kwargs):
	75	result['ok'] = True
	76	return r
	77
	78	self.get(URLS[0], callback=callback)
	79	self.assertTrue(result['ok'])
	80
	81	def test_session_and_cookies(self):
	82	c1 = {'k1': 'v1'}
	83	r = self.get(httpbin('cookies/set'), params=c1).json()
	84	self.assertEqual(r['cookies'], c1)
	85	s = requests.Session()
	86	r = self.get(httpbin('cookies/set'), session=s, params=c1).json()
	87	self.assertEqual(dict(s.cookies), c1)
	88
	89	# ensure all cookies saved
	90	c2 = {'k2': 'v2'}
	91	c1.update(c2)
	92	r = self.get(httpbin('cookies/set'), session=s, params=c2).json()
	93	self.assertEqual(dict(s.cookies), c1)
	94
	95	# ensure new session is created
	96	r = self.get(httpbin('cookies')).json()
	97	self.assertEqual(r['cookies'], {})
	98
	99	# cookies as param
	100	c3 = {'p1': '42'}
	101	r = self.get(httpbin('cookies'), cookies=c3).json()
	102	self.assertEqual(r['cookies'], c3)
	103
	104	def test_calling_request(self):
	105	reqs = [grequests.request('POST', httpbin('post'), data={'p': i})
	106	for i in range(N)]
	107	resp = grequests.map(reqs, size=N)
	108	self.assertEqual([int(r.json()['form']['p']) for r in resp], list(range(N)))
	109
	110	def test_stream_enabled(self):
	111	r = grequests.map([grequests.get(httpbin('stream/10'))],
	112	size=2, stream=True)[0]
	113	self.assertFalse(r._content_consumed)
	114
	115	def test_concurrency_with_delayed_url(self):
	116	t = time.time()
	117	n = 10
	118	reqs = [grequests.get(httpbin('delay/1')) for _ in range(n)]
	119	grequests.map(reqs, size=n)
	120	self.assertLess((time.time() - t), n)
	121
	122	def test_map_timeout_no_exception_handler(self):
	123	"""
	124	compliance with existing 0.2.0 behaviour
	125	"""
	126	reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
	127	responses = grequests.map(reqs)
	128	self.assertIsNone(responses[0])
	129	self.assertTrue(responses[1].ok)
	130	self.assertEqual(len(responses), 2)
	131
	132	def test_map_timeout_exception_handler_no_return(self):
	133	"""
	134	ensure default behaviour for a handler that returns None
	135	"""
	136	def exception_handler(request, exception):
	137	pass
	138	reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
	139	responses = grequests.map(reqs, exception_handler=exception_handler)
	140	self.assertIsNone(responses[0])
	141	self.assertTrue(responses[1].ok)
	142	self.assertEqual(len(responses), 2)
	143
	144	def test_map_timeout_exception_handler_returns_exception(self):
	145	"""
	146	ensure returned value from exception handler is stuffed in the map result
	147	"""
	148	def exception_handler(request, exception):
	149	return exception
	150	reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
	151	responses = grequests.map(reqs, exception_handler=exception_handler)
	152	self.assertIsInstance(responses[0], Timeout)
	153	self.assertTrue(responses[1].ok)
	154	self.assertEqual(len(responses), 2)
	155
	156	def test_imap_timeout_no_exception_handler(self):
	157	"""
	158	compliance with existing 0.2.0 behaviour
	159	"""
	160	reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)]
	161	out = []
	162	try:
	163	for r in grequests.imap(reqs):
	164	out.append(r)
	165	except Timeout:
	166	pass
	167	self.assertEquals(out, [])
	168
	169	def test_imap_timeout_exception_handler_no_return(self):
	170	"""
	171	ensure imap-default behaviour for a handler that returns None
	172	"""
	173	def exception_handler(request, exception):
	174	pass
	175	reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)]
	176	out = []
	177	for r in grequests.imap(reqs, exception_handler=exception_handler):
	178	out.append(r)
	179	self.assertEquals(out, [])
	180
	181
	182	def test_imap_timeout_exception_handler_returns_value(self):
	183	"""
	184	ensure behaviour for a handler that returns a value
	185	"""
	186	def exception_handler(request, exception):
	187	return 'a value'
	188	reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)]
	189	out = []
	190	for r in grequests.imap(reqs, exception_handler=exception_handler):
	191	out.append(r)
	192	self.assertEquals(out, ['a value'])
	193
	194	def test_map_timeout_exception(self):
	195	class ExceptionHandler:
	196	def __init__(self):
	197	self.counter = 0
	198
	199	def callback(self, request, exception):
	200	self.counter += 1
	201	eh = ExceptionHandler()
	202	reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)]
	203	list(grequests.map(reqs, exception_handler=eh.callback))
	204	self.assertEqual(eh.counter, 1)
	205
	206	def test_imap_timeout_exception(self):
	207	class ExceptionHandler:
	208	def __init__(self):
	209	self.counter = 0
	210
	211	def callback(self, request, exception):
	212	self.counter += 1
	213	eh = ExceptionHandler()
	214	reqs = [grequests.get(httpbin('delay/1'), timeout=0.001)]
	215	list(grequests.imap(reqs, exception_handler=eh.callback))
	216	self.assertEqual(eh.counter, 1)
	217
	218	def get(self, url, **kwargs):
	219	return grequests.map([grequests.get(url, **kwargs)])[0]
	220
	221
	222	if __name__ == '__main__':
	223	unittest.main()