Commit 5c4c8aa081e9ada53e2590107fb1b004de26af39 - backoff

+71

-0

.github/workflows/codeql-analysis.yml less more

	0	# For most projects, this workflow file will not need changing; you simply need
	1	# to commit it to your repository.
	2	#
	3	# You may wish to alter this file to override the set of languages analyzed,
	4	# or to provide custom queries or build logic.
	5	name: "CodeQL"
	6
	7	on:
	8	push:
	9	branches: [master]
	10	pull_request:
	11	# The branches below must be a subset of the branches above
	12	branches: [master]
	13	schedule:
	14	- cron: '0 5 * * 6'
	15
	16	jobs:
	17	analyze:
	18	name: Analyze
	19	runs-on: ubuntu-latest
	20
	21	strategy:
	22	fail-fast: false
	23	matrix:
	24	# Override automatic language detection by changing the below list
	25	# Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
	26	language: ['python']
	27	# Learn more...
	28	# https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
	29
	30	steps:
	31	- name: Checkout repository
	32	uses: actions/checkout@v2
	33	with:
	34	# We must fetch at least the immediate parents so that if this is
	35	# a pull request then we can checkout the head.
	36	fetch-depth: 2
	37
	38	# If this run was triggered by a pull request event, then checkout
	39	# the head of the pull request instead of the merge commit.
	40	- run: git checkout HEAD^2
	41	if: ${{ github.event_name == 'pull_request' }}
	42
	43	# Initializes the CodeQL tools for scanning.
	44	- name: Initialize CodeQL
	45	uses: github/codeql-action/init@v1
	46	with:
	47	languages: ${{ matrix.language }}
	48	# If you wish to specify custom queries, you can do so here or in a config file.
	49	# By default, queries listed here will override any specified in a config file.
	50	# Prefix the list here with "+" to use these queries and those in the config file.
	51	# queries: ./path/to/local/query, your-org/your-repo/queries@main
	52
	53	# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
	54	# If this step fails, then you should remove it and run the build manually (see below)
	55	- name: Autobuild
	56	uses: github/codeql-action/autobuild@v1
	57
	58	# ℹ️ Command-line programs to run using the OS shell.
	59	# 📚 https://git.io/JvXDl
	60
	61	# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
	62	# and modify them (or add more) to build your code if your project
	63	# uses a compiled language
	64
	65	#- run: \|
	66	# make bootstrap
	67	# make release
	68
	69	- name: Perform CodeQL Analysis
	70	uses: github/codeql-action/analyze@v1

+3

-0

.travis.yml less more

4	4	- "3.6"
5	5	- "3.7"
6	6	- "3.8"
	7	- "3.9"
7	8	matrix:
8	9	include:
9	10	- python: "3.5"

14	15	env: PYTHONASYNCIODEBUG=x
15	16	- python: "3.8"
16	17	env: PYTHONASYNCIODEBUG=x
	18	- python: "3.9"
	19	env: PYTHONASYNCIODEBUG=x
17	20
18	21	before_install:
19	22	- pip install poetry more-itertools

+16

-1

CHANGELOG.md less more

141	141
142	142	- Don't include tests and changelog in distribution
143	143
144		## [v1.10.0] 2019-12-7
	144	## [v1.10.0] 2019-12-07
145	145	### Changed
146	146
147	147	- Allow sync decorator call from async function
	148
	149	## [v1.11.0] 2021-07-12
	150	### Changed
	151
	152	- Configurable logging levels for backoff and giveup events
	153	- Minor documentation fixes
	154
	155	### NOTE
	156
	157	THIS WILL BE THE FINAL PYTHON 2.7 COMPATIBLE RELEASE.
	158
	159	## [v1.11.1] 2021-07-14
	160	### Changed
	161
	162	- Update __version__ in backoff module

+2

-2

Makefile less more

13	13
14	14	flake8:
15	15	ifeq ($(PY_GTE_35),1)
16		@flake8 backoff tests
	16	@flake8 --ignore=E741,W503,W504 backoff tests
17	17	else
18		@flake8 --exclude tests/python35,backoff/_async.py backoff tests
	18	@flake8 --ignore=E741,W503,W504 --exclude tests/python35,backoff/_async.py backoff tests
19	19	endif
20	20
21	21	clean:

+6

-6

README.rst less more

228	228	.. code-block:: python
229	229
230	230	def backoff_hdlr(details):
231		print ("Backing off {wait:0.1f} seconds afters {tries} tries "
	231	print ("Backing off {wait:0.1f} seconds after {tries} tries "
232	232	"calling function {target} with args {args} and kwargs "
233	233	"{kwargs}".format(**details))
234	234

279	279
280	280	@backoff.on_exception(backoff.expo, aiohttp.ClientError, max_time=60)
281	281	async def get_url(url):
282		async with aiohttp.ClientSession() as session:
	282	async with aiohttp.ClientSession(raise_for_status=True) as session:
283	283	async with session.get(url) as response:
284	284	return await response.text()
285	285

311	311	.. code-block:: python
312	312
313	313	@backoff.on_exception(backoff.expo,
314		requests.exception.RequestException,
	314	requests.exceptions.RequestException,
315	315	logger='my_logger')
316	316	# ...
317	317

322	322
323	323	my_logger = logging.getLogger('my_logger')
324	324	my_handler = logging.StreamHandler()
325		my_logger.add_handler(my_handler)
	325	my_logger.addHandler(my_handler)
326	326	my_logger.setLevel(logging.ERROR)
327	327
328	328	@backoff.on_exception(backoff.expo,
329		requests.exception.RequestException,
330		logger=my_logger)
	329	requests.exceptions.RequestException,
	330	logger=my_logger)
331	331	# ...
332	332
333	333	Default logging can be disabled all together by specifying

+13

-2

backoff/__init__.py less more

11	11	For examples and full documentation see the README at
12	12	https://github.com/litl/backoff
13	13	"""
	14	import sys
	15	import warnings
	16
14	17	from backoff._decorator import on_predicate, on_exception
15	18	from backoff._jitter import full_jitter, random_jitter
16	19	from backoff._wait_gen import constant, expo, fibo

22	25	'expo',
23	26	'fibo',
24	27	'full_jitter',
25		'random_jitter'
	28	'random_jitter',
26	29	]
27	30
28		__version__ = '1.10.0'
	31	__version__ = '1.11.1'
	32
	33
	34	if sys.version_info[0] < 3:
	35	warnings.warn(
	36	"Python 2.7 support is deprecated and will be dropped "
	37	"in the next release",
	38	DeprecationWarning,
	39	) # pragma: no cover

+25

-7

backoff/_common.py less more

4	4	import sys
5	5	import traceback
6	6	import warnings
	7
	8
	9	# python 2.7 -> 3.x compatibility for str and unicode
	10	try:
	11	basestring
	12	except NameError: # pragma: python=3.5
	13	basestring = str
7	14
8	15
9	16	# Use module-specific logger with a default null handler.

40	47
41	48	seconds = value + jitter()
42	49
43		# don't sleep longer than remaining alloted max_time
	50	# don't sleep longer than remaining allotted max_time
44	51	if max_time is not None:
45	52	seconds = min(seconds, max_time - elapsed)
46	53
47	54	return seconds
48	55
49	56
	57	def _prepare_logger(logger):
	58	if isinstance(logger, basestring):
	59	logger = logging.getLogger(logger)
	60	return logger
	61
	62
50	63	# Configure handler list with user specified handler and optionally
51	64	# with a default handler bound to the specified logger.
52		def _config_handlers(user_handlers, default_handler=None, logger=None):
	65	def _config_handlers(
	66	user_handlers, default_handler=None, logger=None, log_level=None
	67	):
53	68	handlers = []
54	69	if logger is not None:
	70	assert log_level is not None, "Log level is not specified"
55	71	# bind the specified logger to the default log handler
56		log_handler = functools.partial(default_handler, logger=logger)
	72	log_handler = functools.partial(
	73	default_handler, logger=logger, log_level=log_level
	74	)
57	75	handlers.append(log_handler)
58	76
59	77	if user_handlers is None:

72	90
73	91
74	92	# Default backoff handler
75		def _log_backoff(details, logger):
	93	def _log_backoff(details, logger, log_level):
76	94	msg = "Backing off %s(...) for %.1fs (%s)"
77	95	log_args = [details['target'].__name__, details['wait']]
78	96

82	100	log_args.append(exc_fmt.rstrip("\n"))
83	101	else:
84	102	log_args.append(details['value'])
85		logger.info(msg, *log_args)
	103	logger.log(log_level, msg, *log_args)
86	104
87	105
88	106	# Default giveup handler
89		def _log_giveup(details, logger):
	107	def _log_giveup(details, logger, log_level):
90	108	msg = "Giving up %s(...) after %d tries (%s)"
91	109	log_args = [details['target'].__name__, details['tries']]
92	110

97	115	else:
98	116	log_args.append(details['value'])
99	117
100		logger.error(msg, *log_args)
	118	logger.log(log_level, msg, *log_args)

+31

-19

backoff/_decorator.py less more

4	4	import operator
5	5	import sys
6	6
7		from backoff._common import (_config_handlers, _log_backoff, _log_giveup)
	7	from backoff._common import (
	8	_prepare_logger,
	9	_config_handlers,
	10	_log_backoff,
	11	_log_giveup
	12	)
8	13	from backoff._jitter import full_jitter
9	14	from backoff import _sync
10
11
12		# python 2.7 -> 3.x compatibility for str and unicode
13		try:
14		basestring
15		except NameError: # pragma: python=3.5
16		basestring = str
17	15
18	16
19	17	def on_predicate(wait_gen,

25	23	on_backoff=None,
26	24	on_giveup=None,
27	25	logger='backoff',
	26	backoff_log_level=logging.INFO,
	27	giveup_log_level=logging.ERROR,
28	28	**wait_gen_kwargs):
29	29	"""Returns decorator for backoff and retry triggered by predicate.
30	30

62	62	about the invocation.
63	63	logger: Name of logger or Logger object to log to. Defaults to
64	64	'backoff'.
	65	backoff_log_level: log level for the backoff event. Defaults to "INFO"
	66	giveup_log_level: log level for the give up event. Defaults to "ERROR"
65	67	**wait_gen_kwargs: Any additional keyword args specified will be
66	68	passed to wait_gen when it is initialized. Any callable
67	69	args will first be evaluated and their return values passed.

69	71	"""
70	72	def decorate(target):
71	73	# change names because python 2.x doesn't have nonlocal
72		logger_ = logger
73		if isinstance(logger_, basestring):
74		logger_ = logging.getLogger(logger_)
	74	logger_ = _prepare_logger(logger)
	75
75	76	on_success_ = _config_handlers(on_success)
76		on_backoff_ = _config_handlers(on_backoff, _log_backoff, logger_)
77		on_giveup_ = _config_handlers(on_giveup, _log_giveup, logger_)
	77	on_backoff_ = _config_handlers(
	78	on_backoff, _log_backoff, logger_, backoff_log_level
	79	)
	80	on_giveup_ = _config_handlers(
	81	on_giveup, _log_giveup, logger_, giveup_log_level
	82	)
78	83
79	84	retry = None
80	85	if sys.version_info >= (3, 5): # pragma: python=3.5

106	111	on_backoff=None,
107	112	on_giveup=None,
108	113	logger='backoff',
	114	backoff_log_level=logging.INFO,
	115	giveup_log_level=logging.ERROR,
109	116	**wait_gen_kwargs):
110	117	"""Returns decorator for backoff and retry triggered by exception.
111	118

116	123	backoff.
117	124	max_tries: The maximum number of attempts to make before giving
118	125	up. Once exhausted, the exception will be allowed to escape.
119		The default value of None means their is no limit to the
	126	The default value of None means there is no limit to the
120	127	number of tries. If a callable is passed, it will be
121	128	evaluated at runtime and its return value used.
122	129	max_time: The maximum total amount of time to try for before

143	150	is exceeded. The parameter is a dict containing details
144	151	about the invocation.
145	152	logger: Name or Logger object to log to. Defaults to 'backoff'.
	153	backoff_log_level: log level for the backoff event. Defaults to "INFO"
	154	giveup_log_level: log level for the give up event. Defaults to "ERROR"
146	155	**wait_gen_kwargs: Any additional keyword args specified will be
147	156	passed to wait_gen when it is initialized. Any callable
148	157	args will first be evaluated and their return values passed.

150	159	"""
151	160	def decorate(target):
152	161	# change names because python 2.x doesn't have nonlocal
153		logger_ = logger
154		if isinstance(logger_, basestring):
155		logger_ = logging.getLogger(logger_)
	162	logger_ = _prepare_logger(logger)
	163
156	164	on_success_ = _config_handlers(on_success)
157		on_backoff_ = _config_handlers(on_backoff, _log_backoff, logger_)
158		on_giveup_ = _config_handlers(on_giveup, _log_giveup, logger_)
	165	on_backoff_ = _config_handlers(
	166	on_backoff, _log_backoff, logger_, backoff_log_level
	167	)
	168	on_giveup_ = _config_handlers(
	169	on_giveup, _log_giveup, logger_, giveup_log_level
	170	)
159	171
160	172	retry = None
161	173	if sys.version_info[:2] >= (3, 5): # pragma: python=3.5

+1

-1

backoff/_wait_gen.py less more

7	7
8	8	Args:
9	9	base: The mathematical base of the exponentiation operation
10		factor: Factor to multiply the exponentation by.
	10	factor: Factor to multiply the exponentiation by.
11	11	max_value: The maximum value to yield. Once the value in the
12	12	true exponential sequence exceeds this, the value
13	13	of max_value will forever after be yielded.

+7

-5

pyproject.toml less more

0	0	[tool.poetry]
1	1	name = "backoff"
2		version = "1.10.0"
	2	version = "1.11.1"
3	3	description = "Function decoration for backoff and retry"
4		authors = ["Bob Green <[email protected]>"]
	4	authors = ["Bob Green <[email protected]>"]
5	5	readme = "README.rst"
6	6	repository = "https://github.com/litl/backoff"
7	7	license = "MIT"

20	20	'Programming Language :: Python :: 3.6',
21	21	'Programming Language :: Python :: 3.7',
22	22	'Programming Language :: Python :: 3.8',
23		'Topic :: Internet :: WWW/HTTP',
	23	'Programming Language :: Python :: 3.9',
	24	'Topic :: Internet :: WWW/HTTP',
24	25	'Topic :: Software Development :: Libraries :: Python Modules',
25	26	'Topic :: Utilities']
26	27	packages = [

35	36	pytest = "^4.0"
36	37	pytest-cov = "^2.6"
37	38	pytest-asyncio = {version = "^0.10.0",python = "^3.5"}
	39	autopep8 = "^1.5.7"
38	40
39	41	[build-system]
40		requires = ["poetry>=0.12"]
41		build-backend = "poetry.masonry.api"
	42	requires = ["poetry-core"]
	43	build-backend = "poetry.core.masonry.api"

+82

-0

tests/test_backoff.py less more

0	0	# coding:utf-8
1	1	import datetime
	2	import itertools
2	3	import logging
3	4	import random
	5	import re
4	6	import sys
5	7	import threading
	8	import unittest.mock
6	9
7	10	import pytest
8	11

732	735	assert len(caplog.records) == 3 # 2 backoffs and 1 giveup
733	736	for record in caplog.records:
734	737	assert record.name == 'my-logger'
	738
	739
	740	def _on_exception_factory(
	741	backoff_log_level, giveup_log_level, max_tries
	742	):
	743	@backoff.on_exception(
	744	backoff.expo,
	745	ValueError,
	746	max_tries=max_tries,
	747	backoff_log_level=backoff_log_level,
	748	giveup_log_level=giveup_log_level,
	749	)
	750	def value_error():
	751	raise ValueError
	752
	753	def func():
	754	with pytest.raises(ValueError):
	755	value_error()
	756
	757	return func
	758
	759
	760	def _on_predicate_factory(
	761	backoff_log_level, giveup_log_level, max_tries
	762	):
	763	@backoff.on_predicate(
	764	backoff.expo,
	765	max_tries=max_tries,
	766	backoff_log_level=backoff_log_level,
	767	giveup_log_level=giveup_log_level,
	768	)
	769	def func():
	770	return False
	771
	772	return func
	773
	774
	775	@pytest.mark.parametrize(
	776	("func_factory", "backoff_log_level", "giveup_log_level"),
	777	(
	778	(factory, backoff_log_level, giveup_log_level)
	779	for backoff_log_level, giveup_log_level in itertools.product(
	780	(
	781	logging.DEBUG,
	782	logging.INFO,
	783	logging.WARNING,
	784	logging.ERROR,
	785	logging.CRITICAL,
	786	),
	787	repeat=2,
	788	)
	789	for factory in (_on_predicate_factory, _on_exception_factory)
	790	)
	791	)
	792	def test_event_log_levels(
	793	caplog, func_factory, backoff_log_level, giveup_log_level
	794	):
	795	max_tries = 3
	796	func = func_factory(backoff_log_level, giveup_log_level, max_tries)
	797
	798	with unittest.mock.patch('time.sleep', return_value=None):
	799	with caplog.at_level(
	800	min(backoff_log_level, giveup_log_level), logger="backoff"
	801	):
	802	func()
	803
	804	backoff_re = re.compile("backing off", re.IGNORECASE)
	805	giveup_re = re.compile("giving up", re.IGNORECASE)
	806
	807	backoff_log_count = 0
	808	giveup_log_count = 0
	809	for logger_name, level, message in caplog.record_tuples:
	810	if level == backoff_log_level and backoff_re.match(message):
	811	backoff_log_count += 1
	812	elif level == giveup_log_level and giveup_re.match(message):
	813	giveup_log_count += 1
	814
	815	assert backoff_log_count == max_tries - 1
	816	assert giveup_log_count == 1