Commit upstream/3.17.1 - python-faraday

New upstream version 3.17.1 Sophie Brun 2 years ago

7 changed file(s) with 57 addition(s) and 54 deletion(s). Raw diff Collapse all Expand all

-0

CHANGELOG/3.17.1/community.md less more

* FIX bug when starting the server, creates a pool for reporting that breaks.

-0

CHANGELOG/3.17.1/date.md less more

Aug 20th, 2021

-0

CHANGELOG/RELEASE.md less more

0	0	New features in the latest update
1	1	=====================================
2	2
	3
	4	3.17.1 [Aug 20th, 2021]:
	5	---
	6	* FIX bug when starting the server, creates a pool for reporting that breaks.
3	7
4	8	3.17.0 [Aug 10th, 2021]:
5	9	---

-0

RELEASE.md less more

0	0	New features in the latest update
1	1	=====================================
2	2
	3
	4	3.17.1 [Aug 20th, 2021]:
	5	---
	6	* FIX bug when starting the server, creates a pool for reporting that breaks.
3	7
4	8	3.17.0 [Aug 10th, 2021]:
5	9	---

-1

faraday/__init__.py less more

1	1	# Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/)
2	2	# See the file 'doc/LICENSE' for the license information
3	3
4		__version__ = '3.17.0'
	4	__version__ = '3.17.1'
5	5	__license_version__ = __version__

-1

faraday/server/config.py less more

138	138	self.agent_registration_secret = None
139	139	self.agent_token_expiration = 60 # Default as 1 min
140	140	self.debug = False
141		self.reports_pool_size = 1
142	141	self.delete_report_after_process = True
143	142
144	143

+46

-52

faraday/server/threads/reports_processor.py less more

5	5	from queue import Queue, Empty
6	6	from typing import Tuple, Optional
7	7	import json
8		import multiprocessing
9	8
10	9	from faraday_plugins.plugins.manager import PluginsManager
11	10	from faraday.server.api.modules.bulk_create import bulk_create, BulkCreateSchema

25	24	def send_report_data(workspace_name: str, command_id: int, report_json: dict,
26	25	user_id: Optional[int], set_end_date: bool):
27	26	logger.info("Send Report data to workspace [%s]", workspace_name)
28		from faraday.server.web import get_app # pylint:disable=import-outside-toplevel
29		with get_app().app_context():
30		ws = Workspace.query.filter_by(name=workspace_name).one()
31		command = Command.query.filter_by(id=command_id).one()
32		schema = BulkCreateSchema()
33		data = schema.load(report_json)
34		if user_id:
35		user = User.query.filter_by(id=user_id).one()
36		data = add_creator(data, user)
37		bulk_create(ws, command, data, True, set_end_date)
	27	ws = Workspace.query.filter_by(name=workspace_name).one()
	28	command = Command.query.filter_by(id=command_id).one()
	29	schema = BulkCreateSchema()
	30	data = schema.load(report_json)
	31	if user_id:
	32	user = User.query.filter_by(id=user_id).one()
	33	data = add_creator(data, user)
	34	bulk_create(ws, command, data, True, set_end_date)
38	35
39	36
40	37	def process_report(workspace_name: str, command_id: int, file_path: Path,
41	38	plugin_id: Optional[int], user_id: Optional[int]):
42		if plugin_id is not None:
43		plugins_manager = PluginsManager(ReportsSettings.settings.custom_plugins_folder,
44		ignore_info=ReportsSettings.settings.ignore_info_severity)
45		logger.info(f"Reports Manager: [Custom plugins folder: "
46		f"[{ReportsSettings.settings.custom_plugins_folder}]"
47		f"[Ignore info severity: {ReportsSettings.settings.ignore_info_severity}]")
48		plugin = plugins_manager.get_plugin(plugin_id)
49		if plugin:
	39	from faraday.server.web import get_app # pylint:disable=import-outside-toplevel
	40	with get_app().app_context():
	41	if plugin_id is not None:
	42	plugins_manager = PluginsManager(ReportsSettings.settings.custom_plugins_folder,
	43	ignore_info=ReportsSettings.settings.ignore_info_severity)
	44	logger.info(f"Reports Manager: [Custom plugins folder: "
	45	f"[{ReportsSettings.settings.custom_plugins_folder}]"
	46	f"[Ignore info severity: {ReportsSettings.settings.ignore_info_severity}]")
	47	plugin = plugins_manager.get_plugin(plugin_id)
	48	if plugin:
	49	try:
	50	logger.info(f"Processing report [{file_path}] with plugin ["
	51	f"{plugin.id}]")
	52	plugin.processReport(str(file_path))
	53	vulns_data = plugin.get_data()
	54	del vulns_data['command']['duration']
	55	except Exception as e:
	56	logger.error("Processing Error: %s", e)
	57	logger.exception(e)
	58	return
	59	else:
	60	logger.error(f"No plugin detected for report [{file_path}]")
	61	return
	62	else:
50	63	try:
51		logger.info(f"Processing report [{file_path}] with plugin ["
52		f"{plugin.id}]")
53		plugin.processReport(str(file_path))
54		vulns_data = plugin.get_data()
55		del vulns_data['command']['duration']
	64	with file_path.open("r") as f:
	65	vulns_data = json.load(f)
56	66	except Exception as e:
57		logger.error("Processing Error: %s", e)
	67	logger.error("Loading data from json file: %s [%s]", file_path, e)
58	68	logger.exception(e)
59	69	return
	70	if plugin_id is None:
	71	logger.debug("Removing file: %s", file_path)
	72	os.remove(file_path)
60	73	else:
61		logger.error(f"No plugin detected for report [{file_path}]")
62		return
63		else:
	74	if faraday_server.delete_report_after_process:
	75	os.remove(file_path)
	76	set_end_date = True
64	77	try:
65		with file_path.open("r") as f:
66		vulns_data = json.load(f)
	78	send_report_data(workspace_name, command_id, vulns_data, user_id, set_end_date)
	79	logger.info("Report processing finished")
67	80	except Exception as e:
68		logger.error("Loading data from json file: %s [%s]", file_path, e)
69	81	logger.exception(e)
70		return
71		if plugin_id is None:
72		logger.debug("Removing file: %s", file_path)
73		os.remove(file_path)
74		else:
75		if faraday_server.delete_report_after_process:
76		os.remove(file_path)
77		set_end_date = True
78		try:
79		send_report_data(workspace_name, command_id, vulns_data, user_id, set_end_date)
80		logger.info("Report processing finished")
81		except Exception as e:
82		logger.exception(e)
83		logger.error("Save Error: %s", e)
	82	logger.error("Save Error: %s", e)
84	83
85	84
86	85	class ReportsManager(Thread):

89	88	super().__init__(name="ReportsManager-Thread", daemon=True, args, *kwargs)
90	89	self.upload_reports_queue = upload_reports_queue
91	90	self.__event = threading.Event()
92		self.processing_pool = multiprocessing.Pool(processes=faraday_server.reports_pool_size)
93	91
94	92	def stop(self):
95	93	logger.info("Reports Manager Thread [Stopping...]")
96	94	self.__event.set()
97	95
98	96	def run(self):
99		logger.info(f"Reports Manager Thread [Start] with Pool Size: {faraday_server.reports_pool_size}")
	97	logger.info("Reports Manager Thread [Start]")
100	98	while not self.__event.is_set():
101	99	try:
102	100	tpl: Tuple[str, int, Path, int, int] = \

106	104
107	105	logger.info(f"Processing raw report {file_path}")
108	106	if file_path.is_file():
109		self.processing_pool.apply_async(process_report,
110		(workspace_name, command_id, file_path, plugin_id, user_id))
	107	process_report(workspace_name, command_id, file_path, plugin_id, user_id)
111	108	else:
112	109	logger.warning(f"Report file [{file_path}] don't exists",
113	110	file_path)

121	118	continue
122	119	else:
123	120	logger.info("Reports Manager Thread [Stop]")
124		self.processing_pool.close()
125		self.processing_pool.terminate()
126		self.processing_pool.join()