Commit a68733b3fb53a90b58bbdabf10f541dbfdf5df76 - python-faraday

+16

-7

.gitlab-ci.yml less more

151	151	- nix-build '<nixpkgs>' -A gtk3 -o result-gtk
152	152	- tar cf /py3.tar $(nix-store --query --requisites $(readlink result) $(readlink result-vte)) $(readlink result-gtk)
153	153	- mkdir /opt
154		- nix-env -i findutils postgresql
	154	- nix-env -i findutils
155	155	- "nix-shell -p python37Packages.virtualenv --command 'virtualenv /opt/faraday'"
	156	# - cp result /opt/faraday/lib/python3.7/site-packages/nix.pth # This doesn't include dependencies of dependencies
	157	- 'for dir in $(nix-store -qR result); do if [[ -d "$dir/lib/python3.7/site-packages" ]]; then echo "$dir/lib/python3.7/site-packages" >>recursive.pth; fi; done'
	158	- mv recursive.pth /opt/faraday/lib/python3.7/site-packages/nix.pth
156	159	- source /opt/faraday/bin/activate
157		- "nix-shell -p libxml2 libxslt zlib cairo gobject-introspection glib pkgconfig --run 'SOURCE_DATE_EPOCH=$(date +%s) /opt/faraday/bin/pip install -r requirements.txt'"
	160	# - "nix-shell -p libxml2 libxslt zlib cairo gobject-introspection glib pkgconfig --run 'SOURCE_DATE_EPOCH=$(date +%s) /opt/faraday/bin/pip install -r requirements.txt'"
158	161	- python setup.py install
159		- cp result /opt/faraday/lib/python3.7/site-packages/nix.pth
	162	- pip install -r requirements.txt # This has to be below setup.py install
	163	- pip freeze
160	164	- mv /opt/faraday/bin/faraday-client /opt/faraday/bin/_faraday_client_novte
161	165	- 'nix-shell -p makeWrapper --command "makeWrapper /opt/faraday/bin/_faraday_client_novte /opt/faraday/bin/faraday-client --prefix GI_TYPELIB_PATH : $(find /nix/store -name "girepository-1.0" \| tr "\n" ":")"'
162	166	- 'nix-shell -p makeWrapper --command "makeWrapper /opt/faraday/bin/python /opt/faraday/bin/test --prefix GI_TYPELIB_PATH : $(find /nix/store -name "girepository-1.0" \| tr "\n" ":")"'
163		- /opt/faraday/bin/test -c "import gi;gi.require_version('Gtk', '3.0');gi.require_version('Vte', '2.91');from gi.repository import Gio, Gtk, GdkPixbuf, Vte, GLib, GObject, Gdk" # Test if GTK will work
	167	- nix-collect-garbage && /opt/faraday/bin/test -c "import gi;gi.require_version('Gtk', '3.0');gi.require_version('Vte', '2.91');from gi.repository import Gio, Gtk, GdkPixbuf, Vte, GLib, GObject, Gdk" # Test if GTK will work
164	168	- tar rvf /py3.tar /opt/faraday
165	169	- mv /py3.tar $CI_PROJECT_DIR
166	170	artifacts:

333	337	tags:
334	338	- macos
335	339	stage: build
	340	allow_failure: true
336	341	before_script:
337	342	- echo 'export PATH="/usr/local/opt/openssl/bin:$PATH"' >> ~/.bash_profile
338	343	- export LDFLAGS="-L/usr/local/opt/openssl/lib"

352	357	- git clone https://github.com/jordansissel/fpm.git
353	358	- cd fpm
354	359	- git checkout d7b466787d17581bc723e474ecf6e18f48226031
355		- git apply ../fpm-patchs/fpm.virtualenv.patch
	360	- git apply ../fpm-patchs/fpm.osx.patch
356	361	- make gem
357	362	- export GEM_HOME="$HOME/.gem"
358	363	- export GEM_PATH=$HOME/.gem
359	364	- export PATH=$PATH:$HOME/.gem/bin
360	365	- gem install fpm-1.11.0.gem
361	366	- cd ../../
362		- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server osxpkg
	367	- sh faraday-linux-installers-builder/build_osx.sh $(eval $IMAGE_TAG) server osxpkg
	368	- mv faraday-server_amd64.pkg ../../Faraday-server.pkg
	369	- sh faraday-linux-installers-builder/build_osx.sh $(eval $IMAGE_TAG) client osxpkg
	370	- mv faraday-client_amd64.pkg ../../Faraday-client.pkg
363	371	- mv faraday-server_amd64.pkg ../../Faraday.pkg
364	372	artifacts:
365	373	name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME-binaries"
366	374	paths:
367		- Faraday.pkg
	375	- Faraday-server.pkg
	376	- Faraday-client.pkg
368	377	expire_in: 1 week
369	378	only:
370	379	variables:

+1

-0

CHANGELOG/3.10.1/date.md less more

0

Jan 10th, 2020

+5

-0

CHANGELOG/3.10.1/white.md less more

	0	* Fix installation with `pip install --no-binary :all: faradaysec`
	1	* Force usage of webargs 5 (webargs 6 broke backwards compatibility)
	2	* Use latest version of faraday-plugins
	3	* Fix broken "Faraday Plugin" menu entry in the GTK client
	4	* Extract export csv to reuse for reports

+1

-0

CHANGELOG/3.10.2/date.md less more

0

Jan 30th, 2020

+7

-0

CHANGELOG/3.10.2/white.md less more

	0	* Fix Cross-Site Request Forgery (CSRF) vulnerability in all JSON API endpoints.
	1	This was caused because a third-party library doesn't implement proper
	2	Content-Type header validation. To mitigate the vulnerability, we set the
	3	session cookie to have the `SameSite: Lax` property.
	4	* Fix Faraday Server logs were always in debug
	5	* Add update date column when exporting vulnerabilities to CSV
	6	* Fix unicode error when exporting vulnerabilities to CSV

+18

-0

CHANGELOG/RELEASE.md less more

7	7	New features in the latest update
8	8	=====================================
9	9
	10
	11	3.10.2 [Jan 30th, 2020]:
	12	---
	13	* Fix Cross-Site Request Forgery (CSRF) vulnerability in all JSON API endpoints.
	14	This was caused because a third-party library doesn't implement proper
	15	Content-Type header validation. To mitigate the vulnerability, we set the
	16	session cookie to have the `SameSite: Lax` property.
	17	* Fix Faraday Server logs were always in debug
	18	* Add update date column when exporting vulnerabilities to CSV
	19	* Fix unicode error when exporting vulnerabilities to CSV
	20
	21	3.10.1 [Jan 10th, 2020]:
	22	---
	23	* Fix installation with `pip install --no-binary :all: faradaysec`
	24	* Force usage of webargs 5 (webargs 6 broke backwards compatibility)
	25	* Use latest version of faraday-plugins
	26	* Fix broken "Faraday Plugin" menu entry in the GTK client
	27	* Extract export csv to reuse for reports
10	28
11	29	3.10 [Dec 19th, 2019]:
12	30	---

+0

-1

~~CHANGELOG/reuse_export_csv.md~~ less more

0

extract export csv to reuse for reports⏎

+2

-1

MANIFEST.in less more

3	3	include faraday/config/default.xml
4	4	include faraday/server/default.ini
5	5	include requirements.txt
	6	include requirements_dev.txt
6	7	include requirements_server.txt
7	8	include faraday/client/zsh/faraday.zsh
8	9	include faraday/client/zsh/faraday-terminal.zsh
9	10	include faraday/client/gui/gtk/menubar.xml
10	11	recursive-include faraday/client/data *
11		include faraday/client/plugins/port_mapper.txt⏎
	12	include faraday/client/plugins/port_mapper.txt

+18

-0

RELEASE.md less more

7	7	New features in the latest update
8	8	=====================================
9	9
	10
	11	3.10.2 [Jan 30th, 2020]:
	12	---
	13	* Fix Cross-Site Request Forgery (CSRF) vulnerability in all JSON API endpoints.
	14	This was caused because a third-party library doesn't implement proper
	15	Content-Type header validation. To mitigate the vulnerability, we set the
	16	session cookie to have the `SameSite: Lax` property.
	17	* Fix Faraday Server logs were always in debug
	18	* Add update date column when exporting vulnerabilities to CSV
	19	* Fix unicode error when exporting vulnerabilities to CSV
	20
	21	3.10.1 [Jan 10th, 2020]:
	22	---
	23	* Fix installation with `pip install --no-binary :all: faradaysec`
	24	* Force usage of webargs 5 (webargs 6 broke backwards compatibility)
	25	* Use latest version of faraday-plugins
	26	* Fix broken "Faraday Plugin" menu entry in the GTK client
	27	* Extract export csv to reuse for reports
10	28
11	29	3.10 [Dec 19th, 2019]:
12	30	---

+1

-1

buildpth.nix less more

1	1
2	2	stdenv.mkDerivation {
3	3	name = "faraday-nix.pth";
4		packages = with python37Packages; [virtualenv pip pyopenssl psycopg2 pillow pygobject3 pynacl matplotlib lxml ldap autobahn gssapi setproctitle simplejson pycairo ];
	4	packages = with python37Packages; [virtualenv pip pyopenssl psycopg2 pillow pygobject3 pynacl matplotlib numpy lxml ldap autobahn gssapi setproctitle simplejson pycairo cffi cairocffi bcrypt twisted];
5	5	builder = ./buildpth.sh;
6	6	}

+1

-1

faraday/__init__.py less more

1	1	# Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/)
2	2	# See the file 'doc/LICENSE' for the license information
3	3
4		__version__ = '3.10.0'
	4	__version__ = '3.10.2'
5	5	__license_version__ = __version__

+1

-1

faraday/client/gui/gtk/application.py less more

977	977	if active_workspace:
978	978	command = fplugin_utils.build_faraday_plugin_command(plugin, active_workspace.getName())
979	979	fd = terminal.get_pty().get_fd()
980		os.write(fd, command)
	980	os.write(fd, command.encode())
981	981
982	982
983	983	# I'm Py3

+1

-1

faraday/config/default.xml less more

1	1	<faraday>
2	2
3	3	<appname>Faraday - Penetration Test IDE</appname>
4		<version>3.10.0</version>
	4	<version>3.10.2</version>
5	5	<debug_status>0</debug_status>
6	6	<font>-Misc-Fixed-medium-r-normal-*-12-100-100-100-c-70-iso8859-1</font>
7	7	<home_path></home_path>

+7

-2

faraday/server/api/modules/vulns.py less more

45	45	)
46	46	from faraday.server.utils.database import get_or_create
47	47	from faraday.server.utils.export import export_vulns_to_csv
	48	from faraday.server.utils.py3 import BytesJSONEncoder
48	49
49	50	from faraday.server.api.modules.services import ServiceSchema
50	51	from faraday.server.schemas import (

710	711	normal_vulns_host = normal_vulns.join(Host).join(Hostname).filter(or_(*or_filters))
711	712	normal_vulns = normal_vulns_host.union(normal_vulns.join(Service).join(Host).join(Hostname).filter(or_(*or_filters)))
712	713
713		normal_vulns = self.schema_class_dict['VulnerabilityWeb'](**marshmallow_params).dumps(normal_vulns.all())
	714	normal_vulns = self.schema_class_dict['VulnerabilityWeb'](**marshmallow_params).dumps(normal_vulns.all(),
	715	cls=BytesJSONEncoder)
714	716	normal_vulns_data = json.loads(normal_vulns.data)
715	717	except Exception as ex:
	718	logger.exception(ex)
716	719	normal_vulns_data = []
717	720	try:
718	721	web_vulns = search(db.session,

725	728	or_filters.append(Hostname.name == hostname_filter['val'])
726	729
727	730	web_vulns = web_vulns.join(Service).join(Host).join(Hostname).filter(or_(*or_filters))
728		web_vulns = self.schema_class_dict['VulnerabilityWeb'](**marshmallow_params).dumps(web_vulns.all())
	731	web_vulns = self.schema_class_dict['VulnerabilityWeb'](**marshmallow_params).dumps(web_vulns.all(),
	732	cls=BytesJSONEncoder)
729	733	web_vulns_data = json.loads(web_vulns.data)
730	734	except Exception as ex:
	735	logger.exception(ex)
731	736	web_vulns_data = []
732	737	return normal_vulns_data + web_vulns_data
733	738

+2

-1

faraday/server/app.py less more

303	303	'plaintext', # TODO: remove it
304	304	],
305	305	'PERMANENT_SESSION_LIFETIME': datetime.timedelta(hours=12),
306		'SESSION_COOKIE_NAME': 'faraday_session',
	306	'SESSION_COOKIE_NAME': 'faraday_session_2',
	307	'SESSION_COOKIE_SAMESITE': 'Lax',
307	308	})
308	309
309	310	store = FilesystemStore(app.config['SESSION_FILE_DIR'])

+2

-1

faraday/server/utils/export.py less more

18	18	"target", "desc", "status", "hostnames", "comments", "owner", "os", "resolution", "easeofresolution", "web_vulnerability",
19	19	"data", "website", "path", "status_code", "request", "method", "params", "pname", "query",
20	20	"policyviolations", "external_id", "impact_confidentiality", "impact_integrity", "impact_availability",
21		"impact_accountability"
	21	"impact_accountability", "update_date"
22	22	]
23	23	headers += custom_fields_columns
24	24	writer = csv.DictWriter(buffer, fieldnames=headers)

43	43	vuln_dict = {"confirmed": vuln['confirmed'],
44	44	"id": vuln.get('_id', None),
45	45	"date": vuln_date,
	46	"update_date": vuln['metadata']['update_time'],
46	47	"severity": vuln.get('severity', None),
47	48	"target": vuln.get('target', None),
48	49	"status": vuln.get('status', None),

+2

-1

faraday/server/utils/logger.py less more

46	46	file_handler = logging.handlers.RotatingFileHandler(
47	47	LOG_FILE, maxBytes=MAX_LOG_FILE_SIZE, backupCount=MAX_LOG_FILE_BACKUP_COUNT)
48	48	file_handler.setFormatter(formatter)
49		file_handler.setLevel(logging.DEBUG)
	49	file_handler.setLevel(faraday.server.config.LOGGING_LEVEL)
50	50	add_handler(file_handler)
	51	LVL_SETTABLE_HANDLERS.append(file_handler)
51	52
52	53
53	54	def add_handler(handler):

+3

-3

requirements_server.txt less more

22	22	sqlalchemy_schemadisplay>=1.3
23	23	tqdm>=4.15.0
24	24	twisted>=18.9.0
25		webargs>=5.1.0
	25	webargs>=5.1.0,<6.0.0
26	26	marshmallow-sqlalchemy==0.15.0
27	27	filteralchemy-fork
28	28	filedepot>=0.5.0

34	34	simplejson>=3.16.0
35	35	syslog-rfc5424-formatter==1.1.1
36	36	beautifulsoup4==4.7.1
37		Flask-KVSession==0.6.2
	37	Flask-KVSession-fork>=0.6.3
38	38	simplekv==0.13.0
39	39	pypcapfile==0.12.0
40	40	html2text==2019.8.11
41	41	distro==1.4.0
42		faraday-plugins==1.0rc1
	42	faraday-plugins>=1.0.1,<2.0.0

+21

-2

tests/test_api_vulnerability.py less more

5	5
6	6	'''
7	7	from __future__ import absolute_import
	8
	9	import datetime
8	10	from builtins import str
9	11
10	12	import json

17	19
18	20
19	21	import pytz
	22	from dateutil import parser
20	23	from depot.manager import DepotManager
21	24
22	25	from faraday.server.fields import FaradayUploadedFile

1900	1903	"target", "desc", "status", "hostnames", "comments", "owner", "os", "resolution", "easeofresolution", "web_vulnerability",
1901	1904	"data", "website", "path", "status_code", "request", "method", "params", "pname", "query",
1902	1905	"policyviolations", "external_id", "impact_confidentiality", "impact_integrity", "impact_availability",
1903		"impact_accountability"
	1906	"impact_accountability", "update_date"
1904	1907	]
1905	1908	assert res.status_code == 200
1906	1909	assert res.data.decode('utf-8').strip('\r\n').split(',') == expected_headers

1949	1952	self._verify_csv(res.data, confirmed=True)
1950	1953
1951	1954	@pytest.mark.usefixtures('ignore_nplusone')
	1955	def test_export_vulns_check_update_time(self, session, test_client):
	1956	self.first_object.confirm =True
	1957	session.add(self.first_object)
	1958	session.commit()
	1959	res = test_client.get(self.url() + 'export_csv/')
	1960	assert res.status_code == 200
	1961	csv_data = csv.DictReader(StringIO(res.data.decode('utf-8')), delimiter=',')
	1962	for index, line in enumerate(csv_data):
	1963	if self.first_object.id == int(line['id']):
	1964	create_date = parser.parse(line['date'])
	1965	update_date = parser.parse(line['update_date'])
	1966	delta = update_date - create_date
	1967	assert create_date < update_date
	1968
	1969
	1970	@pytest.mark.usefixtures('ignore_nplusone')
1952	1971	def test_export_vulns_csv_with_custom_fields(self, session, test_client):
1953	1972
1954	1973	custom_field_schema = CustomFieldsSchemaFactory(

1981	2000	"target", "desc", "status", "hostnames", "comments", "owner", "os", "resolution", "easeofresolution", "web_vulnerability",
1982	2001	"data", "website", "path", "status_code", "request", "method", "params", "pname", "query",
1983	2002	"policyviolations", "external_id", "impact_confidentiality", "impact_integrity", "impact_availability",
1984		"impact_accountability"
	2003	"impact_accountability", "update_date"
1985	2004	]
1986	2005	final_expected_headers = expected_headers + custom_fields
1987	2006	csv_data = csv.reader(StringIO(raw_csv_data.decode('utf-8')), delimiter=',')