"""
Faraday Penetration Test IDE
Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/)
See the file 'doc/LICENSE' for the license information
"""
import json
import logging
from json import JSONDecodeError
from typing import Tuple
import flask
import sqlalchemy
import datetime
from collections import defaultdict
from flask_classful import FlaskView
from sqlalchemy.orm import joinedload, undefer, with_expression
from sqlalchemy.orm.exc import NoResultFound, ObjectDeletedError
from sqlalchemy.inspection import inspect
from sqlalchemy import func, desc, asc
from marshmallow import Schema, EXCLUDE, fields
from marshmallow.validate import Length
from marshmallow_sqlalchemy import ModelConverter
from marshmallow_sqlalchemy.schema import SQLAlchemyAutoSchemaOpts, SQLAlchemyAutoSchemaMeta
from sqlalchemy.sql.elements import BooleanClauseList
from webargs.flaskparser import FlaskParser
from webargs.core import ValidationError
from flask_classful import route
import flask_login
from faraday.server.models import (Workspace,
db,
Command,
CommandObject,
count_vulnerability_severities,
_make_vuln_count_property,
_make_active_agents_count_property)
from faraday.server.schemas import NullToBlankString
from faraday.server.utils.database import (
get_conflict_object,
is_unique_constraint_violation,
not_null_constraint_violation
)
from faraday.server.utils.filters import FlaskRestlessSchema
from faraday.server.utils.search import search
from faraday.server.config import faraday_server
logger = logging.getLogger(__name__)
def output_json(data, code, headers=None):
content_type = 'application/json'
dumped = json.dumps(data)
if headers:
headers.update({'Content-Type': content_type})
else:
headers = {'Content-Type': content_type}
response = flask.make_response(dumped, code, headers)
return response
class InvalidUsage(Exception):
status_code = 400
def __init__(self, message, status_code=None, payload=None):
Exception.__init__(self)
self.message = message
if status_code is not None:
self.status_code = status_code
self.payload = payload
def to_dict(self):
rv = dict(self.payload or ())
rv['message'] = self.message
return rv
# TODO: Require @view decorator to enable custom routes
class GenericView(FlaskView):
"""Abstract class to provide generic views. Inspired in `Django REST
Framework generic viewsets`_.
To create new views, you should create a class inheriting from
GenericView (or from one of its subclasses) and set the model_class,
schema_class, and optionally the rest of class attributes.
Then, you should register it with your app by using the ``register``
classmethod.
.. _Django REST Framework generic viewsets: http://www.django-rest-framework.org/api-guide/viewsets/#genericviewset
"""
# Must-implement attributes
#: **Required**. The class of the SQLAlchemy model this view will handle
model_class = None
#: **Required** (unless _get_schema_class is overwritten).
#: A subclass of `marshmallow.Schema` to serialize and deserialize the
#: data provided by the user
schema_class = None
# Default attributes
#: The prefix where the endpoint should be registered.
#: This is useful for API versioning
route_prefix = '/v3/'
#: Arguments that are passed to the view but shouldn't change the route
#: rule. This should be used when route_prefix is parametrized
#:
#: You tipically won't need this, unless you're creating nested views.
#: For example GenericWorkspacedView use this so the workspace name is
#: prepended to the view URL
base_args = []
#: Decides how you want to format the output response. It is set to dump a
#: JSON object by default.
#: See http://flask-classful.teracy.org/#adding-resource-representations-get-real-classy-and-put-on-a-top-hat
#: for more information
representations = {
'application/json': output_json,
'flask-classful/default': output_json,
}
""
#: Name of the field of the model used to get the object instance in
#: retrieve, update and delete endpoints.
#:
#: For example, if you have a `Tag` model, maybe a `slug` would be good
#: lookup field.
#:
#: .. note::
#: You must use a unique field here instead of one allowing
#: duplicate values
#:
#: .. note::
#: By default the lookup field value must be a valid integer. If you
#: want to allow any string, like with the slug field, make sure that
#: you set lookup_field_type to `string`
lookup_field = 'id'
#: A function that converts the string paremeter passed in the URL to the
#: value that will be queried in the database.
#: It defaults to int to match the type of the default lookup_field_type
#: (id)
lookup_field_type = int
# Attributes to improve the performance of list and retrieve views
#: List of relationships to eagerload in list and retrieve views.
#:
#: This is useful when you when you want to retrieve all childrens
#: of an object in an API response, like for example if you want
#: to have all hostnames of each host in the hosts endpoint.
get_joinedloads = [] # List of relationships to eagerload
#: List of columns that will be loaded directly when performing an
#: eagerloaded query.
#:
#: This is useful when you have a column that is typically deferred because
#: typically is isn't used, like the vuln creator. If you know you will use
#: it, indicate it here to prevent doing an extra SQL query.
get_undefer = [] # List of columns to undefer
trailing_slash = False
def _get_schema_class(self):
"""By default, it returns ``self.schema_class``.
You can override it to define a custom behavior to be used
in all views.
"""
assert self.schema_class is not None, "You must define schema_class"
return self.schema_class
def _get_schema_instance(self, route_kwargs, **kwargs):
"""Instances a model schema.
It also uses _set_schema_context to set the context of the
schema.
"""
kwargs['context'] = self._set_schema_context(
kwargs.get('context', {}), **route_kwargs)
# If the client send us fields that are not in the schema, ignore them
# This is the default in marshmallow 2, but not in marshmallow 3
kwargs['unknown'] = EXCLUDE
return self._get_schema_class()(**kwargs)
def _set_schema_context(self, context, **kwargs):
"""This function can be overriden to update the context passed
to the schema.
"""
return context
def _get_lookup_field(self):
"""Get a Field instance based on ``self.model_class`` and
``self.lookup_field``
"""
return getattr(self.model_class, self.lookup_field)
def _validate_object_id(self, object_id, raise_error=True):
"""
By default, it validates the value of the lookup field set by the user
in the URL by calling ``self.lookup_field_type(object_id)``.
If that raises a ValueError, que view will fail with error
code 404.
"""
try:
self.lookup_field_type(object_id)
except ValueError:
if raise_error:
flask.abort(404, 'Invalid format of lookup field')
return False
return True
def _get_base_query(self):
"""Return the initial query all views should use
.. warning::
When you are creating views, avoid making SQL queries that
don't inherit from this base query. You could easily forget
to add workspace permission checks and similar stuff.
"""
query = self.model_class.query
return query
def _get_eagerloaded_query(self, *args, **kwargs):
"""Load objects related to the current model in a single query.
This is useful to prevent n+1 SQL problems, where a request to an
object with many childs makes many SQL requests that tends to be
slow.
You tipically won't need to overwrite this method, but to set
get_joinedloads and get_undefer attributes that are used by
this method.
In really complex cases where good performance is required,
like in the vulns API endpoint, you will have to overwrite this.
"""
options = []
try:
has_creator = 'owner' in self._get_schema_class().opts.fields
except AttributeError:
has_creator = False
if has_creator:
# APIs for objects with metadata always return the creator's
# username. Do a joinedload to prevent doing one query per object
# (n+1) problem
options.append(joinedload(
getattr(self.model_class, 'creator')).load_only('username'))
query = self._get_base_query(*args, **kwargs)
options += [joinedload(relationship)
for relationship in self.get_joinedloads]
options += [undefer(column) for column in self.get_undefer]
return query.options(*options)
def _filter_query(self, query):
"""Return a new SQLAlchemy query with some filters applied.
By default it doesn't do anything. It is overriden by
:class:`FilterAlchemyMixin` to give support to Filteralchemy
filters.
.. warning::
This is only used by the list endpoints. Don't use this
to restrict the user the access for certain elements (like
for example to restrict the items to one workspace). For
this you must override _get_base_query instead.
Always think that this filtering is optional, just a
feature for the user to only see items he/she is interested
in, so it is the user who will filter the data, not you
"""
return query
def _get_object(self, object_id, eagerload=False, **kwargs):
"""
Given the object_id and extra route params, get an instance of
``self.model_class``
"""
self._validate_object_id(object_id)
if eagerload:
query = self._get_eagerloaded_query(**kwargs)
else:
query = self._get_base_query(**kwargs)
try:
obj = query.filter(self._get_lookup_field() == object_id).one()
except NoResultFound:
flask.abort(404, f'Object with id "{object_id}" not found')
return obj
def _get_objects(self, object_ids, eagerload=False, **kwargs):
"""
Given the object_id and extra route params, get an instance of
``self.model_class``
"""
object_ids = [object_id for object_id in object_ids if self._validate_object_id(object_id, raise_error=False)]
if eagerload:
query = self._get_eagerloaded_query(**kwargs)
else:
query = self._get_base_query(**kwargs)
try:
obj = query.filter(self._get_lookup_field().in_(object_ids)).all()
except NoResultFound:
return []
return obj
def _dump(self, obj, route_kwargs, **kwargs):
"""Serializes an object with the Marshmallow schema class
returned by ``self._get_schema_class()``. Any passed kwargs
will be passed to the ``__init__`` method of the schema.
TODO migration: document route_kwargs
"""
try:
return self._get_schema_instance(route_kwargs, **kwargs).dump(obj)
except ObjectDeletedError:
return []
def _parse_data(self, schema, request, *args, **kwargs):
"""Deserializes from a Flask request to a dict with valid
data. It a ``Marshmallow.Schema`` instance to perform the
deserialization
"""
return FlaskParser(unknown=EXCLUDE).parse(schema, request, location="json",
*args, **kwargs)
@classmethod
def register(cls, app, *args, **kwargs):
"""Register and add JSON error handler. Use error code
400 instead of 409"""
super().register(app, *args, **kwargs)
@app.errorhandler(422)
def handle_error(err): # pylint: disable=unused-variable
# webargs attaches additional metadata to the `data` attribute
exc = getattr(err, 'exc')
if exc:
# Get validations from the ValidationError object
messages = exc.messages
else:
messages = ['Invalid request']
return flask.jsonify({
'messages': messages,
}), 400
@app.errorhandler(409)
def handle_conflict(err): # pylint: disable=unused-variable
# webargs attaches additional metadata to the `data` attribute
exc = getattr(err, 'exc', None) or getattr(err, 'description', None)
if exc:
# Get validations from the ValidationError object
messages = exc.messages
else:
messages = ['Invalid request']
return flask.jsonify(messages), 409
@app.errorhandler(InvalidUsage)
def handle_invalid_usage(error): # pylint: disable=unused-variable
response = flask.jsonify(error.to_dict())
response.status_code = error.status_code
return response
# @app.errorhandler(404)
def handle_not_found(err): # pylint: disable=unused-variable
response = {'success': False, 'message': err.description if faraday_server.debug else err.name}
return flask.jsonify(response), 404
@app.errorhandler(500)
def handle_server_error(err): # pylint: disable=unused-variable
response = {'success': False,
'message': f"Exception: {err.original_exception}" if faraday_server.debug else 'Internal Server Error'}
return flask.jsonify(response), 500
class GenericWorkspacedView(GenericView):
"""Abstract class for a view that depends on the workspace, that is
passed in the URL
.. note::
This view inherits from GenericView, so make sure you understand
that first by checking the docs above, or just by looking at the
source code of server/api/base.py.
"""
# Default attributes
route_prefix = '/v3/ws/<workspace_name>/'
base_args = ['workspace_name'] # Required to prevent double usage of <workspace_name>
def _get_workspace(self, workspace_name):
try:
ws = Workspace.query.filter_by(name=workspace_name).one()
if not ws.active:
flask.abort(403, f"Disabled workspace: {workspace_name}")
except NoResultFound:
flask.abort(404, f"No such workspace: {workspace_name}")
return ws
def _get_base_query(self, workspace_name):
base = super()._get_base_query()
return base.join(Workspace).filter(
Workspace.id == self._get_workspace(workspace_name).id)
def _get_object(self, object_id, workspace_name, eagerload=False):
self._validate_object_id(object_id)
if eagerload:
query = self._get_eagerloaded_query(workspace_name)
else:
query = self._get_base_query(workspace_name)
try:
obj = query.filter(self._get_lookup_field() == object_id).one()
except NoResultFound:
flask.abort(404, f'Object with id "{object_id}" not found')
return obj
def _set_schema_context(self, context, **kwargs):
"""Overriden to pass the workspace name to the schema"""
context.update(kwargs)
return context
def before_request(self, name, *args, **kwargs):
sup = super()
if hasattr(sup, 'before_request'):
sup.before_request(name, *args, **kwargs)
if (self._get_workspace(kwargs['workspace_name']).readonly
and flask.request.method not in ['GET', 'HEAD', 'OPTIONS']):
flask.abort(403, "Altering a readonly workspace is not allowed")
class GenericMultiWorkspacedView(GenericWorkspacedView):
"""Abstract class for a view that depends on the workspace, that is
passed in the URL. The object can be accessed from more than one workspace.
.. note::
This view inherits from GenericWorkspacedView and GenericView, so make
sure you understand those first by checking the docs above, or just
by looking at the source code of server/api/base.py.
"""
def _get_base_query(self, workspace_name):
base = super(GenericWorkspacedView, self)._get_base_query()
return base.filter(
self.model_class.workspaces.any(
name=self._get_workspace(workspace_name).name
)
)
class ListMixin:
"""Add GET / route"""
#: If set (to a SQLAlchemy attribute instance) use this field to order the
#: query by default
order_field = None
def _envelope_list(self, objects, pagination_metadata=None):
"""Override this method to define how a list of objects is
rendered.
See the example of :ref:`envelope-list-example` to learn
when and how it should be used.
"""
return objects
def _paginate(self, query):
"""Overwrite this to implement pagination in the list endpoint.
This is typically overwritten by SortableMixin.
The method takes a query as argument and should return a tuple
containing a new filtered query and a "pagination metadata"
object that will be used by _envelope_list. If you don't need
the latter just set is as None.
"""
return query, None
def _get_order_field(self, **kwargs):
"""Return the field used to sort the query.
By default it returns the value of self.order_field, but it
can be overwritten to something else, as SortableMixin does.
"""
return self.order_field
def index(self, **kwargs):
"""
---
tags: [{tag_name}]
summary: "Get a list of {class_model}."
responses:
200:
description: Ok
content:
application/json:
schema: {schema_class}
"""
query = self._filter_query(self._get_eagerloaded_query(**kwargs))
order_field = self._get_order_field(**kwargs)
if order_field is not None:
if isinstance(order_field, tuple):
query = query.order_by(*order_field)
else:
query = query.order_by(order_field)
objects, pagination_metadata = self._paginate(query)
return self._envelope_list(self._dump(objects, kwargs, many=True),
pagination_metadata)
class SortableMixin:
"""Enables custom sorting by a field specified by the user
See the example of :ref:`pagination-and-sorting-recipe` to learn
how is it used.
Works for both workspaced and non-workspaced views.
"""
sort_field_paremeter_name = "sort"
sort_direction_paremeter_name = "sort_dir"
sort_pass_silently = False
default_sort_direction = "asc"
sort_model_class = None # Override to use a model with more fields
def _get_order_field(self, **kwargs):
try:
order_field = flask.request.args[self.sort_field_paremeter_name]
except KeyError:
# Sort field not specified, return the default
return self.order_field
# Check that the field is in the schema to prevent unwanted fields
# value leaking
schema = self._get_schema_instance(kwargs)
# Add metadata nested field
try:
metadata_field = schema.fields.pop('metadata')
except KeyError:
pass
else:
for (key, value) in metadata_field.target_schema.fields.items():
schema.fields['metadata.' + key] = value
schema.fields[key] = value
try:
field_instance = schema.fields[order_field]
except KeyError:
if self.sort_pass_silently:
logger.warn(f"Unknown field: {order_field}")
return self.order_field
raise InvalidUsage(f"Unknown field: {order_field}")
# Translate from the field name in the schema to the database field
# name
order_field = field_instance.attribute or order_field
# TODO migration: improve this checking or use a whitelist.
# Handle PrimaryKeyRelatedField
model_class = self.sort_model_class or self.model_class
if order_field not in inspect(model_class).attrs:
if self.sort_pass_silently:
logger.warn(f"Field not in the DB: {order_field}")
return self.order_field
# It could be something like fields.Method
raise InvalidUsage(f"Field not in the DB: {order_field}")
if hasattr(model_class, order_field + '_id'):
# Ugly hack to allow sorting by a parent
field = getattr(model_class, order_field + '_id')
else:
field = getattr(model_class, order_field)
sort_dir = flask.request.args.get(self.sort_direction_paremeter_name,
self.default_sort_direction)
if sort_dir not in ('asc', 'desc'):
if self.sort_pass_silently:
logger.warn(f"Invalid value for sorting direction: {sort_dir}")
return self.order_field
raise InvalidUsage(f"Invalid value for sorting direction: {sort_dir}")
try:
if self.order_field is not None:
if not isinstance(self.order_field, tuple):
self.order_field = (self.order_field,)
return (getattr(field, sort_dir)(),) + self.order_field
else:
return getattr(field, sort_dir)()
except NotImplementedError:
if self.sort_pass_silently:
logger.warn(f"field {order_field} doesn't support sorting")
return self.order_field
# There are some fields that can't be used for sorting
raise InvalidUsage(f"field {order_field} doesn't support sorting")
class PaginatedMixin:
"""Add pagination for list route"""
per_page_parameter_name = 'page_size'
page_number_parameter_name = 'page'
def _paginate(self, query):
if self.per_page_parameter_name in flask.request.args:
try:
page = int(flask.request.args.get(
self.page_number_parameter_name, 1))
except (TypeError, ValueError):
flask.abort(404, 'Invalid page number')
try:
per_page = int(flask.request.args[
self.per_page_parameter_name])
except (TypeError, ValueError):
flask.abort(404, 'Invalid per_page value')
pagination_metadata = query.paginate(page=page, per_page=per_page, error_out=False)
return pagination_metadata.items, pagination_metadata
return super()._paginate(query)
class FilterAlchemyMixin:
"""Add querystring parameter filtering to list route
It is done by setting the ViewClass.filterset_class class
attribute
"""
filterset_class = None
def _filter_query(self, query):
assert self.filterset_class is not None, 'You must define a filterset'
return self.filterset_class(query).filter()
class FilterWorkspacedMixin(ListMixin):
"""Add filter endpoint for searching on any workspaced objects columns
"""
@route('/filter')
def filter(self, workspace_name):
"""
---
tags: [Filter, {tag_name}]
description: Filters, sorts and groups workspaced objects using a json with parameters. These parameters must be part of the model.
parameters:
- in: query
name: q
description: recursive json with filters that supports operators. The json could also contain sort and group.
responses:
200:
description: returns filtered, sorted and grouped results
content:
application/json:
schema: FlaskRestlessSchema
400:
description: invalid q was sent to the server
"""
filters = flask.request.args.get('q', '{"filters": []}')
filtered_objs, count = self._filter(filters, workspace_name)
class PageMeta:
total = 0
pagination_metadata = PageMeta()
pagination_metadata.total = count
return self._envelope_list(filtered_objs, pagination_metadata)
def _generate_filter_query(self, filters, workspace, severity_count=False):
filter_query = search(db.session,
self.model_class,
filters)
filter_query = filter_query.filter(self.model_class.workspace == workspace)
if severity_count and 'group_by' not in filters:
filter_query = count_vulnerability_severities(filter_query, self.model_class,
all_severities=True, host_vulns=True)
return filter_query
def _filter(self, filters, workspace_name, severity_count=False):
marshmallow_params = {'many': True, 'context': {}}
try:
filters = FlaskRestlessSchema().load(json.loads(filters)) or {}
except (ValidationError, JSONDecodeError) as ex:
logger.exception(ex)
flask.abort(400, "Invalid filters")
workspace = self._get_workspace(workspace_name)
if 'group_by' not in filters:
offset = None
limit = None
if 'offset' in filters:
offset = filters.pop('offset')
if 'limit' in filters:
limit = filters.pop('limit') # we need to remove pagination, since
try:
filter_query = self._generate_filter_query(
filters,
workspace,
severity_count=severity_count
)
except AttributeError as e:
flask.abort(400, e)
count = filter_query.count()
if limit:
filter_query = filter_query.limit(limit)
if offset:
filter_query = filter_query.offset(offset)
objs = self.schema_class(**marshmallow_params).dumps(filter_query.all())
return json.loads(objs), count
else:
try:
filter_query = self._generate_filter_query(
filters,
workspace,
)
except AttributeError as e:
flask.abort(400, e)
column_names = ['count'] + [field['field'] for field in filters.get('group_by', [])]
rows = [list(zip(column_names, row)) for row in filter_query.all()]
data = []
for row in rows:
data.append({field[0]: field[1] for field in row})
return data, len(rows)
class FilterMixin(ListMixin):
"""Add filter endpoint for searching on any non workspaced objects columns
"""
@route('/filter')
def filter(self):
"""
---
tags: ["Filter", {tag_name}]
description: Filters, sorts and groups non workspaced objects using a json with parameters. These parameters must be part of the model.
parameters:
- in: query
name: q
description: Recursive json with filters that supports operators. The json could also contain sort and group.
responses:
200:
description: Returns filtered, sorted and grouped results
content:
application/json:
schema: FlaskRestlessSchema
400:
description: Invalid q was sent to the server
"""
filters = flask.request.args.get('q', '{"filters": []}')
filtered_objs, count = self._filter(filters)
class PageMeta:
total = 0
pagination_metadata = PageMeta()
pagination_metadata.total = count
return self._envelope_list(filtered_objs, pagination_metadata)
def _generate_filter_query(self, filters, severity_count=False, host_vulns=False):
filter_query = search(db.session,
self.model_class,
filters)
if severity_count and 'group_by' not in filters:
filter_query = count_vulnerability_severities(filter_query, self.model_class,
all_severities=True, host_vulns=host_vulns)
filter_query = filter_query.options(
with_expression(
Workspace.vulnerability_web_count,
_make_vuln_count_property('vulnerability_web', use_column_property=False),
),
with_expression(
Workspace.vulnerability_standard_count,
_make_vuln_count_property('vulnerability', use_column_property=False)
),
with_expression(
Workspace.vulnerability_code_count,
_make_vuln_count_property('vulnerability_code', use_column_property=False),
),
with_expression(
Workspace.active_agents_count,
_make_active_agents_count_property(),
),
)
return filter_query
def _filter(self, filters: str, extra_alchemy_filters: BooleanClauseList = None,
severity_count=False, host_vulns=False) -> Tuple[list, int]:
marshmallow_params = {'many': True, 'context': {}}
try:
filters = FlaskRestlessSchema().load(json.loads(filters)) or {}
except (ValidationError, JSONDecodeError) as ex:
logger.exception(ex)
flask.abort(400, "Invalid filters")
if 'group_by' not in filters:
offset = None
limit = None
if 'offset' in filters:
offset = filters.pop('offset')
if 'limit' in filters:
limit = filters.pop('limit') # we need to remove pagination, since
try:
filter_query = self._generate_filter_query(
filters,
severity_count=severity_count,
host_vulns=host_vulns
)
except AttributeError as e:
flask.abort(400, e)
if extra_alchemy_filters is not None:
filter_query = filter_query.filter(extra_alchemy_filters)
if limit:
filter_query = filter_query.limit(limit)
if offset:
filter_query = filter_query.offset(offset)
count = filter_query.count()
objs = self.schema_class(**marshmallow_params).dumps(filter_query.all())
return json.loads(objs), count
else:
filter_query = self._generate_filter_query(
filters,
)
if extra_alchemy_filters is not None:
filter_query += filter_query.filter(extra_alchemy_filters)
column_names = ['count'] + [field['field'] for field in filters.get('group_by', [])]
rows = [list(zip(column_names, row)) for row in filter_query.all()]
data = []
for row in rows:
data.append({field[0]: field[1] for field in row})
return data, len(rows)
class ListWorkspacedMixin(ListMixin):
"""Add GET /<workspace_name>/<route_base>/ route"""
# There are no differences with the non-workspaced implementations. The code
# inside the view generic methods is enough
class RetrieveMixin:
"""Add GET /<id>/ route"""
def get(self, object_id, **kwargs):
"""
---
tags: ["{tag_name}"]
summary: Retrieves {class_model}
parameters:
- in: path
name: object_id
required: true
schema:
type: integer
responses:
200:
description: Ok
content:
application/json:
schema: {schema_class}
"""
return self._dump(self._get_object(object_id, eagerload=True,
**kwargs), kwargs)
class RetrieveWorkspacedMixin(RetrieveMixin):
"""Add GET /<workspace_name>/<route_base>/<id>/ route"""
# There are no differences with the non-workspaced implementations. The code
# inside the view generic methods is enough
def get(self, object_id, workspace_name=None):
"""
---
tags: ["{tag_name}"]
summary: Retrieves {class_model}
parameters:
- in: path
name: object_id
required: true
schema:
type: integer
- in: path
name: workspace_name
required: true
schema:
type: string
responses:
200:
description: Ok
content:
application/json:
schema: {schema_class}
"""
return super().get(object_id, workspace_name=workspace_name)
class RetrieveMultiWorkspacedMixin(RetrieveWorkspacedMixin):
"""Control GET /<workspace_name>/<route_base>/<id>/ route"""
class ReadOnlyView(SortableMixin,
ListMixin,
RetrieveMixin,
GenericView):
"""A generic view with list and retrieve endpoints
It is just a GenericView inheriting also from ListMixin,
RetrieveMixin and SortableMixin.
"""
class ReadOnlyWorkspacedView(SortableMixin,
ListWorkspacedMixin,
RetrieveWorkspacedMixin,
GenericWorkspacedView):
"""A workspaced generic view with list and retrieve endpoints
It is just a GenericWorkspacedView inheriting also from
ListWorkspacedMixin, RetrieveWorkspacedMixin and SortableMixin"""
class ReadOnlyMultiWorkspacedView(SortableMixin,
ListWorkspacedMixin,
RetrieveMultiWorkspacedMixin,
GenericMultiWorkspacedView):
"""A multi workspaced generic view with list and retrieve endpoints
It is just a GenericMultiWorkspacedView inheriting also from
ListWorkspacedMixin, RetrieveMultiWorkspacedMixin and SortableMixin"""
class CreateMixin:
"""Add POST / route"""
def post(self, **kwargs):
"""
---
tags: ["{tag_name}"]
summary: Creates {class_model}
requestBody:
required: true
content:
application/json:
schema: {schema_class}
responses:
201:
description: Created
content:
application/json:
schema: {schema_class}
409:
description: Duplicated key found
content:
application/json:
schema: {schema_class}
"""
context = {'updating': False}
data = self._parse_data(self._get_schema_instance(kwargs, context=context),
flask.request)
data.pop('id', None)
created = self._perform_create(data, **kwargs)
if not flask_login.current_user.is_anonymous:
created.creator = flask_login.current_user
db.session.commit()
return self._dump(created, kwargs), 201
def _perform_create(self, data, **kwargs):
"""Check for conflicts and create a new object
Is is passed the data parsed by the marshmallow schema (it
transform from raw post data to a JSON)
"""
obj = self.model_class(**data)
# assert not db.session.new
try:
db.session.add(obj)
db.session.commit()
except sqlalchemy.exc.IntegrityError as ex:
if not is_unique_constraint_violation(ex):
if not_null_constraint_violation(ex):
flask.abort(flask.make_response({'message': 'Be sure to send all required parameters.'}, 400))
else:
raise
db.session.rollback()
conflict_obj = get_conflict_object(db.session, obj, data)
if conflict_obj:
flask.abort(409, ValidationError(
{
'message': 'Existing value',
'object': self._get_schema_class()().dump(
conflict_obj),
}
))
else:
raise
return obj
class CommandMixin():
"""
Created the command obj to log model activity after a command
execution via the api (ex. from plugins)
This will use GET parameter command_id.
NOTE: GET parameters are also available in POST requests
"""
def _set_command_id(self, obj, created):
try:
# validates the data type from user input.
command_id = int(flask.request.args.get('command_id', None))
except TypeError:
command_id = None
if command_id:
command = db.session.query(Command).filter(Command.id == command_id,
Command.workspace == obj.workspace).first()
if command is None:
raise InvalidUsage('Command not found.')
# if the object is created and updated in the same command
# the command object already exists
# we skip the creation.
object_type = obj.__class__.__table__.name
command_object = CommandObject.query.filter_by(
object_id=obj.id,
object_type=object_type,
command=command,
workspace=obj.workspace,
).first()
if created or not command_object:
command_object = CommandObject(
object_id=obj.id,
object_type=object_type,
command=command,
workspace=obj.workspace,
created_persistent=created
)
db.session.add(command)
db.session.add(command_object)
class CreateWorkspacedMixin(CreateMixin, CommandMixin):
"""Add POST /<workspace_name>/<route_base>/ route
If a GET parameter command_id is passed, it will create a new
CommandObject associated to that command to register the change in
the database.
"""
def post(self, workspace_name=None):
"""
---
tags: ["{tag_name}"]
summary: Creates {class_model}
parameters:
- in: path
name: workspace_name
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema: {schema_class}
responses:
201:
description: Created
content:
application/json:
schema: {schema_class}
409:
description: Duplicated key found
content:
application/json:
schema: {schema_class}
"""
return super().post(workspace_name=workspace_name)
def _perform_create(self, data, workspace_name):
assert not db.session.new
workspace = self._get_workspace(workspace_name)
obj = self.model_class(**data)
obj.workspace = workspace
# assert not db.session.new
try:
db.session.add(obj)
db.session.commit()
except sqlalchemy.exc.IntegrityError as ex:
if not is_unique_constraint_violation(ex):
raise
db.session.rollback()
workspace = self._get_workspace(workspace_name)
conflict_obj = get_conflict_object(db.session, obj, data, workspace)
if conflict_obj:
flask.abort(409, ValidationError(
{
'message': 'Existing value',
'object': self._get_schema_class()().dump(
conflict_obj),
}
))
else:
raise
self._set_command_id(obj, True)
return obj
class UpdateMixin:
"""Add PUT /<id>/ route"""
def put(self, object_id, **kwargs):
"""
---
tags: ["{tag_name}"]
summary: Updates {class_model}
parameters:
- in: path
name: object_id
required: true
schema:
type: integer
requestBody:
required: true
content:
application/json:
schema: {schema_class}
responses:
200:
description: Ok
content:
application/json:
schema: {schema_class}
409:
description: Duplicated key found
content:
application/json:
schema: {schema_class}
"""
obj = self._get_object(object_id, **kwargs)
context = {'updating': True, 'object': obj}
data = self._parse_data(self._get_schema_instance(kwargs, context=context),
flask.request)
# just in case an schema allows id as writable.
data.pop('id', None)
self._update_object(obj, data, partial=False)
self._perform_update(object_id, obj, data, **kwargs)
return self._dump(obj, kwargs), 200
def _update_object(self, obj, data, **kwargs):
"""Perform changes in the selected object
It modifies the attributes of the SQLAlchemy model to match
the data passed by the Marshmallow schema.
It is common to overwrite this method to do something strange
with some specific field. Typically the new method should call
this one to handle the update of the rest of the fields.
"""
for (key, value) in data.items():
setattr(obj, key, value)
def _perform_update(self, object_id, obj, data, workspace_name=None, partial=False):
"""Commit the SQLAlchemy session, check for updating conflicts"""
try:
db.session.add(obj)
db.session.commit()
except sqlalchemy.exc.IntegrityError as ex:
if not is_unique_constraint_violation(ex):
raise
db.session.rollback()
workspace = None
if workspace_name:
workspace = db.session.query(Workspace).filter_by(name=workspace_name).first()
conflict_obj = get_conflict_object(db.session, obj, data, workspace)
if conflict_obj:
flask.abort(409, ValidationError(
{
'message': 'Existing value',
'object': self._get_schema_class()().dump(
conflict_obj),
}
))
else:
raise
return obj
def patch(self, object_id, **kwargs):
"""
---
tags: ["{tag_name}"]
summary: Updates {class_model}
parameters:
- in: path
name: object_id
required: true
schema:
type: integer
requestBody:
required: true
content:
application/json:
schema: {schema_class}
responses:
200:
description: Ok
content:
application/json:
schema: {schema_class}
409:
description: Duplicated key found
content:
application/json:
schema: {schema_class}
"""
obj = self._get_object(object_id, **kwargs)
context = {'updating': True, 'object': obj}
data = self._parse_data(self._get_schema_instance(kwargs, context=context, partial=True),
flask.request)
# just in case an schema allows id as writable.
data.pop('id', None)
self._update_object(obj, data, partial=True)
self._perform_update(object_id, obj, data, partial=True, **kwargs)
return self._dump(obj, kwargs), 200
class BulkUpdateMixin:
# These mixin should be merged with DeleteMixin after v2 is removed
@route('', methods=['PATCH'])
def bulk_update(self, **kwargs):
"""
---
tags: [{tag_name}]
summary: "Update a group of {class_model} by ids."
responses:
204:
description: Ok
"""
# TODO BULK_UPDATE_SCHEMA
if not flask.request.json or 'ids' not in flask.request.json:
flask.abort(400)
ids = list(filter(lambda x: type(x) == self.lookup_field_type, flask.request.json['ids']))
objects = self._get_objects(ids, **kwargs)
context = {'updating': True, 'objects': objects}
data = self._parse_data(self._get_schema_instance(kwargs, context=context, partial=True),
flask.request)
# just in case an schema allows id as writable.
data.pop('id', None)
data.pop('ids', None)
return self._perform_bulk_update(ids, data, **kwargs), 200
def _bulk_update_query(self, ids, **kwargs):
# It IS better to as is but warn of ON CASCADE
return self.model_class.query.filter(self.model_class.id.in_(ids))
def _pre_bulk_update(self, data, **kwargs):
return {}
def _post_bulk_update(self, ids, extracted_data, **kwargs):
pass
def _perform_bulk_update(self, ids, data, workspace_name=None, **kwargs):
try:
post_bulk_update_data = self._pre_bulk_update(data, **kwargs)
if (len(data) > 0 or len(post_bulk_update_data) > 0) and len(ids) > 0:
queryset = self._bulk_update_query(ids, workspace_name=workspace_name, **kwargs)
updated = queryset.update(data, synchronize_session='fetch')
self._post_bulk_update(ids, post_bulk_update_data, workspace_name=workspace_name)
else:
updated = 0
db.session.commit()
response = {'updated': updated}
return flask.jsonify(response)
except ValueError as e:
db.session.rollback()
flask.abort(400, ValidationError(
{
'message': str(e),
}
))
except sqlalchemy.exc.IntegrityError as ex:
if not is_unique_constraint_violation(ex):
raise
db.session.rollback()
workspace = None
if workspace_name:
workspace = db.session.query(Workspace).filter_by(name=workspace_name).first()
conflict_obj = get_conflict_object(db.session, self.model_class(), data, workspace)
if conflict_obj is not None:
flask.abort(409, ValidationError(
{
'message': 'Existing value',
'object': self._get_schema_class()().dump(
conflict_obj),
}
))
elif len(ids) >= 2:
flask.abort(409, ValidationError(
{
'message': 'Updating more than one object with unique data',
'data': data
}
))
else:
raise
class UpdateWorkspacedMixin(UpdateMixin, CommandMixin):
"""Add PUT /<workspace_name>/<route_base>/<id>/ route
If a GET parameter command_id is passed, it will create a new
CommandObject associated to that command to register the change in
the database.
"""
def put(self, object_id, workspace_name=None):
"""
---
tags: ["{tag_name}"]
summary: Updates {class_model}
parameters:
- in: path
name: object_id
required: true
schema:
type: integer
- in: path
name: workspace_name
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema: {schema_class}
responses:
200:
description: Ok
content:
application/json:
schema: {schema_class}
409:
description: Duplicated key found
content:
application/json:
schema: {schema_class}
"""
return super().put(object_id, workspace_name=workspace_name)
def _perform_update(self, object_id, obj, data, workspace_name=None, partial=False):
# # Make sure that if I created new objects, I had properly commited them
# assert not db.session.new
with db.session.no_autoflush:
obj.workspace = self._get_workspace(workspace_name)
self._set_command_id(obj, False)
return super()._perform_update(object_id, obj, data, workspace_name)
def patch(self, object_id, workspace_name=None):
"""
---
tags: ["{tag_name}"]
summary: Updates {class_model}
parameters:
- in: path
name: object_id
required: true
schema:
type: integer
- in: path
name: workspace_name
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema: {schema_class}
responses:
200:
description: Ok
content:
application/json:
schema: {schema_class}
409:
description: Duplicated key found
content:
application/json:
schema: {schema_class}
"""
return super().patch(object_id, workspace_name=workspace_name)
class BulkUpdateWorkspacedMixin(BulkUpdateMixin):
@route('', methods=['PATCH'])
def bulk_update(self, workspace_name, **kwargs):
"""
---
tags: [{tag_name}]
summary: "Delete a group of {class_model} by ids."
responses:
204:
description: Ok
"""
return super().bulk_update(workspace_name=workspace_name)
def _bulk_update_query(self, ids, **kwargs):
workspace = self._get_workspace(kwargs["workspace_name"])
return super()._bulk_update_query(ids).filter(self.model_class.workspace_id == workspace.id)
class DeleteMixin:
"""Add DELETE /<id>/ route"""
def delete(self, object_id, **kwargs):
"""
---
tags: ["{tag_name}"]
summary: Deletes {class_model}
parameters:
- in: path
name: object_id
required: true
schema:
type: integer
responses:
204:
description: The resource was deleted successfully
"""
obj = self._get_object(object_id, **kwargs)
self._perform_delete(obj, **kwargs)
return None, 204
def _perform_delete(self, obj, workspace_name=None):
db.session.delete(obj)
db.session.commit()
class BulkDeleteMixin:
# These mixin should be merged with DeleteMixin after v2 is removed
@route('', methods=['DELETE'])
def bulk_delete(self, **kwargs):
"""
---
tags: [{tag_name}]
summary: "Delete a group of {class_model} by ids."
responses:
204:
description: Ok
"""
# TODO BULK_DELETE_SCHEMA
if not flask.request.json or 'ids' not in flask.request.json:
flask.abort(400)
# objs = self._get_objects(flask.request.json['ids'], **kwargs)
# self._perform_bulk_delete(objs, **kwargs)
ids = list(filter(lambda x: type(x) == self.lookup_field_type, flask.request.json['ids']))
return self._perform_bulk_delete(ids, **kwargs), 200
def _bulk_delete_query(self, ids, **kwargs):
# It IS better to as is but warn of ON CASCADE
return self.model_class.query.filter(self.model_class.id.in_(ids))
def _perform_bulk_delete(self, ids, **kwargs):
deleted = self._bulk_delete_query(ids, **kwargs).delete(synchronize_session='fetch')
db.session.commit()
response = {'deleted': deleted}
return flask.jsonify(response)
class DeleteWorkspacedMixin(DeleteMixin):
"""Add DELETE /<workspace_name>/<route_base>/<id>/ route"""
def delete(self, object_id, workspace_name=None):
"""
---
tags: ["{tag_name}"]
summary: Deletes {class_model}
parameters:
- in: path
name: object_id
required: true
schema:
type: integer
- in: path
name: workspace_name
required: true
schema:
type: string
responses:
204:
description: The resource was deleted successfully
"""
return super().delete(object_id, workspace_name=workspace_name)
def _perform_delete(self, obj, workspace_name=None):
with db.session.no_autoflush:
obj.workspace = self._get_workspace(workspace_name)
return super()._perform_delete(obj, workspace_name)
class BulkDeleteWorkspacedMixin(BulkDeleteMixin):
# These mixin should be merged with DeleteMixin after v2 is removed
@route('', methods=['DELETE'])
def bulk_delete(self, workspace_name, **kwargs):
"""
---
tags: [{tag_name}]
summary: "Delete a group of {class_model} by ids."
responses:
204:
description: Ok
"""
return super().bulk_delete(workspace_name=workspace_name)
def _bulk_delete_query(self, ids, **kwargs):
workspace = self._get_workspace(kwargs.pop("workspace_name"))
return super()._bulk_delete_query(ids).filter(self.model_class.workspace_id == workspace.id)
class CountWorkspacedMixin:
"""Add GET /<workspace_name>/<route_base>/count/ route
Group objects by the field set in the group_by GET parameter. If it
isn't specified, the view will return a 404 error. For each group,
show the count of elements and its value.
This view is often used by some parts of the web UI. It was designed
to keep backwards compatibility with the count endpoint of Faraday
v2.
"""
#: List of SQLAlchemy query filters to apply when counting
count_extra_filters = []
def count(self, **kwargs):
"""
---
tags: [{tag_name}]
summary: "Group {class_model} by the field set in the group_by GET parameter."
responses:
200:
description: Ok
content:
application/json:
schema: {schema_class}
404:
description: group_by is not specified
"""
res = {
'groups': [],
'total_count': 0
}
group_by = flask.request.args.get('group_by', None)
sort_dir = flask.request.args.get('order', "asc").lower()
# TODO migration: whitelist fields to avoid leaking a confidential
# field's value.
# Example: /users/count/?group_by=password
# Also we should check that the field exists in the db and isn't, for
# example, a relationship
if not group_by or group_by not in inspect(self.model_class).attrs:
flask.abort(400, {"message": "group_by is a required parameter"})
if sort_dir and sort_dir not in ('asc', 'desc'):
flask.abort(400, {"message": "order must be 'desc' or 'asc'"})
workspace_name = kwargs.pop('workspace_name')
# using format is not a great practice.
# the user input is group_by, however it's filtered by column name.
table_name = inspect(self.model_class).tables[0].name
group_by = f'{table_name}.{group_by}'
count = self._filter_query(
db.session.query(self.model_class)
.join(Workspace)
.group_by(group_by)
.filter(Workspace.name == workspace_name,
*self.count_extra_filters))
# order
order_by = group_by
if sort_dir == 'desc':
count = count.order_by(desc(order_by))
else:
count = count.order_by(asc(order_by))
for key, count in count.values(group_by, func.count(group_by)):
res['groups'].append(
{'count': count,
'name': key,
# To add compatibility with the web ui
flask.request.args.get('group_by'): key,
}
)
res['total_count'] += count
return res
class CountMultiWorkspacedMixin:
"""Add GET /<workspace_name>/<route_base>/count_multi_workspace/ route
Receives a list of workspaces separated by comma in the workspaces
GET parameter.
If no workspace is specified, the view will return a 400 error.
Group objects by the field set in the group_by GET parameter. If it
isn't specified, the view will return a 400 error. For each group,
show the count of elements and its value.
This view is often used by some parts of the web UI. It was designed
to keep backwards compatibility with the count endpoint of Faraday
v2.
"""
#: List of SQLAlchemy query filters to apply when counting
count_extra_filters = []
def count_multi_workspace(self, **kwargs):
"""
---
tags: [{tag_name}]
summary: "Count {class_model} by multiples workspaces"
responses:
200:
description: Ok
content:
application/json:
schema: {schema_class}
400:
description: No workspace passed or group_by is not specified
"""
# """head:
# tags: [{tag_name}]
# responses:
# 200:
# description: Ok
# options:
# tags: [{tag_name}]
# responses:
# 200:
# description: Ok
# """
res = {
'groups': defaultdict(dict),
'total_count': 0
}
workspace_names_list = flask.request.args.get('workspaces', None)
if not workspace_names_list:
flask.abort(400, {"message": "workspaces is a required parameter"})
workspace_names_list = workspace_names_list.split(',')
# Enforce workspace permission checking for each workspace
for workspace_name in workspace_names_list:
self._get_workspace(workspace_name)
group_by = flask.request.args.get('group_by', None)
sort_dir = flask.request.args.get('order', "asc").lower()
# TODO migration: whitelist fields to avoid leaking a confidential
# field's value.
# Example: /users/count/?group_by=password
# Also we should check that the field exists in the db and isn't, for
# example, a relationship
if not group_by or group_by not in inspect(self.model_class).attrs:
flask.abort(400, {"message": "group_by is a required parameter"})
if sort_dir and sort_dir not in ('asc', 'desc'):
flask.abort(400, {"message": "order must be 'desc' or 'asc'"})
grouped_attr = getattr(self.model_class, group_by)
q = db.session.query(
Workspace.name,
grouped_attr,
func.count(grouped_attr)
) \
.join(Workspace) \
.group_by(grouped_attr, Workspace.name) \
.filter(Workspace.name.in_(workspace_names_list))
# order
order_by = grouped_attr
if sort_dir == 'desc':
q = q.order_by(desc(Workspace.name), desc(order_by))
else:
q = q.order_by(asc(Workspace.name), asc(order_by))
for workspace, key, count in q.all():
res['groups'][workspace][key] = count
res['total_count'] += count
return res
class ReadWriteView(CreateMixin,
UpdateMixin,
DeleteMixin,
ReadOnlyView):
"""A generic view with list, retrieve and create endpoints
It is just a GenericView inheriting also from ListMixin,
RetrieveMixin, SortableMixin, CreateMixin, UpdateMixin and
DeleteMixin.
"""
class ReadWriteWorkspacedView(CreateWorkspacedMixin,
UpdateWorkspacedMixin,
DeleteWorkspacedMixin,
CountWorkspacedMixin,
ReadOnlyWorkspacedView):
"""A generic workspaced view with list, retrieve and create
endpoints
It is just a GenericWorkspacedView inheriting also from
ListWorkspacedMixin, RetrieveWorkspacedMixin, SortableMixin,
CreateWorkspacedMixin, DeleteWorkspacedMixin and
CountWorkspacedMixin.
"""
class CustomModelConverter(ModelConverter):
"""
Model converter that automatically sets minimum length
validators to not blankable fields
"""
def _add_column_kwargs(self, kwargs, column):
super()._add_column_kwargs(kwargs, column)
if not column.info.get('allow_blank', True):
kwargs['validate'].append(Length(min=1))
class CustomSQLAlchemyAutoSchemaOpts(SQLAlchemyAutoSchemaOpts):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.model_converter = CustomModelConverter
# Restore marshmallow's DateTime field behavior of marshmallow 2 so it adds
# "+00:00" to the serialized date. This ugly hack was done to keep our API
# backwards-compatible. Yes, it's horrible.
# Also, I'm putting it here because this file will be always imported in a very
# early stage, before defining any schemas.
# This commit broke backwards compatibility: https://github.com/marshmallow-code/marshmallow/commit/610ec20ea3be89684f7e4df8035d163c3561c904
# TODO check if we can remove this
def old_isoformat(dt, *args, **kwargs):
"""Return the ISO8601-formatted UTC representation of a datetime object."""
if dt.tzinfo is None:
dt = dt.replace(tzinfo=datetime.timezone.utc)
else:
dt = dt.astimezone(datetime.timezone.utc)
return dt.isoformat(*args, **kwargs)
fields.DateTime.SERIALIZATION_FUNCS['iso'] = old_isoformat
class AutoSchema(Schema, metaclass=SQLAlchemyAutoSchemaMeta):
"""
A Marshmallow schema that does field introspection based on
the SQLAlchemy model specified in Meta.model.
Unlike the marshmallow_sqlalchemy ModelSchema, it doesn't change
the serialization and deserialization proccess.
"""
OPTIONS_CLASS = CustomSQLAlchemyAutoSchemaOpts
# Use NullToBlankString instead of fields.String by default on text fields
TYPE_MAPPING = Schema.TYPE_MAPPING.copy()
TYPE_MAPPING[str] = NullToBlankString
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.unknown = EXCLUDE
class FilterAlchemyModelConverter(ModelConverter):
"""Use this to make all fields of a model not required.
It is used to make filteralchemy support not nullable columns"""
def _add_column_kwargs(self, kwargs, column):
super()._add_column_kwargs(kwargs, column)
kwargs['required'] = False
class AutoSchemaFlaskParser(FlaskParser):
# It is required to use a schema class that has unknown=EXCLUDE by default.
# Otherwise, requests would fail if a not defined query parameter is sent
# (like group_by)
DEFAULT_SCHEMA_CLASS = AutoSchema
class FilterSetMeta:
"""Base Meta class of FilterSet objects"""
parser = AutoSchemaFlaskParser(location='query')
converter = FilterAlchemyModelConverter()
def get_user_permissions(user):
permissions = defaultdict(dict)
# Hardcode all permisions to allowed
ALLOWED = {'allowed': True, 'reason': None}
# TODO schema
generic_entities = {
'licences', 'methodology_templates', 'task_templates', 'users',
'vulnerability_template', 'workspaces',
'agents', 'agents_schedules', 'commands', 'comments', 'hosts',
'executive_reports', 'services', 'methodologies', 'tasks', 'vulns',
'credentials',
}
for entity in generic_entities:
permissions[entity]['view'] = ALLOWED
permissions[entity]['create'] = ALLOWED
permissions[entity]['update'] = ALLOWED
permissions[entity]['delete'] = ALLOWED
extra_permissions = {
'vulns.status_change',
'settings.view',
'settings.update',
'ticketing.jira',
'ticketing.servicenow',
'bulk_create.bulk_create',
'agents.run',
'workspace_comparison.compare',
'data_analysis.view',
}
for permission in extra_permissions:
(entity, action) = permission.split('.')
permissions[entity][action] = ALLOWED
return permissions