Commit a8614bab authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Accept plugin pipelines

Closes #15
parent 70ca74b0
......@@ -113,8 +113,11 @@ def basic_api(f):
@api_blueprint.route('/', methods=['POST', 'GET'])
def api():
response = current_app.senpy.analyse(**request.params)
return response
except Error as ex:
return ex
@api_blueprint.route('/plugins/', methods=['POST', 'GET'])
......@@ -7,7 +7,7 @@ standard_library.install_aliases()
from . import plugins
from .plugins import SenpyPlugin
from .models import Error, Entry, Results
from .models import Error, Entry, Results, from_dict
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
from .api import API_PARAMS, NIF_PARAMS, parse_params
......@@ -78,18 +78,25 @@ class Senpy(object):
logger.debug("Not a folder: %s", folder)
def _find_plugin(self, params):
def _find_plugins(self, params):
if not self.analysis_plugins:
raise Error(
message=("No plugins found."
" Please install one."))
api_params = parse_params(params, spec=API_PARAMS)
algo = None
algos = None
if "algorithm" in api_params and api_params["algorithm"]:
algo = api_params["algorithm"]
elif self.plugins:
algo = self.default_plugin and
if not algo:
algos = api_params["algorithm"].split(',')
elif self.default_plugin:
algos = [, ]
raise Error(
message=("No plugins found."
" Please install one.").format(algo))
message="No default plugin found, and None provided")
plugins = list()
for algo in algos:
if algo not in self.plugins:
logger.debug(("The algorithm '{}' is not valid\n"
"Valid algorithms: {}").format(algo,
......@@ -104,44 +111,68 @@ class Senpy(object):
message=("The algorithm '{}'"
" is not activated yet").format(algo))
return self.plugins[algo]
return plugins
def _get_params(self, params, plugin):
def _get_params(self, params, plugin=None):
nif_params = parse_params(params, spec=NIF_PARAMS)
if plugin:
extra_params = plugin.get('extra_params', {})
specific_params = parse_params(params, spec=extra_params)
return nif_params
def _get_entries(self, params):
entry = None
if params['informat'] == 'text':
results = Results()
entry = Entry(text=params['input'])
elif params['informat'] == 'json-ld':
results = from_dict(params['input'])
raise NotImplemented('Only text input format implemented')
yield entry
raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
return results
def _process_entries(self, entries, plugins, nif_params):
if not plugins:
for i in entries:
yield i
plugin = plugins[0]
specific_params = self._get_params(nif_params, plugin)
results = plugin.analyse_entries(entries, specific_params)
for i in self._process_entries(results, plugins[1:], nif_params):
yield i
def _process_response(self, resp, plugins, nif_params):
entries = resp.entries
resp.entries = []
for plug in plugins:
for i in self._process_entries(entries, plugins, nif_params):
return resp
def analyse(self, **api_params):
Main method that analyses a request, either from CLI or HTTP.
It uses a dictionary of parameters, provided by the user.
logger.debug("analysing with params: {}".format(api_params))
plugin = self._find_plugin(api_params)
nif_params = self._get_params(api_params, plugin)
resp = Results()
plugins = self._find_plugins(api_params)
nif_params = self._get_params(api_params)
resp = self._get_entries(nif_params)
if 'with_parameters' in api_params:
resp.parameters = nif_params
entries = []
for i in self._get_entries(nif_params):
entries += list(plugin.analyse_entry(i, nif_params))
resp.entries = entries
self.convert_emotions(resp, plugin, nif_params)
resp = self._process_response(resp, plugins, nif_params)
self.convert_emotions(resp, plugins, nif_params)
logger.debug("Returning analysis result: {}".format(resp))
except Error as ex:
logger.exception('Error returning analysis result')
resp = ex
except Exception as ex:
except (Error, Exception) as ex:
if not isinstance(ex, Error):
ex = Error(message=str(ex), status=500)
logger.exception('Error returning analysis result')
resp = Error(message=str(ex), status=500)
raise ex
return resp
def _conversion_candidates(self, fromModel, toModel):
......@@ -155,7 +186,7 @@ class Senpy(object):
# logging.debug('Found candidate: {}'.format(candidate))
yield candidate
def convert_emotions(self, resp, plugin, params):
def convert_emotions(self, resp, plugins, params):
Conversion of all emotions in a response.
In addition to converting from one model to another, it has
......@@ -163,16 +194,18 @@ class Senpy(object):
Needless to say, this is far from an elegant solution, but it works.
@todo refactor and clean up
fromModel = plugin.get('onyx:usesEmotionModel', None)
toModel = params.get('emotionModel', None)
output = params.get('conversion', None)
logger.debug('Asked for model: {}'.format(toModel))
logger.debug('Analysis plugin uses model: {}'.format(fromModel))
if not toModel:
logger.debug('Asked for model: {}'.format(toModel))
output = params.get('conversion', None)
candidates = {}
for plugin in plugins:
candidate = next(self._conversion_candidates(fromModel, toModel))
fromModel = plugin.get('onyx:usesEmotionModel', None)
candidates[] = next(self._conversion_candidates(fromModel, toModel))
logger.debug('Analysis plugin {} uses model: {}'.format(, fromModel))
except StopIteration:
e = Error(('No conversion plugin found for: '
'{} -> {}'.format(fromModel, toModel)))
......@@ -180,12 +213,16 @@ class Senpy(object):
e.parameters = params
raise e
newentries = []
resp.analysis = set(resp.analysis)
for i in resp.entries:
if output == "full":
newemotions = copy.deepcopy(i.emotions)
newemotions = []
for j in i.emotions:
plugname = j['prov:wasGeneratedBy']
candidate = candidates[plugname]
for k in candidate.convert(j, fromModel, toModel, params):
k.prov__wasGeneratedBy =
if output == 'nested':
......@@ -194,7 +231,6 @@ class Senpy(object):
i.emotions = newemotions
resp.entries = newentries
def default_plugin(self):
......@@ -57,6 +57,12 @@ class AnalysisPlugin(SenpyPlugin):
for i in results.entries:
yield i
def analyse_entries(self, entries, parameters):
for entry in entries:
logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
for result in self.analyse_entry(entry, parameters):
yield result
class ConversionPlugin(SenpyPlugin):
......@@ -4,4 +4,5 @@ from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
def analyse_entry(self, entry, params):
entry.text = entry.text[::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry
......@@ -10,7 +10,7 @@ except ImportError:
from functools import partial
from senpy.extensions import Senpy
from senpy.models import Error, Results, Entry, EmotionSet, Emotion
from senpy.models import Error, Results, Entry, EmotionSet, Emotion, Plugin
from flask import Flask
from unittest import TestCase
......@@ -98,17 +98,26 @@ class ExtensionsTest(TestCase):
def test_analyse_error(self):
mm = mock.MagicMock()
mm.analyse_entry.side_effect = Error('error on analysis', status=900) = 'magic_mock'
mm.analyse_entries.side_effect = Error('error on analysis', status=500)
self.senpy.plugins['MOCK'] = mm
resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
assert resp['message'] == 'error on analysis'
assert resp['status'] == 900
self.senpy.analyse(input='nothing', algorithm='MOCK')
assert False
except Error as ex:
assert ex['message'] == 'error on analysis'
assert ex['status'] == 500
mm.analyse.side_effect = Exception('generic exception on analysis')
mm.analyse_entry.side_effect = Exception(
mm.analyse_entries.side_effect = Exception(
'generic exception on analysis')
resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
assert resp['message'] == 'generic exception on analysis'
assert resp['status'] == 500
self.senpy.analyse(input='nothing', algorithm='MOCK')
assert False
except Error as ex:
assert ex['message'] == 'generic exception on analysis'
assert ex['status'] == 500
def test_filtering(self):
""" Filtering plugins """
......@@ -125,11 +134,12 @@ class ExtensionsTest(TestCase):
def test_convert_emotions(self):
plugin = {
plugin = Plugin({
'id': 'imaginary',
'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
eSet1 = EmotionSet()
eSet1.prov__wasGeneratedBy = plugin['id']
'emoml:arousal': 1,
'emoml:potency': 0,
......@@ -145,19 +155,19 @@ class ExtensionsTest(TestCase):
'conversion': 'full'}
r1 = deepcopy(response)
[plugin, ],
assert len(r1.entries[0].emotions) == 2
params['conversion'] = 'nested'
r2 = deepcopy(response)
[plugin, ],
assert len(r2.entries[0].emotions) == 1
assert r2.entries[0].emotions[0]['prov:wasDerivedFrom'] == eSet1
params['conversion'] = 'filtered'
r3 = deepcopy(response)
[plugin, ],
assert len(r3.entries[0].emotions) == 1
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment