Commit a8614bab authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Accept plugin pipelines

Closes #15
parent 70ca74b0
...@@ -113,8 +113,11 @@ def basic_api(f): ...@@ -113,8 +113,11 @@ def basic_api(f):
@api_blueprint.route('/', methods=['POST', 'GET']) @api_blueprint.route('/', methods=['POST', 'GET'])
@basic_api @basic_api
def api(): def api():
response = current_app.senpy.analyse(**request.params) try:
return response response = current_app.senpy.analyse(**request.params)
return response
except Error as ex:
return ex
@api_blueprint.route('/plugins/', methods=['POST', 'GET']) @api_blueprint.route('/plugins/', methods=['POST', 'GET'])
......
...@@ -7,7 +7,7 @@ standard_library.install_aliases() ...@@ -7,7 +7,7 @@ standard_library.install_aliases()
from . import plugins from . import plugins
from .plugins import SenpyPlugin from .plugins import SenpyPlugin
from .models import Error, Entry, Results from .models import Error, Entry, Results, from_dict
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
from .api import API_PARAMS, NIF_PARAMS, parse_params from .api import API_PARAMS, NIF_PARAMS, parse_params
...@@ -78,70 +78,101 @@ class Senpy(object): ...@@ -78,70 +78,101 @@ class Senpy(object):
else: else:
logger.debug("Not a folder: %s", folder) logger.debug("Not a folder: %s", folder)
def _find_plugin(self, params): def _find_plugins(self, params):
api_params = parse_params(params, spec=API_PARAMS) if not self.analysis_plugins:
algo = None
if "algorithm" in api_params and api_params["algorithm"]:
algo = api_params["algorithm"]
elif self.plugins:
algo = self.default_plugin and self.default_plugin.name
if not algo:
raise Error( raise Error(
status=404, status=404,
message=("No plugins found." message=("No plugins found."
" Please install one.").format(algo)) " Please install one."))
if algo not in self.plugins: api_params = parse_params(params, spec=API_PARAMS)
logger.debug(("The algorithm '{}' is not valid\n" algos = None
"Valid algorithms: {}").format(algo, if "algorithm" in api_params and api_params["algorithm"]:
self.plugins.keys())) algos = api_params["algorithm"].split(',')
elif self.default_plugin:
algos = [self.default_plugin.name, ]
else:
raise Error( raise Error(
status=404, status=404,
message="The algorithm '{}' is not valid".format(algo)) message="No default plugin found, and None provided")
if not self.plugins[algo].is_activated: plugins = list()
logger.debug("Plugin not activated: {}".format(algo)) for algo in algos:
raise Error( if algo not in self.plugins:
status=400, logger.debug(("The algorithm '{}' is not valid\n"
message=("The algorithm '{}'" "Valid algorithms: {}").format(algo,
" is not activated yet").format(algo)) self.plugins.keys()))
return self.plugins[algo] raise Error(
status=404,
message="The algorithm '{}' is not valid".format(algo))
if not self.plugins[algo].is_activated:
logger.debug("Plugin not activated: {}".format(algo))
raise Error(
status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
plugins.append(self.plugins[algo])
return plugins
def _get_params(self, params, plugin): def _get_params(self, params, plugin=None):
nif_params = parse_params(params, spec=NIF_PARAMS) nif_params = parse_params(params, spec=NIF_PARAMS)
extra_params = plugin.get('extra_params', {}) if plugin:
specific_params = parse_params(params, spec=extra_params) extra_params = plugin.get('extra_params', {})
nif_params.update(specific_params) specific_params = parse_params(params, spec=extra_params)
nif_params.update(specific_params)
return nif_params return nif_params
def _get_entries(self, params): def _get_entries(self, params):
entry = None
if params['informat'] == 'text': if params['informat'] == 'text':
results = Results()
entry = Entry(text=params['input']) entry = Entry(text=params['input'])
results.entries.append(entry)
elif params['informat'] == 'json-ld':
results = from_dict(params['input'])
else: else:
raise NotImplemented('Only text input format implemented') raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
yield entry return results
def _process_entries(self, entries, plugins, nif_params):
if not plugins:
for i in entries:
yield i
return
plugin = plugins[0]
specific_params = self._get_params(nif_params, plugin)
results = plugin.analyse_entries(entries, specific_params)
for i in self._process_entries(results, plugins[1:], nif_params):
yield i
def _process_response(self, resp, plugins, nif_params):
entries = resp.entries
resp.entries = []
for plug in plugins:
resp.analysis.append(plug.id)
for i in self._process_entries(entries, plugins, nif_params):
resp.entries.append(i)
return resp
def analyse(self, **api_params): def analyse(self, **api_params):
"""
Main method that analyses a request, either from CLI or HTTP.
It uses a dictionary of parameters, provided by the user.
"""
logger.debug("analysing with params: {}".format(api_params)) logger.debug("analysing with params: {}".format(api_params))
plugin = self._find_plugin(api_params) plugins = self._find_plugins(api_params)
nif_params = self._get_params(api_params, plugin) nif_params = self._get_params(api_params)
resp = Results() resp = self._get_entries(nif_params)
if 'with_parameters' in api_params: if 'with_parameters' in api_params:
resp.parameters = nif_params resp.parameters = nif_params
try: try:
entries = [] resp = self._process_response(resp, plugins, nif_params)
for i in self._get_entries(nif_params): self.convert_emotions(resp, plugins, nif_params)
entries += list(plugin.analyse_entry(i, nif_params))
resp.entries = entries
self.convert_emotions(resp, plugin, nif_params)
resp.analysis.append(plugin.id)
logger.debug("Returning analysis result: {}".format(resp)) logger.debug("Returning analysis result: {}".format(resp))
except Error as ex: except (Error, Exception) as ex:
logger.exception('Error returning analysis result') if not isinstance(ex, Error):
resp = ex ex = Error(message=str(ex), status=500)
except Exception as ex:
logger.exception('Error returning analysis result') logger.exception('Error returning analysis result')
resp = Error(message=str(ex), status=500) raise ex
return resp return resp
def _conversion_candidates(self, fromModel, toModel): def _conversion_candidates(self, fromModel, toModel):
...@@ -155,7 +186,7 @@ class Senpy(object): ...@@ -155,7 +186,7 @@ class Senpy(object):
# logging.debug('Found candidate: {}'.format(candidate)) # logging.debug('Found candidate: {}'.format(candidate))
yield candidate yield candidate
def convert_emotions(self, resp, plugin, params): def convert_emotions(self, resp, plugins, params):
""" """
Conversion of all emotions in a response. Conversion of all emotions in a response.
In addition to converting from one model to another, it has In addition to converting from one model to another, it has
...@@ -163,29 +194,35 @@ class Senpy(object): ...@@ -163,29 +194,35 @@ class Senpy(object):
Needless to say, this is far from an elegant solution, but it works. Needless to say, this is far from an elegant solution, but it works.
@todo refactor and clean up @todo refactor and clean up
""" """
fromModel = plugin.get('onyx:usesEmotionModel', None)
toModel = params.get('emotionModel', None) toModel = params.get('emotionModel', None)
output = params.get('conversion', None)
logger.debug('Asked for model: {}'.format(toModel))
logger.debug('Analysis plugin uses model: {}'.format(fromModel))
if not toModel: if not toModel:
return return
try:
candidate = next(self._conversion_candidates(fromModel, toModel)) logger.debug('Asked for model: {}'.format(toModel))
except StopIteration: output = params.get('conversion', None)
e = Error(('No conversion plugin found for: ' candidates = {}
'{} -> {}'.format(fromModel, toModel))) for plugin in plugins:
e.original_response = resp try:
e.parameters = params fromModel = plugin.get('onyx:usesEmotionModel', None)
raise e candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
except StopIteration:
e = Error(('No conversion plugin found for: '
'{} -> {}'.format(fromModel, toModel)))
e.original_response = resp
e.parameters = params
raise e
newentries = [] newentries = []
resp.analysis = set(resp.analysis)
for i in resp.entries: for i in resp.entries:
if output == "full": if output == "full":
newemotions = copy.deepcopy(i.emotions) newemotions = copy.deepcopy(i.emotions)
else: else:
newemotions = [] newemotions = []
for j in i.emotions: for j in i.emotions:
plugname = j['prov:wasGeneratedBy']
candidate = candidates[plugname]
resp.analysis.add(candidate.id)
for k in candidate.convert(j, fromModel, toModel, params): for k in candidate.convert(j, fromModel, toModel, params):
k.prov__wasGeneratedBy = candidate.id k.prov__wasGeneratedBy = candidate.id
if output == 'nested': if output == 'nested':
...@@ -194,7 +231,6 @@ class Senpy(object): ...@@ -194,7 +231,6 @@ class Senpy(object):
i.emotions = newemotions i.emotions = newemotions
newentries.append(i) newentries.append(i)
resp.entries = newentries resp.entries = newentries
resp.analysis.append(candidate.id)
@property @property
def default_plugin(self): def default_plugin(self):
......
...@@ -57,6 +57,12 @@ class AnalysisPlugin(SenpyPlugin): ...@@ -57,6 +57,12 @@ class AnalysisPlugin(SenpyPlugin):
for i in results.entries: for i in results.entries:
yield i yield i
def analyse_entries(self, entries, parameters):
for entry in entries:
logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
for result in self.analyse_entry(entry, parameters):
yield result
class ConversionPlugin(SenpyPlugin): class ConversionPlugin(SenpyPlugin):
pass pass
......
...@@ -4,4 +4,5 @@ from senpy.plugins import SentimentPlugin ...@@ -4,4 +4,5 @@ from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin): class DummyPlugin(SentimentPlugin):
def analyse_entry(self, entry, params): def analyse_entry(self, entry, params):
entry.text = entry.text[::-1] entry.text = entry.text[::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry yield entry
...@@ -10,7 +10,7 @@ except ImportError: ...@@ -10,7 +10,7 @@ except ImportError:
from functools import partial from functools import partial
from senpy.extensions import Senpy from senpy.extensions import Senpy
from senpy.models import Error, Results, Entry, EmotionSet, Emotion from senpy.models import Error, Results, Entry, EmotionSet, Emotion, Plugin
from flask import Flask from flask import Flask
from unittest import TestCase from unittest import TestCase
...@@ -98,17 +98,26 @@ class ExtensionsTest(TestCase): ...@@ -98,17 +98,26 @@ class ExtensionsTest(TestCase):
def test_analyse_error(self): def test_analyse_error(self):
mm = mock.MagicMock() mm = mock.MagicMock()
mm.analyse_entry.side_effect = Error('error on analysis', status=900) mm.id = 'magic_mock'
mm.analyse_entries.side_effect = Error('error on analysis', status=500)
self.senpy.plugins['MOCK'] = mm self.senpy.plugins['MOCK'] = mm
resp = self.senpy.analyse(input='nothing', algorithm='MOCK') try:
assert resp['message'] == 'error on analysis' self.senpy.analyse(input='nothing', algorithm='MOCK')
assert resp['status'] == 900 assert False
except Error as ex:
assert ex['message'] == 'error on analysis'
assert ex['status'] == 500
mm.analyse.side_effect = Exception('generic exception on analysis') mm.analyse.side_effect = Exception('generic exception on analysis')
mm.analyse_entry.side_effect = Exception( mm.analyse_entries.side_effect = Exception(
'generic exception on analysis') 'generic exception on analysis')
resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
assert resp['message'] == 'generic exception on analysis' try:
assert resp['status'] == 500 self.senpy.analyse(input='nothing', algorithm='MOCK')
assert False
except Error as ex:
assert ex['message'] == 'generic exception on analysis'
assert ex['status'] == 500
def test_filtering(self): def test_filtering(self):
""" Filtering plugins """ """ Filtering plugins """
...@@ -125,11 +134,12 @@ class ExtensionsTest(TestCase): ...@@ -125,11 +134,12 @@ class ExtensionsTest(TestCase):
def test_convert_emotions(self): def test_convert_emotions(self):
self.senpy.activate_all() self.senpy.activate_all()
plugin = { plugin = Plugin({
'id': 'imaginary', 'id': 'imaginary',
'onyx:usesEmotionModel': 'emoml:fsre-dimensions' 'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
} })
eSet1 = EmotionSet() eSet1 = EmotionSet()
eSet1.prov__wasGeneratedBy = plugin['id']
eSet1['onyx:hasEmotion'].append(Emotion({ eSet1['onyx:hasEmotion'].append(Emotion({
'emoml:arousal': 1, 'emoml:arousal': 1,
'emoml:potency': 0, 'emoml:potency': 0,
...@@ -145,19 +155,19 @@ class ExtensionsTest(TestCase): ...@@ -145,19 +155,19 @@ class ExtensionsTest(TestCase):
'conversion': 'full'} 'conversion': 'full'}
r1 = deepcopy(response) r1 = deepcopy(response)
self.senpy.convert_emotions(r1, self.senpy.convert_emotions(r1,
plugin, [plugin, ],
params) params)
assert len(r1.entries[0].emotions) == 2 assert len(r1.entries[0].emotions) == 2
params['conversion'] = 'nested' params['conversion'] = 'nested'
r2 = deepcopy(response) r2 = deepcopy(response)
self.senpy.convert_emotions(r2, self.senpy.convert_emotions(r2,
plugin, [plugin, ],
params) params)
assert len(r2.entries[0].emotions) == 1 assert len(r2.entries[0].emotions) == 1
assert r2.entries[0].emotions[0]['prov:wasDerivedFrom'] == eSet1 assert r2.entries[0].emotions[0]['prov:wasDerivedFrom'] == eSet1
params['conversion'] = 'filtered' params['conversion'] = 'filtered'
r3 = deepcopy(response) r3 = deepcopy(response)
self.senpy.convert_emotions(r3, self.senpy.convert_emotions(r3,
plugin, [plugin, ],
params) params)
assert len(r3.entries[0].emotions) == 1 assert len(r3.entries[0].emotions) == 1
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment