Commit bfc588a9 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez

Several fixes

* Refactored BaseModel for efficiency
* Added plugin metaclass to keep track of plugin types
* Moved plugins to examples dir (in a previous commit)
* Simplified validation in parse_params
* Added convenience methods to mock requests in tests
* Changed help schema to use `.valid_parameters` instead of `.parameters`,
which was used in results to show parameters provided by the user.
* Improved UI
    * Added basic parameters
    * Fixed bugs in parameter handling
    * Refactored and cleaned code
parent f93eed2c
from senpy.plugins import AnalysisPlugin
import multiprocessing
def _train(process_number):
return process_number
class AsyncPlugin(AnalysisPlugin):
def _do_async(self, num_processes):
pool = multiprocessing.Pool(processes=num_processes)
values = pool.map(_train, range(num_processes))
return values
def activate(self):
self.value = self._do_async(4)
def analyse_entry(self, entry, params):
values = self._do_async(2)
entry.async_values = values
yield entry
def test(self):
pass
---
name: Async
module: asyncplugin
description: I am async
author: "@balkian"
version: '0.1'
async: true
extra_params: {}
\ No newline at end of file
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
def analyse_entry(self, entry, params):
entry['nif:isString'] = entry['nif:isString'][::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry
def test(self):
pass
{
"name": "Dummy",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": false,
"default": 0
}
}
}
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
description = 'This is a dummy self-contained plugin'
author = '@balkian'
version = '0.1'
def analyse_entry(self, entry, params):
entry['nif:isString'] = entry['nif:isString'][::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry
test_cases = [{
"entry": {
"nif:isString": "Hello world!"
},
"expected": [{
"nif:isString": "!dlrow olleH"
}]
}]
if __name__ == '__main__':
d = DummyPlugin()
d.test()
name: DummyNoInfo
module: dummy_noinfo
{
"name": "DummyRequired",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": true
}
}
}
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
import noop
from senpy.plugins import AnalysisPlugin
from time import sleep
class SleepPlugin(AnalysisPlugin):
def activate(self, *args, **kwargs):
sleep(self.timeout)
def analyse_entry(self, entry, params):
sleep(float(params.get("timeout", self.timeout)))
yield entry
def test(self):
pass
{
"name": "Sleep",
"module": "sleep",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"timeout": 0.05,
"extra_params": {
"timeout": {
"@id": "timeout_sleep",
"aliases": ["timeout", "to"],
"required": false,
"default": 0
}
}
}
...@@ -3,7 +3,6 @@ requests>=2.4.1 ...@@ -3,7 +3,6 @@ requests>=2.4.1
tornado>=4.4.3 tornado>=4.4.3
PyLD>=0.6.5 PyLD>=0.6.5
nltk nltk
six
future future
jsonschema jsonschema
jsonref jsonref
......
...@@ -19,6 +19,9 @@ Sentiment analysis server in Python ...@@ -19,6 +19,9 @@ Sentiment analysis server in Python
""" """
from .version import __version__ from .version import __version__
from future.standard_library import install_aliases
install_aliases()
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
...@@ -67,7 +67,7 @@ def main(): ...@@ -67,7 +67,7 @@ def main():
'--plugins-folder', '--plugins-folder',
'-f', '-f',
type=str, type=str,
default='plugins', default='.',
help='Where to look for plugins.') help='Where to look for plugins.')
parser.add_argument( parser.add_argument(
'--only-install', '--only-install',
......
...@@ -13,8 +13,9 @@ API_PARAMS = { ...@@ -13,8 +13,9 @@ API_PARAMS = {
"expanded-jsonld": { "expanded-jsonld": {
"@id": "expanded-jsonld", "@id": "expanded-jsonld",
"aliases": ["expanded"], "aliases": ["expanded"],
"options": "boolean",
"required": True, "required": True,
"default": 0 "default": False
}, },
"with_parameters": { "with_parameters": {
"aliases": ['withparameters', "aliases": ['withparameters',
...@@ -23,13 +24,6 @@ API_PARAMS = { ...@@ -23,13 +24,6 @@ API_PARAMS = {
"default": False, "default": False,
"required": True "required": True
}, },
"plugin_type": {
"@id": "pluginType",
"description": 'What kind of plugins to list',
"aliases": ["pluginType"],
"required": True,
"default": "analysisPlugin"
},
"outformat": { "outformat": {
"@id": "outformat", "@id": "outformat",
"aliases": ["o"], "aliases": ["o"],
...@@ -59,6 +53,16 @@ API_PARAMS = { ...@@ -59,6 +53,16 @@ API_PARAMS = {
} }
} }
PLUGINS_PARAMS = {
"plugin_type": {
"@id": "pluginType",
"description": 'What kind of plugins to list',
"aliases": ["pluginType"],
"required": True,
"default": 'analysisPlugin'
}
}
WEB_PARAMS = { WEB_PARAMS = {
"inHeaders": { "inHeaders": {
"aliases": ["headers"], "aliases": ["headers"],
...@@ -126,24 +130,26 @@ def parse_params(indict, *specs): ...@@ -126,24 +130,26 @@ def parse_params(indict, *specs):
wrong_params = {} wrong_params = {}
for spec in specs: for spec in specs:
for param, options in iteritems(spec): for param, options in iteritems(spec):
if param[0] != "@": # Exclude json-ld properties if param[0] == "@": # Exclude json-ld properties
for alias in options.get("aliases", []): continue
# Replace each alias with the correct name of the parameter for alias in options.get("aliases", []):
if alias in indict and alias is not param: # Replace each alias with the correct name of the parameter
outdict[param] = indict[alias] if alias in indict and alias is not param:
del indict[alias] outdict[param] = indict[alias]
continue del indict[alias]
if param not in outdict: continue
if options.get("required", False) and "default" not in options: if param not in outdict:
wrong_params[param] = spec[param] if "default" in options:
else: # We assume the default is correct
if "default" in options: outdict[param] = options["default"]
outdict[param] = options["default"] elif options.get("required", False):
elif "options" in spec[param]: wrong_params[param] = spec[param]
if spec[param]["options"] == "boolean": continue
outdict[param] = outdict[param] in [None, True, 'true', '1'] if "options" in options:
elif outdict[param] not in spec[param]["options"]: if options["options"] == "boolean":
wrong_params[param] = spec[param] outdict[param] = outdict[param] in [None, True, 'true', '1']
elif outdict[param] not in options["options"]:
wrong_params[param] = spec[param]
if wrong_params: if wrong_params:
logger.debug("Error parsing: %s", wrong_params) logger.debug("Error parsing: %s", wrong_params)
message = Error( message = Error(
...@@ -158,7 +164,7 @@ def parse_params(indict, *specs): ...@@ -158,7 +164,7 @@ def parse_params(indict, *specs):
return outdict return outdict
def get_extra_params(request, plugin=None): def parse_extra_params(request, plugin=None):
params = request.parameters.copy() params = request.parameters.copy()
if plugin: if plugin:
extra_params = parse_params(params, plugin.get('extra_params', {})) extra_params = parse_params(params, plugin.get('extra_params', {}))
...@@ -177,6 +183,6 @@ def parse_call(params): ...@@ -177,6 +183,6 @@ def parse_call(params):
elif params['informat'] == 'json-ld': elif params['informat'] == 'json-ld':
results = from_string(params['input'], cls=Results) results = from_string(params['input'], cls=Results)
else: else:
raise NotImplemented('Informat {} is not implemented'.format(params['informat'])) raise NotImplementedError('Informat {} is not implemented'.format(params['informat']))
results.parameters = params results.parameters = params
return results return results
...@@ -25,6 +25,7 @@ from .version import __version__ ...@@ -25,6 +25,7 @@ from .version import __version__
from functools import wraps from functools import wraps
import logging import logging
import traceback
import json import json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -72,12 +73,19 @@ def schema(schema="definitions"): ...@@ -72,12 +73,19 @@ def schema(schema="definitions"):
def basic_api(f): def basic_api(f):
default_params = {
'inHeaders': False,
'expanded-jsonld': False,
'outformat': 'json-ld',
'with_parameters': True,
}
@wraps(f) @wraps(f)
def decorated_function(*args, **kwargs): def decorated_function(*args, **kwargs):
raw_params = get_params(request) raw_params = get_params(request)
headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)} headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)}
params = default_params
outformat = 'json-ld'
try: try:
print('Getting request:') print('Getting request:')
print(request) print(request)
...@@ -87,26 +95,32 @@ def basic_api(f): ...@@ -87,26 +95,32 @@ def basic_api(f):
else: else:
request.parameters = params request.parameters = params
response = f(*args, **kwargs) response = f(*args, **kwargs)
except Error as ex: except (Exception) as ex:
response = ex
response.parameters = params
logger.error(ex)
if current_app.debug: if current_app.debug:
raise raise
if not isinstance(ex, Error):
msg = "{}:\n\t{}".format(ex,
traceback.format_exc())
ex = Error(message=msg, status=500)
logger.exception('Error returning analysis result')
response = ex
response.parameters = raw_params
logger.error(ex)
in_headers = params['inHeaders'] if 'parameters' in response and not params['with_parameters']:
expanded = params['expanded-jsonld'] print(response)
outformat = params['outformat'] print(response.data)
del response.parameters
return response.flask( return response.flask(
in_headers=in_headers, in_headers=params['inHeaders'],
headers=headers, headers=headers,
prefix=url_for('.api_root', _external=True), prefix=url_for('.api_root', _external=True),
context_uri=url_for('api.context', context_uri=url_for('api.context',
entity=type(response).__name__, entity=type(response).__name__,
_external=True), _external=True),
outformat=outformat, outformat=params['outformat'],
expanded=expanded) expanded=params['expanded-jsonld'])
return decorated_function return decorated_function
...@@ -116,19 +130,18 @@ def basic_api(f): ...@@ -116,19 +130,18 @@ def basic_api(f):
def api_root(): def api_root():
if request.parameters['help']: if request.parameters['help']:
dic = dict(api.API_PARAMS, **api.NIF_PARAMS) dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
response = Help(parameters=dic) response = Help(valid_parameters=dic)
return response
else:
req = api.parse_call(request.parameters)
response = current_app.senpy.analyse(req)
return response return response
req = api.parse_call(request.parameters)
return current_app.senpy.analyse(req)
@api_blueprint.route('/plugins/', methods=['POST', 'GET']) @api_blueprint.route('/plugins/', methods=['POST', 'GET'])
@basic_api @basic_api
def plugins(): def plugins():
sp = current_app.senpy sp = current_app.senpy
ptype = request.parameters.get('plugin_type') params = api.parse_params(request.parameters, api.PLUGINS_PARAMS)
ptype = params.get('plugin_type')
plugins = sp.filter_plugins(plugin_type=ptype) plugins = sp.filter_plugins(plugin_type=ptype)
dic = Plugins(plugins=list(plugins.values())) dic = Plugins(plugins=list(plugins.values()))
return dic return dic
......
import requests import requests
import logging import logging
from . import models from . import models
from .plugins import default_plugin_type
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -13,8 +12,8 @@ class Client(object): ...@@ -13,8 +12,8 @@ class Client(object):
def analyse(self, input, method='GET', **kwargs): def analyse(self, input, method='GET', **kwargs):
return self.request('/', method=method, input=input, **kwargs) return self.request('/', method=method, input=input, **kwargs)
def plugins(self, ptype=default_plugin_type): def plugins(self, *args, **kwargs):
resp = self.request(path='/plugins', plugin_type=ptype).plugins resp = self.request(path='/plugins').plugins
return {p.name: p for p in resp} return {p.name: p for p in resp}
def request(self, path=None, method='GET', **params): def request(self, path=None, method='GET', **params):
......
...@@ -123,7 +123,7 @@ class Senpy(object): ...@@ -123,7 +123,7 @@ class Senpy(object):
return return
plugin = plugins[0] plugin = plugins[0]
self._activate(plugin) # Make sure the plugin is activated self._activate(plugin) # Make sure the plugin is activated
specific_params = api.get_extra_params(req, plugin) specific_params = api.parse_extra_params(req, plugin)
req.analysis.append({'plugin': plugin, req.analysis.append({'plugin': plugin,
'parameters': specific_params}) 'parameters': specific_params})
results = plugin.analyse_entries(entries, specific_params) results = plugin.analyse_entries(entries, specific_params)
...@@ -262,17 +262,11 @@ class Senpy(object): ...@@ -262,17 +262,11 @@ class Senpy(object):
with plugin._lock: with plugin._lock:
if plugin.is_activated: if plugin.is_activated:
return return
try: plugin.activate()
plugin.activate() msg = "Plugin activated: {}".format(plugin.name)
msg = "Plugin activated: {}".format(plugin.name) logger.info(msg)
logger.info(msg) success = True
success = True self._set_active(plugin, success)
self._set_active(plugin, success)
except Exception as ex:
msg = "Error activating plugin {} - {} : \n\t{}".format(
plugin.name, ex, traceback.format_exc())
logger.error(msg)
raise Error(msg)
def activate_plugin(self, plugin_name, sync=True): def activate_plugin(self, plugin_name, sync=True):
try: try:
...@@ -294,13 +288,8 @@ class Senpy(object): ...@@ -294,13 +288,8 @@ class Senpy(object):
with plugin._lock: with plugin._lock:
if not plugin.is_activated: if not plugin.is_activated:
return return
try: plugin.deactivate()
plugin.deactivate() logger.info("Plugin deactivated: {}".format(plugin.name))
logger.info("Plugin deactivated: {}".format(plugin.name))
except Exception as ex:
logger.error(
"Error deactivating plugin {}: {}".format(plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc()))
def deactivate_plugin(self, plugin_name, sync=True): def deactivate_plugin(self, plugin_name, sync=True):
try: try:
......
This diff is collapsed.
from future import standard_library from future import standard_library
standard_library.install_aliases() standard_library.install_aliases()
from future.utils import with_metaclass
import os.path import os.path
import os import os
...@@ -16,21 +17,33 @@ import yaml ...@@ -16,21 +17,33 @@ import yaml
import threading import threading
from .. import models, utils from .. import models, utils
from ..api import API_PARAMS from .. import api
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class Plugin(models.Plugin): class PluginMeta(models.BaseMeta):
def __init__(self, info=None, data_folder=None):
def __new__(mcs, name, bases, attrs, **kwargs):
plugin_type = []
if hasattr(bases[0], 'plugin_type'):
plugin_type += bases[0].plugin_type
plugin_type.append(name)
attrs['plugin_type'] = plugin_type
return super(PluginMeta, mcs).__new__(mcs, name, bases, attrs)
class Plugin(with_metaclass(PluginMeta, models.Plugin)):
def __init__(self, info=None, data_folder=None, **kwargs):
""" """
Provides a canonical name for plugins and serves as base for other Provides a canonical name for plugins and serves as base for other
kinds of plugins. kinds of plugins.
""" """
logger.debug("Initialising {}".format(info)) logger.debug("Initialising {}".format(info))
super(Plugin, self).__init__(**kwargs)
if info: if info:
self.update(info) self.update(info)
super(Plugin, self).__init__(**self)
if not self.validate(): if not self.validate():
raise models.Error(message=("You need to provide configuration" raise models.Error(message=("You need to provide configuration"
"information for the plugin.")) "information for the plugin."))
...@@ -57,7 +70,8 @@ class Plugin(models.Plugin): ...@@ -57,7 +70,8 @@ class Plugin(models.Plugin):
'test cases').format(self.id, inspect.getfile(self.__class__))) 'test cases').format(self.id, inspect.getfile(self.__class__)))
for case in self.test_cases: for case in self.test_cases:
entry = models.Entry(case['entry']) entry = models.Entry(case['entry'])
params = case.get('params', {}) given_parameters = case.get('params', {})
params = api.parse_params(given_parameters, self.extra_params)
fails = case.get('fails', False) fails = case.get('fails', False)
try: try:
res = list(self.analyse_entry(entry, params)) res = list(self.analyse_entry(entry, params))
...@@ -90,7 +104,7 @@ SenpyPlugin = Plugin ...@@ -90,7 +104,7 @@ SenpyPlugin = Plugin
class AnalysisPlugin(Plugin): class AnalysisPlugin(Plugin):
def analyse(self, *args, **kwargs): def analyse(self, *args, **kwargs):
raise NotImplemented( raise NotImplementedError(
'Your method should implement either analyse or analyse_entry') 'Your method should implement either analyse or analyse_entry')
def analyse_entry(self, entry, parameters): def analyse_entry(self, entry, parameters):
...@@ -118,17 +132,17 @@ class ConversionPlugin(Plugin): ...@@ -118,17 +132,17 @@ class ConversionPlugin(Plugin):
pass pass