Commit bfc588a9 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez

Several fixes

* Refactored BaseModel for efficiency
* Added plugin metaclass to keep track of plugin types
* Moved plugins to examples dir (in a previous commit)
* Simplified validation in parse_params
* Added convenience methods to mock requests in tests
* Changed help schema to use `.valid_parameters` instead of `.parameters`,
which was used in results to show parameters provided by the user.
* Improved UI
    * Added basic parameters
    * Fixed bugs in parameter handling
    * Refactored and cleaned code
parent f93eed2c
from senpy.plugins import AnalysisPlugin
import multiprocessing
def _train(process_number):
return process_number
class AsyncPlugin(AnalysisPlugin):
def _do_async(self, num_processes):
pool = multiprocessing.Pool(processes=num_processes)
values = pool.map(_train, range(num_processes))
return values
def activate(self):
self.value = self._do_async(4)
def analyse_entry(self, entry, params):
values = self._do_async(2)
entry.async_values = values
yield entry
def test(self):
pass
---
name: Async
module: asyncplugin
description: I am async
author: "@balkian"
version: '0.1'
async: true
extra_params: {}
\ No newline at end of file
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
def analyse_entry(self, entry, params):
entry['nif:isString'] = entry['nif:isString'][::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry
def test(self):
pass
{
"name": "Dummy",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": false,
"default": 0
}
}
}
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
description = 'This is a dummy self-contained plugin'
author = '@balkian'
version = '0.1'
def analyse_entry(self, entry, params):
entry['nif:isString'] = entry['nif:isString'][::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry
test_cases = [{
"entry": {
"nif:isString": "Hello world!"
},
"expected": [{
"nif:isString": "!dlrow olleH"
}]
}]
if __name__ == '__main__':
d = DummyPlugin()
d.test()
name: DummyNoInfo
module: dummy_noinfo
{
"name": "DummyRequired",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": true
}
}
}
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
import noop
from senpy.plugins import AnalysisPlugin
from time import sleep
class SleepPlugin(AnalysisPlugin):
def activate(self, *args, **kwargs):
sleep(self.timeout)
def analyse_entry(self, entry, params):
sleep(float(params.get("timeout", self.timeout)))
yield entry
def test(self):
pass
{
"name": "Sleep",
"module": "sleep",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"timeout": 0.05,
"extra_params": {
"timeout": {
"@id": "timeout_sleep",
"aliases": ["timeout", "to"],
"required": false,
"default": 0
}
}
}
......@@ -3,7 +3,6 @@ requests>=2.4.1
tornado>=4.4.3
PyLD>=0.6.5
nltk
six
future
jsonschema
jsonref
......
......@@ -19,6 +19,9 @@ Sentiment analysis server in Python
"""
from .version import __version__
from future.standard_library import install_aliases
install_aliases()
import logging
logger = logging.getLogger(__name__)
......
......@@ -67,7 +67,7 @@ def main():
'--plugins-folder',
'-f',
type=str,
default='plugins',
default='.',
help='Where to look for plugins.')
parser.add_argument(
'--only-install',
......
......@@ -13,8 +13,9 @@ API_PARAMS = {
"expanded-jsonld": {
"@id": "expanded-jsonld",
"aliases": ["expanded"],
"options": "boolean",
"required": True,
"default": 0
"default": False
},
"with_parameters": {
"aliases": ['withparameters',
......@@ -23,13 +24,6 @@ API_PARAMS = {
"default": False,
"required": True
},
"plugin_type": {
"@id": "pluginType",
"description": 'What kind of plugins to list',
"aliases": ["pluginType"],
"required": True,
"default": "analysisPlugin"
},
"outformat": {
"@id": "outformat",
"aliases": ["o"],
......@@ -59,6 +53,16 @@ API_PARAMS = {
}
}
PLUGINS_PARAMS = {
"plugin_type": {
"@id": "pluginType",
"description": 'What kind of plugins to list',
"aliases": ["pluginType"],
"required": True,
"default": 'analysisPlugin'
}
}
WEB_PARAMS = {
"inHeaders": {
"aliases": ["headers"],
......@@ -126,24 +130,26 @@ def parse_params(indict, *specs):
wrong_params = {}
for spec in specs:
for param, options in iteritems(spec):
if param[0] != "@": # Exclude json-ld properties
for alias in options.get("aliases", []):
# Replace each alias with the correct name of the parameter
if alias in indict and alias is not param:
outdict[param] = indict[alias]
del indict[alias]
continue
if param not in outdict:
if options.get("required", False) and "default" not in options:
wrong_params[param] = spec[param]
else:
if "default" in options:
outdict[param] = options["default"]
elif "options" in spec[param]:
if spec[param]["options"] == "boolean":
outdict[param] = outdict[param] in [None, True, 'true', '1']
elif outdict[param] not in spec[param]["options"]:
wrong_params[param] = spec[param]
if param[0] == "@": # Exclude json-ld properties
continue
for alias in options.get("aliases", []):
# Replace each alias with the correct name of the parameter
if alias in indict and alias is not param:
outdict[param] = indict[alias]
del indict[alias]
continue
if param not in outdict:
if "default" in options:
# We assume the default is correct
outdict[param] = options["default"]
elif options.get("required", False):
wrong_params[param] = spec[param]
continue
if "options" in options:
if options["options"] == "boolean":
outdict[param] = outdict[param] in [None, True, 'true', '1']
elif outdict[param] not in options["options"]:
wrong_params[param] = spec[param]
if wrong_params:
logger.debug("Error parsing: %s", wrong_params)
message = Error(
......@@ -158,7 +164,7 @@ def parse_params(indict, *specs):
return outdict
def get_extra_params(request, plugin=None):
def parse_extra_params(request, plugin=None):
params = request.parameters.copy()
if plugin:
extra_params = parse_params(params, plugin.get('extra_params', {}))
......@@ -177,6 +183,6 @@ def parse_call(params):
elif params['informat'] == 'json-ld':
results = from_string(params['input'], cls=Results)
else:
raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
raise NotImplementedError('Informat {} is not implemented'.format(params['informat']))
results.parameters = params
return results
......@@ -25,6 +25,7 @@ from .version import __version__
from functools import wraps
import logging
import traceback
import json
logger = logging.getLogger(__name__)
......@@ -72,12 +73,19 @@ def schema(schema="definitions"):
def basic_api(f):
default_params = {
'inHeaders': False,
'expanded-jsonld': False,
'outformat': 'json-ld',
'with_parameters': True,
}
@wraps(f)
def decorated_function(*args, **kwargs):
raw_params = get_params(request)
headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)}
params = default_params
outformat = 'json-ld'
try:
print('Getting request:')
print(request)
......@@ -87,26 +95,32 @@ def basic_api(f):
else:
request.parameters = params
response = f(*args, **kwargs)
except Error as ex:
response = ex
response.parameters = params
logger.error(ex)
except (Exception) as ex:
if current_app.debug:
raise
if not isinstance(ex, Error):
msg = "{}:\n\t{}".format(ex,
traceback.format_exc())
ex = Error(message=msg, status=500)
logger.exception('Error returning analysis result')
response = ex
response.parameters = raw_params
logger.error(ex)
in_headers = params['inHeaders']
expanded = params['expanded-jsonld']
outformat = params['outformat']
if 'parameters' in response and not params['with_parameters']:
print(response)
print(response.data)
del response.parameters
return response.flask(
in_headers=in_headers,
in_headers=params['inHeaders'],
headers=headers,
prefix=url_for('.api_root', _external=True),
context_uri=url_for('api.context',
entity=type(response).__name__,
_external=True),
outformat=outformat,
expanded=expanded)
outformat=params['outformat'],
expanded=params['expanded-jsonld'])
return decorated_function
......@@ -116,19 +130,18 @@ def basic_api(f):
def api_root():
if request.parameters['help']:
dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
response = Help(parameters=dic)
return response
else:
req = api.parse_call(request.parameters)
response = current_app.senpy.analyse(req)
response = Help(valid_parameters=dic)
return response
req = api.parse_call(request.parameters)
return current_app.senpy.analyse(req)
@api_blueprint.route('/plugins/', methods=['POST', 'GET'])
@basic_api
def plugins():
sp = current_app.senpy
ptype = request.parameters.get('plugin_type')
params = api.parse_params(request.parameters, api.PLUGINS_PARAMS)
ptype = params.get('plugin_type')
plugins = sp.filter_plugins(plugin_type=ptype)
dic = Plugins(plugins=list(plugins.values()))
return dic
......
import requests
import logging
from . import models
from .plugins import default_plugin_type
logger = logging.getLogger(__name__)
......@@ -13,8 +12,8 @@ class Client(object):
def analyse(self, input, method='GET', **kwargs):
return self.request('/', method=method, input=input, **kwargs)
def plugins(self, ptype=default_plugin_type):
resp = self.request(path='/plugins', plugin_type=ptype).plugins
def plugins(self, *args, **kwargs):
resp = self.request(path='/plugins').plugins
return {p.name: p for p in resp}
def request(self, path=None, method='GET', **params):
......
......@@ -123,7 +123,7 @@ class Senpy(object):
return
plugin = plugins[0]
self._activate(plugin) # Make sure the plugin is activated
specific_params = api.get_extra_params(req, plugin)
specific_params = api.parse_extra_params(req, plugin)
req.analysis.append({'plugin': plugin,
'parameters': specific_params})
results = plugin.analyse_entries(entries, specific_params)
......@@ -262,17 +262,11 @@ class Senpy(object):
with plugin._lock:
if plugin.is_activated:
return
try:
plugin.activate()
msg = "Plugin activated: {}".format(plugin.name)
logger.info(msg)
success = True
self._set_active(plugin, success)
except Exception as ex:
msg = "Error activating plugin {} - {} : \n\t{}".format(
plugin.name, ex, traceback.format_exc())
logger.error(msg)
raise Error(msg)
plugin.activate()
msg = "Plugin activated: {}".format(plugin.name)
logger.info(msg)
success = True
self._set_active(plugin, success)
def activate_plugin(self, plugin_name, sync=True):
try:
......@@ -294,13 +288,8 @@ class Senpy(object):
with plugin._lock:
if not plugin.is_activated:
return
try:
plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name))
except Exception as ex:
logger.error(
"Error deactivating plugin {}: {}".format(plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc()))
plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name))
def deactivate_plugin(self, plugin_name, sync=True):
try:
......
......@@ -6,7 +6,11 @@ For compatibility with Py3 and for easier debugging, this new version drops
introspection and adds all arguments to the models.
'''
from __future__ import print_function
from six import string_types
from future import standard_library
standard_library.install_aliases()
from future.utils import with_metaclass
from past.builtins import basestring
import time
import copy
......@@ -15,6 +19,8 @@ import os
import jsonref
import jsonschema
import inspect
from collections import UserDict
from abc import ABCMeta
from flask import Response as FlaskResponse
from pyld import jsonld
......@@ -62,7 +68,7 @@ class Context(dict):
return contexts
elif isinstance(context, dict):
return Context(context)
elif isinstance(context, string_types):
elif isinstance(context, basestring):
try:
with open(context) as f:
return Context(json.loads(f.read()))
......@@ -75,9 +81,154 @@ class Context(dict):
base_context = Context.load(CONTEXT_PATH)
class SenpyMixin(object):
class BaseMeta(ABCMeta):
'''
Metaclass for models. It extracts the default values for the fields in
the model.
For instance, instances of the following class wouldn't need to mark
their version or description on initialization:
.. code-block:: python
class MyPlugin(Plugin):
version=0.3
description='A dull plugin'
Note that these operations could be included in the __init__ of the
class, but it would be very inefficient.
'''
def __new__(mcs, name, bases, attrs, **kwargs):
defaults = {}
if 'schema' in attrs:
defaults = mcs.get_defaults(attrs['schema'])
for b in bases:
if hasattr(b, 'defaults'):
defaults.update(b.defaults)
info = mcs.attrs_to_dict(attrs)
defaults.update(info)
attrs['defaults'] = defaults
return super(BaseMeta, mcs).__new__(mcs, name, bases, attrs)
@staticmethod
def attrs_to_dict(attrs):
'''
Extract the attributes of the class.
This allows adding default values in the class definition.
e.g.:
'''
def is_attr(k, v):
return (not(inspect.isroutine(v) or
inspect.ismethod(v) or
inspect.ismodule(v) or
isinstance(v, property)) and
k[0] != '_' and
k != 'schema' and
k != 'data')
return {key: copy.deepcopy(value) for key, value in attrs.items() if is_attr(key, value)}
@staticmethod
def get_defaults(schema):
temp = {}
for obj in [
schema,
] + schema.get('allOf', []):
for k, v in obj.get('properties', {}).items():
if 'default' in v and k not in temp:
temp[k] = copy.deepcopy(v['default'])
return temp
class CustomDict(UserDict, object):
'''
A dictionary whose elements can also be accessed as attributes. Since some
characters are not valid in the dot-notation, the attribute names also
converted. e.g.:
> d = CustomDict()
> d.key = d['ns:name'] = 1
> d.key == d['key']
True
> d.ns__name == d['ns:name']
'''
defaults = []
def __init__(self, *args, **kwargs):
temp = copy.deepcopy(self.defaults)
for arg in args:
temp.update(copy.deepcopy(arg))
for k, v in kwargs.items():
temp[self._get_key(k)] = v
super(CustomDict, self).__init__(temp)
@staticmethod
def _get_key(key):
if key is 'id':
key = '@id'
key = key.replace("__", ":", 1)
return key
@staticmethod
def _internal_key(key):
return key[0] == '_' or key == 'data'
def __getattr__(self, key):
'''
__getattr__ only gets called when the attribute could not be found
in the __dict__. So we only need to look for the the element in the
dictionary, or raise an Exception.
'''
mkey = self._get_key(key)
if not self._internal_key(key) and mkey in self:
return self[mkey]
raise AttributeError(key)
def __setattr__(self, key, value):
# Work as usual for internal properties or already existing
# properties
if self._internal_key(key) or key in self.__dict__:
return super(CustomDict, self).__setattr__(key, value)
key = self._get_key(key)
return self.__setitem__(self._get_key(key), value)
def __delattr__(self, key):
if self._internal_key(key):
return object.__delattr__(self, key)
key = self._get_key(key)
self.__delitem__(self._get_key(key))
class BaseModel(with_metaclass(BaseMeta, CustomDict)):
'''
Entities of the base model are a special kind of dictionary that emulates
a JSON-LD object. The structure of the dictionary is checked via JSON-schema.
For convenience, the values can also be accessed as attributes
(a la Javascript). e.g.:
> myobject.key == myobject['key']