Commit bfc588a9 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez

Several fixes

* Refactored BaseModel for efficiency
* Added plugin metaclass to keep track of plugin types
* Moved plugins to examples dir (in a previous commit)
* Simplified validation in parse_params
* Added convenience methods to mock requests in tests
* Changed help schema to use `.valid_parameters` instead of `.parameters`,
which was used in results to show parameters provided by the user.
* Improved UI
    * Added basic parameters
    * Fixed bugs in parameter handling
    * Refactored and cleaned code
parent f93eed2c
from senpy.plugins import AnalysisPlugin
import multiprocessing
def _train(process_number):
return process_number
class AsyncPlugin(AnalysisPlugin):
def _do_async(self, num_processes):
pool = multiprocessing.Pool(processes=num_processes)
values = pool.map(_train, range(num_processes))
return values
def activate(self):
self.value = self._do_async(4)
def analyse_entry(self, entry, params):
values = self._do_async(2)
entry.async_values = values
yield entry
def test(self):
pass
---
name: Async
module: asyncplugin
description: I am async
author: "@balkian"
version: '0.1'
async: true
extra_params: {}
\ No newline at end of file
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
def analyse_entry(self, entry, params):
entry['nif:isString'] = entry['nif:isString'][::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry
def test(self):
pass
{
"name": "Dummy",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": false,
"default": 0
}
}
}
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
description = 'This is a dummy self-contained plugin'
author = '@balkian'
version = '0.1'
def analyse_entry(self, entry, params):
entry['nif:isString'] = entry['nif:isString'][::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry
test_cases = [{
"entry": {
"nif:isString": "Hello world!"
},
"expected": [{
"nif:isString": "!dlrow olleH"
}]
}]
if __name__ == '__main__':
d = DummyPlugin()
d.test()
name: DummyNoInfo
module: dummy_noinfo
{
"name": "DummyRequired",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": true
}
}
}
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
import noop
from senpy.plugins import AnalysisPlugin
from time import sleep
class SleepPlugin(AnalysisPlugin):
def activate(self, *args, **kwargs):
sleep(self.timeout)
def analyse_entry(self, entry, params):
sleep(float(params.get("timeout", self.timeout)))
yield entry
def test(self):
pass
{
"name": "Sleep",
"module": "sleep",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"timeout": 0.05,
"extra_params": {
"timeout": {
"@id": "timeout_sleep",
"aliases": ["timeout", "to"],
"required": false,
"default": 0
}
}
}
......@@ -3,7 +3,6 @@ requests>=2.4.1
tornado>=4.4.3
PyLD>=0.6.5
nltk
six
future
jsonschema
jsonref
......
......@@ -19,6 +19,9 @@ Sentiment analysis server in Python
"""
from .version import __version__
from future.standard_library import install_aliases
install_aliases()
import logging
logger = logging.getLogger(__name__)
......
......@@ -67,7 +67,7 @@ def main():
'--plugins-folder',
'-f',
type=str,
default='plugins',
default='.',
help='Where to look for plugins.')
parser.add_argument(
'--only-install',
......
......@@ -13,8 +13,9 @@ API_PARAMS = {
"expanded-jsonld": {
"@id": "expanded-jsonld",
"aliases": ["expanded"],
"options": "boolean",
"required": True,
"default": 0
"default": False
},
"with_parameters": {
"aliases": ['withparameters',
......@@ -23,13 +24,6 @@ API_PARAMS = {
"default": False,
"required": True
},
"plugin_type": {
"@id": "pluginType",
"description": 'What kind of plugins to list',
"aliases": ["pluginType"],
"required": True,
"default": "analysisPlugin"
},
"outformat": {
"@id": "outformat",
"aliases": ["o"],
......@@ -59,6 +53,16 @@ API_PARAMS = {
}
}
PLUGINS_PARAMS = {
"plugin_type": {
"@id": "pluginType",
"description": 'What kind of plugins to list',
"aliases": ["pluginType"],
"required": True,
"default": 'analysisPlugin'
}
}
WEB_PARAMS = {
"inHeaders": {
"aliases": ["headers"],
......@@ -126,24 +130,26 @@ def parse_params(indict, *specs):
wrong_params = {}
for spec in specs:
for param, options in iteritems(spec):
if param[0] != "@": # Exclude json-ld properties
for alias in options.get("aliases", []):
# Replace each alias with the correct name of the parameter
if alias in indict and alias is not param:
outdict[param] = indict[alias]
del indict[alias]
continue
if param not in outdict:
if options.get("required", False) and "default" not in options:
wrong_params[param] = spec[param]
else:
if "default" in options:
outdict[param] = options["default"]
elif "options" in spec[param]:
if spec[param]["options"] == "boolean":
outdict[param] = outdict[param] in [None, True, 'true', '1']
elif outdict[param] not in spec[param]["options"]:
wrong_params[param] = spec[param]
if param[0] == "@": # Exclude json-ld properties
continue
for alias in options.get("aliases", []):
# Replace each alias with the correct name of the parameter
if alias in indict and alias is not param:
outdict[param] = indict[alias]
del indict[alias]
continue
if param not in outdict:
if "default" in options:
# We assume the default is correct
outdict[param] = options["default"]
elif options.get("required", False):
wrong_params[param] = spec[param]
continue
if "options" in options:
if options["options"] == "boolean":
outdict[param] = outdict[param] in [None, True, 'true', '1']
elif outdict[param] not in options["options"]:
wrong_params[param] = spec[param]
if wrong_params:
logger.debug("Error parsing: %s", wrong_params)
message = Error(
......@@ -158,7 +164,7 @@ def parse_params(indict, *specs):
return outdict
def get_extra_params(request, plugin=None):
def parse_extra_params(request, plugin=None):
params = request.parameters.copy()
if plugin:
extra_params = parse_params(params, plugin.get('extra_params', {}))
......@@ -177,6 +183,6 @@ def parse_call(params):
elif params['informat'] == 'json-ld':
results = from_string(params['input'], cls=Results)
else:
raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
raise NotImplementedError('Informat {} is not implemented'.format(params['informat']))
results.parameters = params
return results
......@@ -25,6 +25,7 @@ from .version import __version__
from functools import wraps
import logging
import traceback
import json
logger = logging.getLogger(__name__)
......@@ -72,12 +73,19 @@ def schema(schema="definitions"):
def basic_api(f):
default_params = {
'inHeaders': False,
'expanded-jsonld': False,
'outformat': 'json-ld',
'with_parameters': True,
}
@wraps(f)
def decorated_function(*args, **kwargs):
raw_params = get_params(request)
headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)}
params = default_params
outformat = 'json-ld'
try:
print('Getting request:')
print(request)
......@@ -87,26 +95,32 @@ def basic_api(f):
else:
request.parameters = params
response = f(*args, **kwargs)
except Error as ex:
response = ex
response.parameters = params
logger.error(ex)
except (Exception) as ex:
if current_app.debug:
raise
if not isinstance(ex, Error):
msg = "{}:\n\t{}".format(ex,
traceback.format_exc())
ex = Error(message=msg, status=500)
logger.exception('Error returning analysis result')
response = ex
response.parameters = raw_params
logger.error(ex)
in_headers = params['inHeaders']
expanded = params['expanded-jsonld']
outformat = params['outformat']
if 'parameters' in response and not params['with_parameters']:
print(response)
print(response.data)
del response.parameters
return response.flask(
in_headers=in_headers,
in_headers=params['inHeaders'],
headers=headers,
prefix=url_for('.api_root', _external=True),
context_uri=url_for('api.context',
entity=type(response).__name__,
_external=True),
outformat=outformat,
expanded=expanded)
outformat=params['outformat'],
expanded=params['expanded-jsonld'])
return decorated_function
......@@ -116,19 +130,18 @@ def basic_api(f):
def api_root():
if request.parameters['help']:
dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
response = Help(parameters=dic)
return response
else:
req = api.parse_call(request.parameters)
response = current_app.senpy.analyse(req)
response = Help(valid_parameters=dic)
return response
req = api.parse_call(request.parameters)
return current_app.senpy.analyse(req)
@api_blueprint.route('/plugins/', methods=['POST', 'GET'])
@basic_api
def plugins():
sp = current_app.senpy
ptype = request.parameters.get('plugin_type')
params = api.parse_params(request.parameters, api.PLUGINS_PARAMS)
ptype = params.get('plugin_type')
plugins = sp.filter_plugins(plugin_type=ptype)
dic = Plugins(plugins=list(plugins.values()))
return dic
......
import requests
import logging
from . import models
from .plugins import default_plugin_type
logger = logging.getLogger(__name__)
......@@ -13,8 +12,8 @@ class Client(object):
def analyse(self, input, method='GET', **kwargs):
return self.request('/', method=method, input=input, **kwargs)
def plugins(self, ptype=default_plugin_type):
resp = self.request(path='/plugins', plugin_type=ptype).plugins
def plugins(self, *args, **kwargs):
resp = self.request(path='/plugins').plugins
return {p.name: p for p in resp}
def request(self, path=None, method='GET', **params):
......
......@@ -123,7 +123,7 @@ class Senpy(object):
return
plugin = plugins[0]
self._activate(plugin) # Make sure the plugin is activated
specific_params = api.get_extra_params(req, plugin)
specific_params = api.parse_extra_params(req, plugin)
req.analysis.append({'plugin': plugin,
'parameters': specific_params})
results = plugin.analyse_entries(entries, specific_params)
......@@ -262,17 +262,11 @@ class Senpy(object):
with plugin._lock:
if plugin.is_activated:
return
try:
plugin.activate()
msg = "Plugin activated: {}".format(plugin.name)
logger.info(msg)
success = True
self._set_active(plugin, success)
except Exception as ex:
msg = "Error activating plugin {} - {} : \n\t{}".format(
plugin.name, ex, traceback.format_exc())
logger.error(msg)
raise Error(msg)
plugin.activate()
msg = "Plugin activated: {}".format(plugin.name)
logger.info(msg)
success = True
self._set_active(plugin, success)
def activate_plugin(self, plugin_name, sync=True):
try:
......@@ -294,13 +288,8 @@ class Senpy(object):
with plugin._lock:
if not plugin.is_activated:
return
try:
plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name))
except Exception as ex:
logger.error(
"Error deactivating plugin {}: {}".format(plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc()))
plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name))
def deactivate_plugin(self, plugin_name, sync=True):
try:
......
This diff is collapsed.
from future import standard_library
standard_library.install_aliases()
from future.utils import with_metaclass
import os.path
import os
......@@ -16,21 +17,33 @@ import yaml
import threading
from .. import models, utils
from ..api import API_PARAMS
from .. import api
logger = logging.getLogger(__name__)
class Plugin(models.Plugin):
def __init__(self, info=None, data_folder=None):
class PluginMeta(models.BaseMeta):
def __new__(mcs, name, bases, attrs, **kwargs):
plugin_type = []
if hasattr(bases[0], 'plugin_type'):
plugin_type += bases[0].plugin_type
plugin_type.append(name)
attrs['plugin_type'] = plugin_type
return super(PluginMeta, mcs).__new__(mcs, name, bases, attrs)
class Plugin(with_metaclass(PluginMeta, models.Plugin)):
def __init__(self, info=None, data_folder=None, **kwargs):
"""
Provides a canonical name for plugins and serves as base for other
kinds of plugins.
"""
logger.debug("Initialising {}".format(info))
super(Plugin, self).__init__(**kwargs)
if info:
self.update(info)
super(Plugin, self).__init__(**self)
if not self.validate():
raise models.Error(message=("You need to provide configuration"
"information for the plugin."))
......@@ -57,7 +70,8 @@ class Plugin(models.Plugin):
'test cases').format(self.id, inspect.getfile(self.__class__)))
for case in self.test_cases:
entry = models.Entry(case['entry'])
params = case.get('params', {})
given_parameters = case.get('params', {})
params = api.parse_params(given_parameters, self.extra_params)
fails = case.get('fails', False)
try:
res = list(self.analyse_entry(entry, params))
......@@ -90,7 +104,7 @@ SenpyPlugin = Plugin
class AnalysisPlugin(Plugin):
def analyse(self, *args, **kwargs):
raise NotImplemented(
raise NotImplementedError(
'Your method should implement either analyse or analyse_entry')
def analyse_entry(self, entry, parameters):
......@@ -118,17 +132,17 @@ class ConversionPlugin(Plugin):
pass
class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin):
class SentimentPlugin(AnalysisPlugin, models.SentimentPlugin):
minPolarityValue = 0
maxPolarityValue = 1
class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin):
class EmotionPlugin(AnalysisPlugin, models.EmotionPlugin):
minEmotionValue = 0
maxEmotionValue = 1
class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin):
class EmotionConversionPlugin(ConversionPlugin):
pass
......@@ -171,19 +185,18 @@ class ShelfMixin(object):
pickle.dump(self._sh, f)
default_plugin_type = API_PARAMS['plugin_type']['default']
def pfilter(plugins, **kwargs):
""" Filter plugins by different criteria """
if isinstance(plugins, models.Plugins):
plugins = plugins.plugins
elif isinstance(plugins, dict):
plugins = plugins.values()
ptype = kwargs.pop('plugin_type', default_plugin_type)
ptype = kwargs.pop('plugin_type', AnalysisPlugin)
logger.debug('#' * 100)
logger.debug('ptype {}'.format(ptype))
if ptype:
if isinstance(ptype, PluginMeta):
ptype = ptype.__name__
try:
ptype = ptype[0].upper() + ptype[1:]
pclass = globals()[ptype]
......
......@@ -4,7 +4,7 @@ from senpy.plugins import EmotionPlugin
from senpy.models import EmotionSet, Emotion, Entry
class RmoRandPlugin(EmotionPlugin):
class EmoRandPlugin(EmotionPlugin):
def analyse_entry(self, entry, params):
category = "emoml:big6happiness"
number = max(-1, min(1, random.gauss(0, 0.5)))
......
......@@ -11,7 +11,7 @@ class SplitPlugin(AnalysisPlugin):
nltk.download('punkt')
def analyse_entry(self, entry, params):
chunker_type = params.get("delimiter", "sentence")
chunker_type = params["delimiter"]
original_text = entry['nif:isString']
if chunker_type == "sentence":
tokenizer = PunktSentenceTokenizer()
......
......@@ -7,7 +7,7 @@ from senpy.models import Sentiment
class Sentiment140Plugin(SentimentPlugin):
def analyse_entry(self, entry, params):
lang = params.get("language", "auto")
lang = params["language"]
res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
json.dumps({
"language": lang,
......@@ -35,6 +35,18 @@ class Sentiment140Plugin(SentimentPlugin):
entry.language = lang
yield entry
def test(self, *args, **kwargs):
'''
To avoid calling the sentiment140 API, we will mock the results
from requests.
'''
from senpy.test import patch_requests
expected = {"data": [{"polarity": 10}]}
with patch_requests(expected) as (request, response):
super(Sentiment140Plugin, self).test(*args, **kwargs)
assert request.called
assert response.json.called
test_cases = [
{
'entry': {
......
......@@ -16,6 +16,7 @@ extra_params:
- es
- en
- auto
default: auto
requirements: {}
maxPolarityValue: 1
minPolarityValue: 0
\ No newline at end of file
......@@ -7,11 +7,11 @@
"description": "Help containing accepted parameters",
"type": "object",
"properties": {
"parameters": {
"valid_parameters": {
"type": "object"
}
},
"required": "parameters"
"required": "valid_parameters"
}
]
}
\ No newline at end of file
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"required": ["@id", "extra_params"],
"required": ["@id", "name", "description", "version", "plugin_type"],
"properties": {
"@id": {
"type": "string",
......@@ -9,7 +9,19 @@
},
"name": {
"type": "string",
"description": "The name of the plugin, which will be used in the algorithm detection phase"
"description": "The name of the plugin, which will be used in the algorithm detection phase."
},
"description": {
"type": "string",
"description": "A summary of what the plugin does, and pointers to further information."
},
"version": {
"type": "string",
"description": "The version of the plugin."
},
"plugin_type": {
"type": "string",
"description": "Sub-type of plugin. e.g. sentimentPlugin"
},
"extra_params": {
"type": "object",
......
......@@ -152,3 +152,18 @@ textarea{
/* background: white; */
display: none;
}
.deco-none {
color: inherit;
text-decoration: inherit;
}
.deco-none:link {
color: inherit;
text-decoration: inherit;
}
.deco-none:hover {
color: inherit;
text-decoration: inherit;
}
This diff is collapsed.
......@@ -88,28 +88,52 @@
<div class="tab-pane active" id="test">
<div class="well">
<form id="form" onsubmit="return getPlugins();" accept-charset="utf-8">
<div id="inputswrapper">
<form id="form" class="container" onsubmit="return getPlugins();" accept-charset="utf-8">
<div><textarea id="input" class="boxsizingBorder" rows="5" name="i">This text makes me sad.
whilst this text makes me happy and surprised at the same time.
I cannot believe it!</textarea></div>
<label>Select the plugin:</label>
<select id="plugins" name="plugins" onchange="change_params()">
</select>
</br>
<div id ="params">
I cannot believe it!</textarea>
</div>
<div>
<label>Select the plugin:</label>
<select id="plugins" name="plugins" onchange="draw_extra_parameters()">
</select>
</div>
<!-- PARAMETERS -->
<div class="panel-group" id="parameters">
<div class="panel panel-default">
<a data-toggle="collapse" class="deco-none" href="#basic_params">
<div class="panel-heading">
<h4 class="panel-title">
Basic API parameters
</h4>
</div>
</a>
<div id="basic_params" class="panel-collapse collapse panel-body">
</div>
</br>
</div>
<div class="panel panel-default">
<a data-toggle="collapse" class="deco-none" href="#extra_params">
<div class="panel-heading">
<h4 class="panel-title">
Plugin extra parameters
</h4>
</div>
</a>
<div id="extra_params" class="panel-collapse collapse in panel-body">
</div>
</div>