Commit 0204e0b8 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez

Several changes

* Simplified setattr
* Added loading attributes in class
* Added ability to specify failing test cases in plugins
parent 701f46b9
...@@ -83,7 +83,7 @@ class Senpy(object): ...@@ -83,7 +83,7 @@ class Senpy(object):
self._search_folders.add(folder) self._search_folders.add(folder)
self._outdated = True self._outdated = True
else: else:
logger.debug("Not a folder: %s", folder) raise AttributeError("Not a folder: %s", folder)
def _get_plugins(self, request): def _get_plugins(self, request):
if not self.analysis_plugins: if not self.analysis_plugins:
......
...@@ -14,6 +14,7 @@ import json ...@@ -14,6 +14,7 @@ import json
import os import os
import jsonref import jsonref
import jsonschema import jsonschema
import inspect
from flask import Response as FlaskResponse from flask import Response as FlaskResponse
from pyld import jsonld from pyld import jsonld
...@@ -102,7 +103,7 @@ class SenpyMixin(object): ...@@ -102,7 +103,7 @@ class SenpyMixin(object):
}) })
return FlaskResponse( return FlaskResponse(
response=content, response=content,
status=getattr(self, "status", 200), status=self.get('status', 200),
headers=headers, headers=headers,
mimetype=mimetype) mimetype=mimetype)
...@@ -188,34 +189,61 @@ class SenpyMixin(object): ...@@ -188,34 +189,61 @@ class SenpyMixin(object):
class BaseModel(SenpyMixin, dict): class BaseModel(SenpyMixin, dict):
'''
Entities of the base model are a special kind of dictionary that emulates
a JSON-LD object. For convenience, the values can also be accessed as attributes
(a la Javascript). e.g.:
> myobject.key == myobject['key']
True
> myobject.ns__name == myobject['ns:name']
True
'''
schema = base_schema schema = base_schema
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.attrs_to_dict()
if 'id' in kwargs: if 'id' in kwargs:
self.id = kwargs.pop('id') self.id = kwargs.pop('id')
elif kwargs.pop('_auto_id', True): elif kwargs.pop('_auto_id', True):
self.id = '_:{}_{}'.format(type(self).__name__, time.time()) self.id = '_:{}_{}'.format(type(self).__name__, time.time())
temp = dict(*args, **kwargs)
temp = self.get_defaults()
temp.update(dict(*args))
for k, v in kwargs.items():
temp[self._get_key(k)] = v
super(BaseModel, self).__init__(temp)
if '@type' not in self:
logger.warn('Created an instance of an unknown model')
def get_defaults(self):
temp = {}
for obj in [ for obj in [
self.schema, self.schema,
] + self.schema.get('allOf', []): ] + self.schema.get('allOf', []):
for k, v in obj.get('properties', {}).items(): for k, v in obj.get('properties', {}).items():
if 'default' in v and k not in temp: if 'default' in v and k not in temp:
temp[k] = copy.deepcopy(v['default']) temp[k] = copy.deepcopy(v['default'])
return temp
for i in temp: def attrs_to_dict(self):
nk = self._get_key(i) '''
if nk != i: Copy the attributes of the class to the instance.
temp[nk] = temp[i]
del temp[i]
try:
temp['@type'] = getattr(self, '@type')
except AttributeError:
logger.warn('Creating an instance of an unknown model')
super(BaseModel, self).__init__(temp) This allows adding default values in the class definition.
e.g.:
class MyPlugin(Plugin):
version=0.3
description='A dull plugin'
'''
def is_attr(x):
return not(inspect.isroutine(x) or inspect.ismethod(x) or isinstance(x, property))
for key, value in inspect.getmembers(self.__class__, is_attr):
if key[0] != '_' and key != 'schema':
self[key] = value
def _get_key(self, key): def _get_key(self, key):
if key is 'id': if key is 'id':
...@@ -224,26 +252,37 @@ class BaseModel(SenpyMixin, dict): ...@@ -224,26 +252,37 @@ class BaseModel(SenpyMixin, dict):
return key return key
def __delitem__(self, key): def __delitem__(self, key):
key = self._get_key(key)
dict.__delitem__(self, key) dict.__delitem__(self, key)
def _internal_key(self, key):
return key[0] == '_' or key in self.__dict__
def _plain_dict(self):
d = {k: v for (k, v) in self.items() if k[0] != "_"}
return d
def __getattr__(self, key): def __getattr__(self, key):
try: '''
return self.__getitem__(self._get_key(key)) __getattr__ only gets called when the attribute could not
except KeyError: be found in the __dict__. So we only need to look for the
the element in the dictionary, or raise an Exception.
'''
if self._internal_key(key):
raise AttributeError(key) raise AttributeError(key)
return self.__getitem__(self._get_key(key))
def __setattr__(self, key, value): def __setattr__(self, key, value):
self.__setitem__(self._get_key(key), value) if self._internal_key(key):
return super(BaseModel, self).__setattr__(key, value)
key = self._get_key(key)
return self.__setitem__(self._get_key(key), value)
def __delattr__(self, key): def __delattr__(self, key):
try: if self._internal_key(key):
object.__delattr__(self, key) return object.__delattr__(self, key)
except AttributeError: key = self._get_key(key)
self.__delitem__(self._get_key(key)) self.__delitem__(self._get_key(key))
def _plain_dict(self):
d = {k: v for (k, v) in self.items() if k[0] != "_"}
return d
def register(rsubclass, rtype=None): def register(rsubclass, rtype=None):
......
...@@ -15,8 +15,6 @@ import importlib ...@@ -15,8 +15,6 @@ import importlib
import yaml import yaml
import threading import threading
from contextlib import contextmanager
from .. import models, utils from .. import models, utils
from ..api import API_PARAMS from ..api import API_PARAMS
...@@ -29,16 +27,21 @@ class Plugin(models.Plugin): ...@@ -29,16 +27,21 @@ class Plugin(models.Plugin):
Provides a canonical name for plugins and serves as base for other Provides a canonical name for plugins and serves as base for other
kinds of plugins. kinds of plugins.
""" """
if not info: logger.debug("Initialising {}".format(info))
if info:
self.update(info)
super(Plugin, self).__init__(**self)
if not self.validate():
raise models.Error(message=("You need to provide configuration" raise models.Error(message=("You need to provide configuration"
"information for the plugin.")) "information for the plugin."))
logger.debug("Initialising {}".format(info)) self.id = 'plugins/{}_{}'.format(self['name'], self['version'])
id = 'plugins/{}_{}'.format(info['name'], info['version'])
super(Plugin, self).__init__(id=id, **info)
self.is_activated = False self.is_activated = False
self._lock = threading.Lock() self._lock = threading.Lock()
self.data_folder = data_folder or os.getcwd() self.data_folder = data_folder or os.getcwd()
def validate(self):
return all(x in self for x in ('name', 'description', 'version'))
def get_folder(self): def get_folder(self):
return os.path.dirname(inspect.getfile(self.__class__)) return os.path.dirname(inspect.getfile(self.__class__))
...@@ -50,12 +53,21 @@ class Plugin(models.Plugin): ...@@ -50,12 +53,21 @@ class Plugin(models.Plugin):
def test(self): def test(self):
if not hasattr(self, 'test_cases'): if not hasattr(self, 'test_cases'):
import inspect
raise AttributeError(('Plugin {} [{}] does not have any defined ' raise AttributeError(('Plugin {} [{}] does not have any defined '
'test cases').format(self.id, inspect.getfile(self.__class__))) 'test cases').format(self.id, inspect.getfile(self.__class__)))
for case in self.test_cases: for case in self.test_cases:
res = list(self.analyse_entry(models.Entry(case['entry']), entry = models.Entry(case['entry'])
case['params'])) params = case.get('params', {})
fails = case.get('fails', False)
try:
res = list(self.analyse_entry(entry, params))
except models.Error:
if fails:
continue
raise
if fails:
raise Exception('This test should have raised an exception.')
exp = case['expected'] exp = case['expected']
if not isinstance(exp, list): if not isinstance(exp, list):
exp = [exp] exp = [exp]
...@@ -63,12 +75,13 @@ class Plugin(models.Plugin): ...@@ -63,12 +75,13 @@ class Plugin(models.Plugin):
for r in res: for r in res:
r.validate() r.validate()
@contextmanager
def open(self, fpath, *args, **kwargs): def open(self, fpath, *args, **kwargs):
if not os.path.isabs(fpath): if not os.path.isabs(fpath):
fpath = os.path.join(self.data_folder, fpath) fpath = os.path.join(self.data_folder, fpath)
with open(fpath, *args, **kwargs) as f: return open(fpath, *args, **kwargs)
yield f
def serve(self, **kwargs):
utils.serve(plugin=self, **kwargs)
SenpyPlugin = Plugin SenpyPlugin = Plugin
...@@ -106,17 +119,13 @@ class ConversionPlugin(Plugin): ...@@ -106,17 +119,13 @@ class ConversionPlugin(Plugin):
class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin): class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin):
def __init__(self, info, *args, **kwargs): minPolarityValue = 0
super(SentimentPlugin, self).__init__(info, *args, **kwargs) maxPolarityValue = 1
self.minPolarityValue = float(info.get("minPolarityValue", 0))
self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin): class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin):
def __init__(self, info, *args, **kwargs): minEmotionValue = 0
super(EmotionPlugin, self).__init__(info, *args, **kwargs) maxEmotionValue = 1
self.minEmotionValue = float(info.get("minEmotionValue", -1))
self.maxEmotionValue = float(info.get("maxEmotionValue", 1))
class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin): class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin):
...@@ -127,11 +136,11 @@ class ShelfMixin(object): ...@@ -127,11 +136,11 @@ class ShelfMixin(object):
@property @property
def sh(self): def sh(self):
if not hasattr(self, '_sh') or self._sh is None: if not hasattr(self, '_sh') or self._sh is None:
self.__dict__['_sh'] = {} self._sh = {}
if os.path.isfile(self.shelf_file): if os.path.isfile(self.shelf_file):
try: try:
with self.open(self.shelf_file, 'rb') as p: with self.open(self.shelf_file, 'rb') as p:
self.__dict__['_sh'] = pickle.load(p) self._sh = pickle.load(p)
except (IndexError, EOFError, pickle.UnpicklingError): except (IndexError, EOFError, pickle.UnpicklingError):
logger.warning('{} has a corrupted shelf file!'.format(self.id)) logger.warning('{} has a corrupted shelf file!'.format(self.id))
if not self.get('force_shelf', False): if not self.get('force_shelf', False):
...@@ -142,9 +151,13 @@ class ShelfMixin(object): ...@@ -142,9 +151,13 @@ class ShelfMixin(object):
def sh(self): def sh(self):
if os.path.isfile(self.shelf_file): if os.path.isfile(self.shelf_file):
os.remove(self.shelf_file) os.remove(self.shelf_file)
del self.__dict__['_sh'] del self._sh
self.save() self.save()
@sh.setter
def sh(self, value):
self._sh = value
@property @property
def shelf_file(self): def shelf_file(self):
if 'shelf_file' not in self or not self['shelf_file']: if 'shelf_file' not in self or not self['shelf_file']:
...@@ -196,7 +209,7 @@ def pfilter(plugins, **kwargs): ...@@ -196,7 +209,7 @@ def pfilter(plugins, **kwargs):
def validate_info(info): def validate_info(info):
return all(x in info for x in ('name', 'module', 'description', 'version')) return all(x in info for x in ('name',))
def load_module(name, root=None): def load_module(name, root=None):
...@@ -235,6 +248,17 @@ def install_deps(*plugins): ...@@ -235,6 +248,17 @@ def install_deps(*plugins):
return installed return installed
def get_plugin_class(module):
candidate = None
for _, obj in inspect.getmembers(module):
if inspect.isclass(obj) and inspect.getmodule(obj) == module:
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj)))
candidate = obj
break
return candidate
def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs): def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs):
if not root and '_path' in info: if not root and '_path' in info:
root = os.path.dirname(info['_path']) root = os.path.dirname(info['_path'])
...@@ -249,18 +273,12 @@ def load_plugin_from_info(info, root=None, validator=validate_info, install=True ...@@ -249,18 +273,12 @@ def load_plugin_from_info(info, root=None, validator=validate_info, install=True
raise raise
install_deps(info) install_deps(info)
tmp = load_module(module, root) tmp = load_module(module, root)
candidate = None cls = None
for _, obj in inspect.getmembers(tmp): if '@type' not in info:
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp: cls = get_plugin_class(tmp)
logger.debug(("Found plugin class:" if not cls:
" {}@{}").format(obj, inspect.getmodule(obj))) raise Exception("No valid plugin for: {}".format(module))
candidate = obj return cls(info=info, *args, **kwargs)
break
if not candidate:
logger.debug("No valid plugin for: {}".format(module))
return
module = candidate(info=info, *args, **kwargs)
return module
def parse_plugin_info(fpath): def parse_plugin_info(fpath):
......
...@@ -45,7 +45,7 @@ class SplitPlugin(AnalysisPlugin): ...@@ -45,7 +45,7 @@ class SplitPlugin(AnalysisPlugin):
}, },
{ {
'entry': { 'entry': {
"id": ":test", "@id": ":test",
'nif:isString': 'Hello\nWorld' 'nif:isString': 'Hello\nWorld'
}, },
'params': { 'params': {
......
...@@ -23,3 +23,23 @@ def check_template(indict, template): ...@@ -23,3 +23,23 @@ def check_template(indict, template):
else: else:
if indict != template: if indict != template:
raise models.Error('{} and {} are different'.format(indict, template)) raise models.Error('{} and {} are different'.format(indict, template))
def easy(app=None, plugin=None, host='0.0.0.0', port=5000, **kwargs):
'''
Run a server with a specific plugin.
'''
from flask import Flask
from senpy.extensions import Senpy
if not app:
app = Flask(__name__)
sp = Senpy(app)
if plugin:
sp.add_plugin(plugin)
sp.install_deps()
app.run(host,
port,
debug=app.debug,
**kwargs)
...@@ -34,6 +34,11 @@ setup( ...@@ -34,6 +34,11 @@ setup(
install_requires=install_reqs, install_requires=install_reqs,
tests_require=test_reqs, tests_require=test_reqs,
setup_requires=['pytest-runner', ], setup_requires=['pytest-runner', ],
extras_require={
'evaluation': [
'gsitk'
]
},
include_package_data=True, include_package_data=True,
entry_points={ entry_points={
'console_scripts': 'console_scripts':
......
from senpy.plugins import AnalysisPlugin
import multiprocessing
def _train(process_number):
return process_number
class AsyncPlugin(AnalysisPlugin):
def _do_async(self, num_processes):
pool = multiprocessing.Pool(processes=num_processes)
values = pool.map(_train, range(num_processes))
return values
def activate(self):
self.value = self._do_async(4)
def analyse_entry(self, entry, params):
values = self._do_async(2)
entry.async_values = values
yield entry
def test(self):
pass
---
name: Async
module: asyncplugin
description: I am async
author: "@balkian"
version: '0.1'
async: true
extra_params: {}
\ No newline at end of file
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
def analyse_entry(self, entry, params):
entry['nif:iString'] = entry['nif:isString'][::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry
def test(self):
pass
{
"name": "Dummy",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": false,
"default": 0
}
}
}
{
"name": "DummyRequired",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": true
}
}
}
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
import noop
from senpy.plugins import AnalysisPlugin
from time import sleep
class SleepPlugin(AnalysisPlugin):
def activate(self, *args, **kwargs):
sleep(self.timeout)
def analyse_entry(self, entry, params):
sleep(float(params.get("timeout", self.timeout)))
yield entry
def test(self):
pass
{
"name": "Sleep",
"module": "sleep",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"timeout": 0.05,
"extra_params": {
"timeout": {
"@id": "timeout_sleep",
"aliases": ["timeout", "to"],
"required": false,
"default": 0
}
}
}
...@@ -25,8 +25,8 @@ def analyse(instance, **kwargs): ...@@ -25,8 +25,8 @@ def analyse(instance, **kwargs):
class ExtensionsTest(TestCase): class ExtensionsTest(TestCase):
def setUp(self): def setUp(self):
self.app = Flask('test_extensions') self.app = Flask('test_extensions')
self.dir = os.path.dirname(__file__) self.examples_dir = os.path.join(os.path.dirname(__file__), '..', 'example-plugins')
self.senpy = Senpy(plugin_folder=self.dir, self.senpy = Senpy(plugin_folder=self.examples_dir,
app=self.app, app=self.app,
default_plugins=False) default_plugins=False)
self.senpy.activate_plugin("Dummy", sync=True) self.senpy.activate_plugin("Dummy", sync=True)
...@@ -41,7 +41,7 @@ class ExtensionsTest(TestCase): ...@@ -41,7 +41,7 @@ class ExtensionsTest(TestCase):
def test_discovery(self): def test_discovery(self):
""" Discovery of plugins in given folders. """ """ Discovery of plugins in given folders. """
# noinspection PyProtectedMember # noinspection PyProtectedMember
assert self.dir in self.senpy._search_folders assert self.examples_dir in self.senpy._search_folders
print(self.senpy.plugins) print(self.senpy.plugins)