Commit 0204e0b8 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez

Several changes

* Simplified setattr
* Added loading attributes in class
* Added ability to specify failing test cases in plugins
parent 701f46b9
......@@ -83,7 +83,7 @@ class Senpy(object):
self._search_folders.add(folder)
self._outdated = True
else:
logger.debug("Not a folder: %s", folder)
raise AttributeError("Not a folder: %s", folder)
def _get_plugins(self, request):
if not self.analysis_plugins:
......
......@@ -14,6 +14,7 @@ import json
import os
import jsonref
import jsonschema
import inspect
from flask import Response as FlaskResponse
from pyld import jsonld
......@@ -102,7 +103,7 @@ class SenpyMixin(object):
})
return FlaskResponse(
response=content,
status=getattr(self, "status", 200),
status=self.get('status', 200),
headers=headers,
mimetype=mimetype)
......@@ -188,34 +189,61 @@ class SenpyMixin(object):
class BaseModel(SenpyMixin, dict):
'''
Entities of the base model are a special kind of dictionary that emulates
a JSON-LD object. For convenience, the values can also be accessed as attributes
(a la Javascript). e.g.:
> myobject.key == myobject['key']
True
> myobject.ns__name == myobject['ns:name']
True
'''
schema = base_schema
def __init__(self, *args, **kwargs):
self.attrs_to_dict()
if 'id' in kwargs:
self.id = kwargs.pop('id')
elif kwargs.pop('_auto_id', True):
self.id = '_:{}_{}'.format(type(self).__name__, time.time())
temp = dict(*args, **kwargs)
temp = self.get_defaults()
temp.update(dict(*args))
for k, v in kwargs.items():
temp[self._get_key(k)] = v
super(BaseModel, self).__init__(temp)
if '@type' not in self:
logger.warn('Created an instance of an unknown model')
def get_defaults(self):
temp = {}
for obj in [
self.schema,
] + self.schema.get('allOf', []):
for k, v in obj.get('properties', {}).items():
if 'default' in v and k not in temp:
temp[k] = copy.deepcopy(v['default'])
return temp
for i in temp:
nk = self._get_key(i)
if nk != i:
temp[nk] = temp[i]
del temp[i]
try:
temp['@type'] = getattr(self, '@type')
except AttributeError:
logger.warn('Creating an instance of an unknown model')
def attrs_to_dict(self):
'''
Copy the attributes of the class to the instance.
super(BaseModel, self).__init__(temp)
This allows adding default values in the class definition.
e.g.:
class MyPlugin(Plugin):
version=0.3
description='A dull plugin'
'''
def is_attr(x):
return not(inspect.isroutine(x) or inspect.ismethod(x) or isinstance(x, property))
for key, value in inspect.getmembers(self.__class__, is_attr):
if key[0] != '_' and key != 'schema':
self[key] = value
def _get_key(self, key):
if key is 'id':
......@@ -224,26 +252,37 @@ class BaseModel(SenpyMixin, dict):
return key
def __delitem__(self, key):
key = self._get_key(key)
dict.__delitem__(self, key)
def _internal_key(self, key):
return key[0] == '_' or key in self.__dict__
def _plain_dict(self):
d = {k: v for (k, v) in self.items() if k[0] != "_"}
return d
def __getattr__(self, key):
try:
return self.__getitem__(self._get_key(key))
except KeyError:
'''
__getattr__ only gets called when the attribute could not
be found in the __dict__. So we only need to look for the
the element in the dictionary, or raise an Exception.
'''
if self._internal_key(key):
raise AttributeError(key)
return self.__getitem__(self._get_key(key))
def __setattr__(self, key, value):
self.__setitem__(self._get_key(key), value)
if self._internal_key(key):
return super(BaseModel, self).__setattr__(key, value)
key = self._get_key(key)
return self.__setitem__(self._get_key(key), value)
def __delattr__(self, key):
try:
object.__delattr__(self, key)
except AttributeError:
self.__delitem__(self._get_key(key))
def _plain_dict(self):
d = {k: v for (k, v) in self.items() if k[0] != "_"}
return d
if self._internal_key(key):
return object.__delattr__(self, key)
key = self._get_key(key)
self.__delitem__(self._get_key(key))
def register(rsubclass, rtype=None):
......
......@@ -15,8 +15,6 @@ import importlib
import yaml
import threading
from contextlib import contextmanager
from .. import models, utils
from ..api import API_PARAMS
......@@ -29,16 +27,21 @@ class Plugin(models.Plugin):
Provides a canonical name for plugins and serves as base for other
kinds of plugins.
"""
if not info:
logger.debug("Initialising {}".format(info))
if info:
self.update(info)
super(Plugin, self).__init__(**self)
if not self.validate():
raise models.Error(message=("You need to provide configuration"
"information for the plugin."))
logger.debug("Initialising {}".format(info))
id = 'plugins/{}_{}'.format(info['name'], info['version'])
super(Plugin, self).__init__(id=id, **info)
self.id = 'plugins/{}_{}'.format(self['name'], self['version'])
self.is_activated = False
self._lock = threading.Lock()
self.data_folder = data_folder or os.getcwd()
def validate(self):
return all(x in self for x in ('name', 'description', 'version'))
def get_folder(self):
return os.path.dirname(inspect.getfile(self.__class__))
......@@ -50,12 +53,21 @@ class Plugin(models.Plugin):
def test(self):
if not hasattr(self, 'test_cases'):
import inspect
raise AttributeError(('Plugin {} [{}] does not have any defined '
'test cases').format(self.id, inspect.getfile(self.__class__)))
for case in self.test_cases:
res = list(self.analyse_entry(models.Entry(case['entry']),
case['params']))
entry = models.Entry(case['entry'])
params = case.get('params', {})
fails = case.get('fails', False)
try:
res = list(self.analyse_entry(entry, params))
except models.Error:
if fails:
continue
raise
if fails:
raise Exception('This test should have raised an exception.')
exp = case['expected']
if not isinstance(exp, list):
exp = [exp]
......@@ -63,12 +75,13 @@ class Plugin(models.Plugin):
for r in res:
r.validate()
@contextmanager
def open(self, fpath, *args, **kwargs):
if not os.path.isabs(fpath):
fpath = os.path.join(self.data_folder, fpath)
with open(fpath, *args, **kwargs) as f:
yield f
return open(fpath, *args, **kwargs)
def serve(self, **kwargs):
utils.serve(plugin=self, **kwargs)
SenpyPlugin = Plugin
......@@ -106,17 +119,13 @@ class ConversionPlugin(Plugin):
class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin):
def __init__(self, info, *args, **kwargs):
super(SentimentPlugin, self).__init__(info, *args, **kwargs)
self.minPolarityValue = float(info.get("minPolarityValue", 0))
self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
minPolarityValue = 0
maxPolarityValue = 1
class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin):
def __init__(self, info, *args, **kwargs):
super(EmotionPlugin, self).__init__(info, *args, **kwargs)
self.minEmotionValue = float(info.get("minEmotionValue", -1))
self.maxEmotionValue = float(info.get("maxEmotionValue", 1))
minEmotionValue = 0
maxEmotionValue = 1
class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin):
......@@ -127,11 +136,11 @@ class ShelfMixin(object):
@property
def sh(self):
if not hasattr(self, '_sh') or self._sh is None:
self.__dict__['_sh'] = {}
self._sh = {}
if os.path.isfile(self.shelf_file):
try:
with self.open(self.shelf_file, 'rb') as p:
self.__dict__['_sh'] = pickle.load(p)
self._sh = pickle.load(p)
except (IndexError, EOFError, pickle.UnpicklingError):
logger.warning('{} has a corrupted shelf file!'.format(self.id))
if not self.get('force_shelf', False):
......@@ -142,9 +151,13 @@ class ShelfMixin(object):
def sh(self):
if os.path.isfile(self.shelf_file):
os.remove(self.shelf_file)
del self.__dict__['_sh']
del self._sh
self.save()
@sh.setter
def sh(self, value):
self._sh = value
@property
def shelf_file(self):
if 'shelf_file' not in self or not self['shelf_file']:
......@@ -196,7 +209,7 @@ def pfilter(plugins, **kwargs):
def validate_info(info):
return all(x in info for x in ('name', 'module', 'description', 'version'))
return all(x in info for x in ('name',))
def load_module(name, root=None):
......@@ -235,6 +248,17 @@ def install_deps(*plugins):
return installed
def get_plugin_class(module):
candidate = None
for _, obj in inspect.getmembers(module):
if inspect.isclass(obj) and inspect.getmodule(obj) == module:
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj)))
candidate = obj
break
return candidate
def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs):
if not root and '_path' in info:
root = os.path.dirname(info['_path'])
......@@ -249,18 +273,12 @@ def load_plugin_from_info(info, root=None, validator=validate_info, install=True
raise
install_deps(info)
tmp = load_module(module, root)
candidate = None
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj)))
candidate = obj
break
if not candidate:
logger.debug("No valid plugin for: {}".format(module))
return
module = candidate(info=info, *args, **kwargs)
return module
cls = None
if '@type' not in info:
cls = get_plugin_class(tmp)
if not cls:
raise Exception("No valid plugin for: {}".format(module))
return cls(info=info, *args, **kwargs)
def parse_plugin_info(fpath):
......
......@@ -45,7 +45,7 @@ class SplitPlugin(AnalysisPlugin):
},
{
'entry': {
"id": ":test",
"@id": ":test",
'nif:isString': 'Hello\nWorld'
},
'params': {
......
......@@ -23,3 +23,23 @@ def check_template(indict, template):
else:
if indict != template:
raise models.Error('{} and {} are different'.format(indict, template))
def easy(app=None, plugin=None, host='0.0.0.0', port=5000, **kwargs):
'''
Run a server with a specific plugin.
'''
from flask import Flask
from senpy.extensions import Senpy
if not app:
app = Flask(__name__)
sp = Senpy(app)
if plugin:
sp.add_plugin(plugin)
sp.install_deps()
app.run(host,
port,
debug=app.debug,
**kwargs)
......@@ -34,6 +34,11 @@ setup(
install_requires=install_reqs,
tests_require=test_reqs,
setup_requires=['pytest-runner', ],
extras_require={
'evaluation': [
'gsitk'
]
},
include_package_data=True,
entry_points={
'console_scripts':
......
from senpy.plugins import AnalysisPlugin
import multiprocessing
def _train(process_number):
return process_number
class AsyncPlugin(AnalysisPlugin):
def _do_async(self, num_processes):
pool = multiprocessing.Pool(processes=num_processes)
values = pool.map(_train, range(num_processes))
return values
def activate(self):
self.value = self._do_async(4)
def analyse_entry(self, entry, params):
values = self._do_async(2)
entry.async_values = values
yield entry
def test(self):
pass
---
name: Async
module: asyncplugin
description: I am async
author: "@balkian"
version: '0.1'
async: true
extra_params: {}
\ No newline at end of file
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
def analyse_entry(self, entry, params):
entry['nif:iString'] = entry['nif:isString'][::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry
def test(self):
pass
{
"name": "Dummy",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": false,
"default": 0
}
}
}
{
"name": "DummyRequired",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": true
}
}
}
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
import noop
from senpy.plugins import AnalysisPlugin
from time import sleep
class SleepPlugin(AnalysisPlugin):
def activate(self, *args, **kwargs):
sleep(self.timeout)
def analyse_entry(self, entry, params):
sleep(float(params.get("timeout", self.timeout)))
yield entry
def test(self):
pass
{
"name": "Sleep",
"module": "sleep",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"timeout": 0.05,
"extra_params": {
"timeout": {
"@id": "timeout_sleep",
"aliases": ["timeout", "to"],
"required": false,
"default": 0
}
}
}
......@@ -25,8 +25,8 @@ def analyse(instance, **kwargs):
class ExtensionsTest(TestCase):
def setUp(self):
self.app = Flask('test_extensions')
self.dir = os.path.dirname(__file__)
self.senpy = Senpy(plugin_folder=self.dir,
self.examples_dir = os.path.join(os.path.dirname(__file__), '..', 'example-plugins')
self.senpy = Senpy(plugin_folder=self.examples_dir,
app=self.app,
default_plugins=False)
self.senpy.activate_plugin("Dummy", sync=True)
......@@ -41,7 +41,7 @@ class ExtensionsTest(TestCase):
def test_discovery(self):
""" Discovery of plugins in given folders. """
# noinspection PyProtectedMember
assert self.dir in self.senpy._search_folders
assert self.examples_dir in self.senpy._search_folders
print(self.senpy.plugins)
assert "Dummy" in self.senpy.plugins
......@@ -54,7 +54,7 @@ class ExtensionsTest(TestCase):
'requirements': ['noop'],
'version': 0
}
root = os.path.join(self.dir, 'plugins', 'noop')
root = os.path.join(self.examples_dir, 'noop')
module = plugins.load_plugin_from_info(info, root=root, install=True)
assert module.name == 'TestPip'
assert module
......@@ -166,7 +166,7 @@ class ExtensionsTest(TestCase):
self.senpy.filter_plugins(name="Dummy", is_activated=True))
def test_load_default_plugins(self):
senpy = Senpy(plugin_folder=self.dir, default_plugins=True)
senpy = Senpy(plugin_folder=self.examples_dir, default_plugins=True)
assert len(senpy.plugins) > 1
def test_convert_emotions(self):
......
......@@ -100,6 +100,7 @@ class ModelsTest(TestCase):
def test_plugins(self):
self.assertRaises(Error, plugins.Plugin)
p = plugins.Plugin({"name": "dummy",
"description": "I do nothing",
"version": 0,
"extra_params": {
"none": {
......@@ -123,7 +124,9 @@ class ModelsTest(TestCase):
def test_str(self):
"""The string representation shouldn't include private variables"""
r = Results()
p = plugins.Plugin({"name": "STR test", "version": 0})
p = plugins.Plugin({"name": "STR test",
"description": "Test of private variables.",
"version": 0})
p._testing = 0
s = str(p)
assert "_testing" not in s
......
......@@ -43,6 +43,7 @@ class PluginsTest(TestCase):
def test_shelf_file(self):
a = ShelfDummyPlugin(
info={'name': 'default_shelve_file',
'description': 'Dummy plugin for tests',
'version': 'test'})
a.activate()
assert os.path.isfile(a.shelf_file)
......@@ -53,6 +54,7 @@ class PluginsTest(TestCase):
newfile = self.shelf_file + "new"
a = ShelfDummyPlugin(info={
'name': 'shelve',
'description': 'Shelf plugin for tests',
'version': 'test',
'shelf_file': newfile
})
......@@ -75,6 +77,7 @@ class PluginsTest(TestCase):
pickle.dump({'counter': 99}, f)
a = ShelfDummyPlugin(info={
'name': 'DummyShelf',
'description': 'Dummy plugin for tests',
'shelf_file': self.shelf_file,
'version': 'test'
})
......@@ -105,7 +108,8 @@ class PluginsTest(TestCase):
with open(fn, 'rb') as f:
msg, error = files[fn]
a = ShelfDummyPlugin(info={
'name': 'shelve',
'name': 'test_corrupt_shelf_{}'.format(msg),
'description': 'Dummy plugin for tests',
'version': 'test',
'shelf_file': f.name
})
......@@ -126,6 +130,7 @@ class PluginsTest(TestCase):
''' Reusing the values of a previous shelf '''
a = ShelfDummyPlugin(info={
'name': 'shelve',
'description': 'Dummy plugin for tests',
'version': 'test',
'shelf_file': self.shelf_file
})
......@@ -136,6 +141,7 @@ class PluginsTest(TestCase):
b = ShelfDummyPlugin(info={
'name': 'shelve',
'description': 'Dummy plugin for tests',
'version': 'test',
'shelf_file': self.shelf_file
})
......@@ -148,6 +154,7 @@ class PluginsTest(TestCase):
''' Should be able to set extra parameters'''
a = ShelfDummyPlugin(info={
'name': 'shelve',
'description': 'Dummy shelf plugin for tests',
'version': 'test',
'shelf_file': self.shelf_file,
'extra_params': {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment