Commit 48d7d1d0 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Improved plugins API and loading

Also:

* added drone-ci integration: tests for py2.7 and py3
parent 14c9f618
build:
image: python:$$PYTHON_VERSION
commands:
- python setup.py test
matrix:
PYTHON_VERSION:
- 2.7
- 3.4
...@@ -63,7 +63,9 @@ def main(): ...@@ -63,7 +63,9 @@ def main():
default="plugins", default="plugins",
help='Where to look for plugins.') help='Where to look for plugins.')
args = parser.parse_args() args = parser.parse_args()
logging.basicConfig(level=getattr(logging, args.level)) logging.basicConfig()
rl = logging.getLogger()
rl.setLevel(getattr(logging, args.level))
app = Flask(__name__) app = Flask(__name__)
app.debug = args.debug app.debug = args.debug
sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins) sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
......
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
Blueprints for Senpy Blueprints for Senpy
""" """
from flask import Blueprint, request, current_app, render_template from flask import Blueprint, request, current_app, render_template
from .models import Error, Response from .models import Error, Response, Plugins
from future.utils import iteritems from future.utils import iteritems
import json import json
...@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__) ...@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__) nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__) demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__)
BASIC_PARAMS = { API_PARAMS = {
"algorithm": { "algorithm": {
"aliases": ["algorithm", "a", "algo"], "aliases": ["algorithm", "a", "algo"],
"required": False, "required": False,
...@@ -41,6 +41,63 @@ BASIC_PARAMS = { ...@@ -41,6 +41,63 @@ BASIC_PARAMS = {
} }
} }
BASIC_PARAMS = {
"algorithm": {
"aliases": ["algorithm", "a", "algo"],
"required": False,
},
"inHeaders": {
"aliases": ["inHeaders", "headers"],
"required": True,
"default": "0"
},
"input": {
"@id": "input",
"aliases": ["i", "input"],
"required": True,
"help": "Input text"
},
"informat": {
"@id": "informat",
"aliases": ["f", "informat"],
"required": False,
"default": "text",
"options": ["turtle", "text"],
},
"intype": {
"@id": "intype",
"aliases": ["intype", "t"],
"required": False,
"default": "direct",
"options": ["direct", "url", "file"],
},
"outformat": {
"@id": "outformat",
"aliases": ["outformat", "o"],
"default": "json-ld",
"required": False,
"options": ["json-ld"],
},
"language": {
"@id": "language",
"aliases": ["language", "l"],
"required": False,
},
"prefix": {
"@id": "prefix",
"aliases": ["prefix", "p"],
"required": True,
"default": "",
},
"urischeme": {
"@id": "urischeme",
"aliases": ["urischeme", "u"],
"required": False,
"default": "RFC5147String",
"options": "RFC5147String"
},
}
def get_params(req, params=BASIC_PARAMS): def get_params(req, params=BASIC_PARAMS):
if req.method == 'POST': if req.method == 'POST':
indict = req.form indict = req.form
...@@ -119,36 +176,29 @@ def api(): ...@@ -119,36 +176,29 @@ def api():
return ex.message.flask() return ex.message.flask()
@nif_blueprint.route("/default")
def default():
# return current_app.senpy.default_plugin
plug = current_app.senpy.default_plugin
if plug:
return plugins(action="list", plugin=plug.name)
else:
error = Error(status=404, message="No plugins found")
return error.flask()
@nif_blueprint.route('/plugins/', methods=['POST', 'GET']) @nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
def plugins():
in_headers = get_params(request, API_PARAMS)["inHeaders"] != "0"
sp = current_app.senpy
dic = Plugins(plugins=list(sp.plugins.values()))
return dic.flask(in_headers=in_headers)
@nif_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET']) @nif_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET']) @nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
def plugins(plugin=None, action="list"): def plugin(plugin=None, action="list"):
filt = {} filt = {}
sp = current_app.senpy sp = current_app.senpy
if plugin: plugs = sp.filter_plugins(name=plugin)
filt["name"] = plugin if plugin == 'default' and sp.default_plugin:
plugs = sp.filter_plugins(**filt) response = sp.default_plugin
if plugin and not plugs: plugin = response.name
return "Plugin not found", 400 elif plugin in sp.plugins:
response = sp.plugins[plugin]
else:
return Error(message="Plugin not found", status=404).flask()
if action == "list": if action == "list":
in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0" in_headers = get_params(request, API_PARAMS)["inHeaders"] != "0"
if plugin: return response.flask(in_headers=in_headers)
dic = plugs[plugin]
else:
dic = Response(
{plug: plugs[plug].serializable() for plug in plugs})
return dic.flask(in_headers=in_headers)
method = "{}_plugin".format(action) method = "{}_plugin".format(action)
if(hasattr(sp, method)): if(hasattr(sp, method)):
getattr(sp, method)(plugin) getattr(sp, method)(plugin)
...@@ -156,7 +206,6 @@ def plugins(plugin=None, action="list"): ...@@ -156,7 +206,6 @@ def plugins(plugin=None, action="list"):
else: else:
return Error(message="action '{}' not allowed".format(action)).flask() return Error(message="action '{}' not allowed".format(action)).flask()
if __name__ == '__main__': if __name__ == '__main__':
import config import config
......
...@@ -34,6 +34,7 @@ class Senpy(object): ...@@ -34,6 +34,7 @@ class Senpy(object):
self.app = app self.app = app
self._search_folders = set() self._search_folders = set()
self._plugin_list = []
self._outdated = True self._outdated = True
self.add_folder(plugin_folder) self.add_folder(plugin_folder)
...@@ -65,10 +66,8 @@ class Senpy(object): ...@@ -65,10 +66,8 @@ class Senpy(object):
if os.path.isdir(folder): if os.path.isdir(folder):
self._search_folders.add(folder) self._search_folders.add(folder)
self._outdated = True self._outdated = True
return True
else: else:
logger.debug("Not a folder: %s", folder) logger.debug("Not a folder: %s", folder)
return False
def analyse(self, **params): def analyse(self, **params):
algo = None algo = None
...@@ -113,7 +112,7 @@ class Senpy(object): ...@@ -113,7 +112,7 @@ class Senpy(object):
def parameters(self, algo): def parameters(self, algo):
return getattr(self.plugins.get(algo) or self.default_plugin, return getattr(self.plugins.get(algo) or self.default_plugin,
"params", "extra_params",
{}) {})
def activate_all(self, sync=False): def activate_all(self, sync=False):
...@@ -129,13 +128,18 @@ class Senpy(object): ...@@ -129,13 +128,18 @@ class Senpy(object):
return ps return ps
def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs): def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
''' We're using a variable in the plugin itself to activate/deactive plugins.\
Note that plugins may activate themselves by setting this variable.
'''
self.plugins[plugin_name].is_activated = active self.plugins[plugin_name].is_activated = active
def activate_plugin(self, plugin_name, sync=False): def activate_plugin(self, plugin_name, sync=False):
plugin = self.plugins[plugin_name] plugin = self.plugins[plugin_name]
logger.info("Activating plugin: {}".format(plugin.name))
def act(): def act():
try: try:
plugin.activate() plugin.activate()
logger.info("Plugin activated: {}".format(plugin.name))
except Exception as ex: except Exception as ex:
logger.error("Error activating plugin {}: {}".format(plugin.name, logger.error("Error activating plugin {}: {}".format(plugin.name,
ex)) ex))
...@@ -149,19 +153,33 @@ class Senpy(object): ...@@ -149,19 +153,33 @@ class Senpy(object):
def deactivate_plugin(self, plugin_name, sync=False): def deactivate_plugin(self, plugin_name, sync=False):
plugin = self.plugins[plugin_name] plugin = self.plugins[plugin_name]
th = gevent.spawn(plugin.deactivate)
def deact():
try:
plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name))
except Exception as ex:
logger.error("Error deactivating plugin {}: {}".format(plugin.name,
ex))
logger.error("Trace: {}".format(traceback.format_exc()))
th = gevent.spawn(deact)
th.link_value(partial(self._set_active_plugin, plugin_name, False)) th.link_value(partial(self._set_active_plugin, plugin_name, False))
if sync: if sync:
th.join() th.join()
else: else:
return th return th
def reload_plugin(self, plugin): def reload_plugin(self, name):
logger.debug("Reloading {}".format(plugin)) logger.debug("Reloading {}".format(name))
plug = self.plugins[plugin] plugin = self.plugins[name]
nplug = self._load_plugin(plug.module, plug.path) try:
del self.plugins[plugin] del self.plugins[name]
self.plugins[nplug.name] = nplug nplug = self._load_plugin(plugin.module, plugin.path)
self.plugins[nplug.name] = nplug
except Exception as ex:
logger.error('Error reloading {}: {}'.format(name, ex))
self.plugins[name] = plugin
@staticmethod @staticmethod
def _load_plugin(root, filename): def _load_plugin(root, filename):
...@@ -206,7 +224,7 @@ class Senpy(object): ...@@ -206,7 +224,7 @@ class Senpy(object):
for root, dirnames, filenames in os.walk(search_folder): for root, dirnames, filenames in os.walk(search_folder):
for filename in fnmatch.filter(filenames, '*.senpy'): for filename in fnmatch.filter(filenames, '*.senpy'):
name, plugin = self._load_plugin(root, filename) name, plugin = self._load_plugin(root, filename)
if plugin: if plugin and name not in self._plugin_list:
plugins[name] = plugin plugins[name] = plugin
self._outdated = False self._outdated = False
...@@ -218,9 +236,9 @@ class Senpy(object): ...@@ -218,9 +236,9 @@ class Senpy(object):
@property @property
def plugins(self): def plugins(self):
""" Return the plugins registered for a given application. """ """ Return the plugins registered for a given application. """
if not hasattr(self, 'senpy_plugins') or self._outdated: if self._outdated:
self.senpy_plugins = self._load_plugins() self._plugin_list = self._load_plugins()
return self.senpy_plugins return self._plugin_list
def filter_plugins(self, **kwargs): def filter_plugins(self, **kwargs):
""" Filter plugins by different criteria """ """ Filter plugins by different criteria """
......
...@@ -117,11 +117,16 @@ class SenpyMixin(object): ...@@ -117,11 +117,16 @@ class SenpyMixin(object):
sort_keys=True) sort_keys=True)
return js return js
def validate(self, obj=None):
if not obj:
obj = self
if hasattr(obj, "jsonld"):
obj = obj.jsonld()
jsonschema.validate(obj, self.schema)
class SenpyModel(SenpyMixin, dict): class SenpyModel(SenpyMixin, dict):
schema = base_schema schema = base_schema
prefix = None
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
temp = dict(*args, **kwargs) temp = dict(*args, **kwargs)
...@@ -161,14 +166,6 @@ class SenpyModel(SenpyMixin, dict): ...@@ -161,14 +166,6 @@ class SenpyModel(SenpyMixin, dict):
def __delattr__(self, key): def __delattr__(self, key):
self.__delitem__(self._get_key(key)) self.__delitem__(self._get_key(key))
def validate(self, obj=None):
if not obj:
obj = self
if hasattr(obj, "jsonld"):
obj = obj.jsonld()
jsonschema.validate(obj, self.schema)
@classmethod @classmethod
......
...@@ -9,55 +9,6 @@ from .models import Response, PluginModel, Error ...@@ -9,55 +9,6 @@ from .models import Response, PluginModel, Error
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
PARAMS = {
"input": {
"@id": "input",
"aliases": ["i", "input"],
"required": True,
"help": "Input text"
},
"informat": {
"@id": "informat",
"aliases": ["f", "informat"],
"required": False,
"default": "text",
"options": ["turtle", "text"],
},
"intype": {
"@id": "intype",
"aliases": ["intype", "t"],
"required": False,
"default": "direct",
"options": ["direct", "url", "file"],
},
"outformat": {
"@id": "outformat",
"aliases": ["outformat", "o"],
"default": "json-ld",
"required": False,
"options": ["json-ld"],
},
"language": {
"@id": "language",
"aliases": ["language", "l"],
"required": False,
},
"prefix": {
"@id": "prefix",
"aliases": ["prefix", "p"],
"required": True,
"default": "",
},
"urischeme": {
"@id": "urischeme",
"aliases": ["urischeme", "u"],
"required": False,
"default": "RFC5147String",
"options": "RFC5147String"
},
}
class SenpyPlugin(PluginModel): class SenpyPlugin(PluginModel):
def __init__(self, info=None): def __init__(self, info=None):
...@@ -65,14 +16,12 @@ class SenpyPlugin(PluginModel): ...@@ -65,14 +16,12 @@ class SenpyPlugin(PluginModel):
raise Error(message=("You need to provide configuration" raise Error(message=("You need to provide configuration"
"information for the plugin.")) "information for the plugin."))
logger.debug("Initialising {}".format(info)) logger.debug("Initialising {}".format(info))
self.name = info["name"] super(SenpyPlugin, self).__init__(info)
self.version = info["version"] self.params = info.get("extra_params", {})
self.params = info.get("params", PARAMS.copy()) self._info = info
if "@id" not in self.params: if "@id" not in self.params:
self.params["@id"] = "params_%s" % self.id self.params["@id"] = "params_%s" % self.id
self.is_activated = False self.is_activated = False
self._info = info
super(SenpyPlugin, self).__init__()
def get_folder(self): def get_folder(self):
return os.path.dirname(inspect.getfile(self.__class__)) return os.path.dirname(inspect.getfile(self.__class__))
......
...@@ -15,7 +15,7 @@ except AttributeError: ...@@ -15,7 +15,7 @@ except AttributeError:
install_reqs = [str(ir.req) for ir in install_reqs] install_reqs = [str(ir.req) for ir in install_reqs]
test_reqs = [str(ir.req) for ir in test_reqs] test_reqs = [str(ir.req) for ir in test_reqs]
VERSION = "0.5" VERSION = "0.5.1"
setup( setup(
name='senpy', name='senpy',
......
...@@ -56,7 +56,10 @@ class BlueprintsTest(TestCase): ...@@ -56,7 +56,10 @@ class BlueprintsTest(TestCase):
resp = self.client.get("/api/plugins/") resp = self.client.get("/api/plugins/")
self.assert200(resp) self.assert200(resp)
logging.debug(resp.json) logging.debug(resp.json)
assert "Dummy" in resp.json assert 'plugins' in resp.json
plugins = resp.json['plugins']
assert len(plugins) > 1
assert list(p for p in plugins if p['name'] == "Dummy")
assert "@context" in resp.json assert "@context" in resp.json
def test_headers(self): def test_headers(self):
...@@ -98,7 +101,7 @@ class BlueprintsTest(TestCase): ...@@ -98,7 +101,7 @@ class BlueprintsTest(TestCase):
def test_default(self): def test_default(self):
""" Show only one plugin""" """ Show only one plugin"""
resp = self.client.get("/api/default") resp = self.client.get("/api/plugins/default/")
self.assert200(resp) self.assert200(resp)
logging.debug(resp.json) logging.debug(resp.json)
assert "@id" in resp.json assert "@id" in resp.json
...@@ -106,5 +109,5 @@ class BlueprintsTest(TestCase): ...@@ -106,5 +109,5 @@ class BlueprintsTest(TestCase):
resp = self.client.get("/api/plugins/Dummy/deactivate") resp = self.client.get("/api/plugins/Dummy/deactivate")
self.assert200(resp) self.assert200(resp)
sleep(0.5) sleep(0.5)
resp = self.client.get("/api/default") resp = self.client.get("/api/plugins/default/")
self.assert404(resp) self.assert404(resp)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment