Commit ff149250 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Improved plugins, better tests, gevent

Moved from Yapsy again (it is not flexible enough), now we use a
custom solution.
The activation and deactivation of plugins is asynchronous, so
that plugins that take a long time don't interfere with the rest.
parent 10f4782a
...@@ -19,18 +19,25 @@ Simple Sentiment Analysis server for EUROSENTIMENT ...@@ -19,18 +19,25 @@ Simple Sentiment Analysis server for EUROSENTIMENT
This class shows how to use the nif_server module to create custom services. This class shows how to use the nif_server module to create custom services.
""" """
from gevent.monkey import patch_all; patch_all()
import gevent
import config import config
from flask import Flask from flask import Flask
from senpy.extensions import Senpy from senpy.extensions import Senpy
import logging import logging
import os import os
from gevent.wsgi import WSGIServer
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
app = Flask(__name__) app = Flask(__name__)
mypath = os.path.dirname(os.path.realpath(__file__)) mypath = os.path.dirname(os.path.realpath(__file__))
sp = Senpy(app, os.path.join(mypath, "plugins")) sp = Senpy(app, os.path.join(mypath, "plugins"))
sp.activate_all()
if __name__ == '__main__': if __name__ == '__main__':
import logging
logging.basicConfig(level=config.DEBUG)
app.debug = config.DEBUG app.debug = config.DEBUG
app.run(host="0.0.0.0", use_reloader=False) http_server = WSGIServer(('', 5000), app)
http_server.serve_forever()
\ No newline at end of file
...@@ -6,19 +6,6 @@ from senpy.models import Response, Opinion, Entry ...@@ -6,19 +6,6 @@ from senpy.models import Response, Opinion, Entry
class Sentiment140Plugin(SentimentPlugin): class Sentiment140Plugin(SentimentPlugin):
EXTRA_PARAMS = {
"language": {"aliases": ["language", "l"],
"required": False,
"options": ["es", "en", "auto"],
}
}
def __init__(self, **kwargs):
super(Sentiment140Plugin, self).__init__(name="sentiment140",
version="2.0",
extraparams=self.EXTRA_PARAMS,
**kwargs)
def analyse(self, **params): def analyse(self, **params):
lang = params.get("language", "auto") lang = params.get("language", "auto")
res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson", res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
...@@ -45,5 +32,3 @@ class Sentiment140Plugin(SentimentPlugin): ...@@ -45,5 +32,3 @@ class Sentiment140Plugin(SentimentPlugin):
entry.language = lang entry.language = lang
response.entries.append(entry) response.entries.append(entry)
return response return response
plugin = Sentiment140Plugin()
{
"name": "sentiment140",
"module": "sentiment140",
"description": "What my plugin broadly does",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"language": {
"aliases": ["language", "l"],
"required": false,
"options": ["es", "en", "auto"]
}
},
"requirements": {}
}
Flask==0.10.1 Flask==0.10.1
gunicorn==19.0.0 gunicorn==19.0.0
requests==2.4.1 requests==2.4.1
GitPython==0.3.2.RC1 GitPython==0.3.2.RC1
\ No newline at end of file Yapsy>=1.10.423
gevent>=1.0.1
\ No newline at end of file
...@@ -110,23 +110,22 @@ def default(): ...@@ -110,23 +110,22 @@ def default():
@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET']) @nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
def plugins(plugin=None, action="list"): def plugins(plugin=None, action="list"):
filt = {} filt = {}
sp = current_app.senpy
if plugin: if plugin:
filt["name"] = plugin filt["name"] = plugin
plugs = current_app.senpy.filter_plugins(**filt) plugs = sp.filter_plugins(**filt)
if plugin and not plugs: if plugin and not plugs:
return "Plugin not found", 400 return "Plugin not found", 400
if action == "list": if action == "list":
with_params = request.args.get("params", "") == "1" with_params = request.args.get("params", "") == "1"
dic = {plug: plugs[plug].jsonable(with_params) for plug in plugs} if plugin:
dic = plugs[plugin].jsonable(with_params)
else:
dic = {plug: plugs[plug].jsonable(with_params) for plug in plugs}
return jsonify(dic) return jsonify(dic)
if action == "disable": method = "{}_plugin".format(action)
current_app.senpy.disable_plugin(plugin) if(hasattr(sp, method)):
return "Ok" getattr(sp, method)(plugin)
elif action == "enable":
current_app.senpy.enable_plugin(plugin)
return "Ok"
elif action == "reload":
current_app.senpy.reload_plugin(plugin)
return "Ok" return "Ok"
else: else:
return "action '{}' not allowed".format(action), 400 return "action '{}' not allowed".format(action), 400
......
""" """
""" """
import os import os
import fnmatch
import inspect
import sys import sys
import imp import imp
import logging import logging
import gevent
import json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from .plugins import SentimentPlugin, EmotionPlugin from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .blueprints import nif_blueprint from .blueprints import nif_blueprint
from git import Repo, InvalidGitRepositoryError from git import Repo, InvalidGitRepositoryError
from functools import partial
class Senpy(object): class Senpy(object):
...@@ -45,11 +50,13 @@ class Senpy(object): ...@@ -45,11 +50,13 @@ class Senpy(object):
app.register_blueprint(nif_blueprint) app.register_blueprint(nif_blueprint)
def add_folder(self, folder): def add_folder(self, folder):
logger.debug("Adding folder: %s", folder)
if os.path.isdir(folder): if os.path.isdir(folder):
self._search_folders.add(folder) self._search_folders.add(folder)
self._outdated = True self._outdated = True
return True return True
else: else:
logger.debug("Not a folder: %s", folder)
return False return False
def analyse(self, **params): def analyse(self, **params):
...@@ -59,17 +66,20 @@ class Senpy(object): ...@@ -59,17 +66,20 @@ class Senpy(object):
algo = params["algorithm"] algo = params["algorithm"]
elif self.plugins: elif self.plugins:
algo = self.default_plugin algo = self.default_plugin
if algo in self.plugins and self.plugins[algo].enabled: if algo in self.plugins:
plug = self.plugins[algo] if self.plugins[algo].is_activated:
resp = plug.analyse(**params) plug = self.plugins[algo]
resp.analysis.append(plug.jsonable()) resp = plug.analyse(**params)
return resp resp.analysis.append(plug.jsonable())
return resp
logger.debug("Plugin not activated: {}".format(algo))
else: else:
logger.debug("The algorithm '{}' is not valid\nValid algorithms: {}".format(algo, self.plugins.keys()))
return {"status": 400, "message": "The algorithm '{}' is not valid".format(algo)} return {"status": 400, "message": "The algorithm '{}' is not valid".format(algo)}
@property @property
def default_plugin(self): def default_plugin(self):
candidates = self.filter_plugins(enabled=True) candidates = self.filter_plugins(is_activated=True)
if len(candidates) > 0: if len(candidates) > 0:
candidate = candidates.keys()[0] candidate = candidates.keys()[0]
logger.debug("Default: {}".format(candidate)) logger.debug("Default: {}".format(candidate))
...@@ -80,11 +90,38 @@ class Senpy(object): ...@@ -80,11 +90,38 @@ class Senpy(object):
def parameters(self, algo): def parameters(self, algo):
return getattr(self.plugins.get(algo or self.default_plugin), "params", {}) return getattr(self.plugins.get(algo or self.default_plugin), "params", {})
def enable_plugin(self, plugin): def activate_all(self, sync=False):
self.plugins[plugin].enable() ps = []
for plug in self.plugins.keys():
def disable_plugin(self, plugin): ps.append(self.activate_plugin(plug, sync=sync))
self.plugins[plugin].disable() return ps
def deactivate_all(self, sync=False):
ps = []
for plug in self.plugins.keys():
ps.append(self.deactivate_plugin(plug, sync=sync))
return ps
def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
self.plugins[plugin_name].is_activated = active
def activate_plugin(self, plugin_name, sync=False):
plugin = self.plugins[plugin_name]
th = gevent.spawn(plugin.activate)
th.link_value(partial(self._set_active_plugin, plugin_name, True))
if sync:
th.join()
else:
return th
def deactivate_plugin(self, plugin_name, sync=False):
plugin = self.plugins[plugin_name]
th = gevent.spawn(plugin.deactivate)
th.link_value(partial(self._set_active_plugin, plugin_name, False))
if sync:
th.join()
else:
return th
def reload_plugin(self, plugin): def reload_plugin(self, plugin):
logger.debug("Reloading {}".format(plugin)) logger.debug("Reloading {}".format(plugin))
...@@ -94,38 +131,47 @@ class Senpy(object): ...@@ -94,38 +131,47 @@ class Senpy(object):
self.plugins[nplug.name] = nplug self.plugins[nplug.name] = nplug
@staticmethod @staticmethod
def _load_plugin(plugin, search_folder, enabled=True): def _load_plugin(root, filename):
logger.debug("Loading plugins") logger.debug("Loading plugin: {}".format(filename))
sys.path.append(search_folder) fpath = os.path.join(root, filename)
(fp, pathname, desc) = imp.find_module(plugin) with open(fpath,'r') as f:
info = json.load(f)
logger.debug("Info: {}".format(info))
sys.path.append(root)
module = info["module"]
name = info["name"]
(fp, pathname, desc) = imp.find_module(module, [root,])
try: try:
tmp = imp.load_module(plugin, fp, pathname, desc).plugin tmp = imp.load_module(module, fp, pathname, desc)
sys.path.remove(search_folder) sys.path.remove(root)
tmp.path = search_folder candidate = None
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug("Found plugin class: {}@{}".format(obj, inspect.getmodule(obj)))
candidate = obj
break
if not candidate:
logger.debug("No valid plugin for: {}".format(filename))
return
module = candidate(info=info)
try: try:
repo_path = os.path.join(search_folder, plugin) repo_path = root
tmp.repo = Repo(repo_path) module.repo = Repo(repo_path)
except InvalidGitRepositoryError: except InvalidGitRepositoryError:
tmp.repo = None module.repo = None
if not hasattr(tmp, "enabled"):
tmp.enabled = enabled
tmp.module = plugin
except Exception as ex: except Exception as ex:
tmp = None logger.debug("Exception importing {}: {}".format(filename, ex))
logger.debug("Exception importing {}: {}".format(plugin, ex)) return None, None
return tmp return name, module
def _load_plugins(self): def _load_plugins(self):
plugins = {} plugins = {}
for search_folder in self._search_folders: for search_folder in self._search_folders:
for item in os.listdir(search_folder): for root, dirnames, filenames in os.walk(search_folder):
if os.path.isdir(os.path.join(search_folder, item)) \ for filename in fnmatch.filter(filenames, '*.senpy'):
and os.path.exists(os.path.join(search_folder, name, plugin = self._load_plugin(root, filename)
item,
"__init__.py")):
plugin = self._load_plugin(item, search_folder)
if plugin: if plugin:
plugins[plugin.name] = plugin plugins[name] = plugin
self._outdated = False self._outdated = False
return plugins return plugins
...@@ -133,10 +179,6 @@ class Senpy(object): ...@@ -133,10 +179,6 @@ class Senpy(object):
def teardown(self, exception): def teardown(self, exception):
pass pass
def enable_all(self):
for plugin in self.plugins:
self.enable_plugin(plugin)
@property @property
def plugins(self): def plugins(self):
""" Return the plugins registered for a given application. """ """ Return the plugins registered for a given application. """
......
...@@ -77,7 +77,7 @@ class Entry(Leaf): ...@@ -77,7 +77,7 @@ class Entry(Leaf):
class Opinion(Leaf): class Opinion(Leaf):
opinionContext = {} opinionContext = {"@vocab": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"}
def __init__(self, polarityValue=None, hasPolarity=None, *args, **kwargs): def __init__(self, polarityValue=None, hasPolarity=None, *args, **kwargs):
super(Opinion, self).__init__(context=self.opinionContext, super(Opinion, self).__init__(context=self.opinionContext,
*args, *args,
...@@ -93,7 +93,14 @@ class EmotionSet(Leaf): ...@@ -93,7 +93,14 @@ class EmotionSet(Leaf):
def __init__(self, emotions=None, *args, **kwargs): def __init__(self, emotions=None, *args, **kwargs):
if not emotions: if not emotions:
emotions = [] emotions = []
super(EmotionSet, self).__init__(context=self.emotionContext, super(EmotionSet, self).__init__(context=EmotionSet.emotionContext,
*args, *args,
**kwargs) **kwargs)
self.emotions = emotions or [] self.emotions = emotions or []
class Emotion(Leaf):
emotionContext = {}
def __init__(self, emotions=None, *args, **kwargs):
super(EmotionSet, self).__init__(context=Emotion.emotionContext,
*args,
**kwargs)
import logging import logging
import ConfigParser
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -38,27 +39,28 @@ PARAMS = {"input": {"aliases": ["i", "input"], ...@@ -38,27 +39,28 @@ PARAMS = {"input": {"aliases": ["i", "input"],
class SenpyPlugin(object): class SenpyPlugin(object):
def __init__(self, name=None, version=None, extraparams=None, params=None): def __init__(self, info=None):
logger.debug("Initialising {}".format(name)) if not info:
self.name = name raise ValueError("You need to provide configuration information for the plugin.")
self.version = version logger.debug("Initialising {}".format(info))
if params: self.name = info["name"]
self.params = params self.version = info["version"]
else: self.params = info.get("params", PARAMS.copy())
self.params = PARAMS.copy() self.extra_params = info.get("extra_params", {})
if extraparams: self.params.update(self.extra_params)
self.params.update(extraparams) self.is_activated = False
self.extraparams = extraparams or {} self.info = info
self.enabled = True
def analyse(self, *args, **kwargs): def analyse(self, *args, **kwargs):
logger.debug("Analysing with: {} {}".format(self.name, self.version))
pass pass
def enable(self): def activate(self):
self.enabled = True pass
def deactivate(self):
pass
def disable(self):
self.enabled = False
@property @property
def id(self): def id(self):
...@@ -66,15 +68,15 @@ class SenpyPlugin(object): ...@@ -66,15 +68,15 @@ class SenpyPlugin(object):
def jsonable(self, parameters=False): def jsonable(self, parameters=False):
resp = { resp = {
"@id": self.id, "@id": "{}_{}".format(self.name, self.version),
"enabled": self.enabled, "is_activated": self.is_activated,
} }
if hasattr(self, "repo") and self.repo: if hasattr(self, "repo") and self.repo:
resp["repo"] = self.repo.remotes[0].url resp["repo"] = self.repo.remotes[0].url
if parameters: if parameters:
resp["parameters"] = self.params resp["parameters"] = self.params
elif self.extraparams: elif self.extra_params:
resp["extra_parameters"] = self.extraparams resp["extra_parameters"] = self.extra_params
return resp return resp
......
...@@ -8,7 +8,7 @@ install_reqs = parse_requirements("requirements.txt") ...@@ -8,7 +8,7 @@ install_reqs = parse_requirements("requirements.txt")
# e.g. ['django==1.5.1', 'mezzanine==1.4.6'] # e.g. ['django==1.5.1', 'mezzanine==1.4.6']
reqs = [str(ir.req) for ir in install_reqs] reqs = [str(ir.req) for ir in install_reqs]
VERSION = "0.2.9" VERSION = "0.3.0"
print(reqs) print(reqs)
......
...@@ -9,6 +9,7 @@ except ImportError: ...@@ -9,6 +9,7 @@ except ImportError:
from senpy.extensions import Senpy from senpy.extensions import Senpy
from flask import Flask from flask import Flask
from flask.ext.testing import TestCase from flask.ext.testing import TestCase
from gevent import sleep
def check_dict(indic, template): def check_dict(indic, template):
...@@ -22,6 +23,7 @@ class BlueprintsTest(TestCase): ...@@ -22,6 +23,7 @@ class BlueprintsTest(TestCase):
self.senpy.init_app(self.app) self.senpy.init_app(self.app)
self.dir = os.path.join(os.path.dirname(__file__), "..") self.dir = os.path.join(os.path.dirname(__file__), "..")
self.senpy.add_folder(self.dir) self.senpy.add_folder(self.dir)
self.senpy.activate_plugin("Dummy", sync=True)
return self.app return self.app
def test_home(self): def test_home(self):
...@@ -36,3 +38,43 @@ class BlueprintsTest(TestCase): ...@@ -36,3 +38,43 @@ class BlueprintsTest(TestCase):
} }
assert check_dict(resp.json, atleast) assert check_dict(resp.json, atleast)
def test_analysis(self):
""" The dummy plugin returns an empty response, it should contain the context """
resp = self.client.get("/?i=My aloha mohame")
self.assert200(resp)
logging.debug(resp.json)
assert "@context" in resp.json
assert check_dict(resp.json["@context"], {"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"})
assert "entries" in resp.json
def test_list(self):
""" List the plugins """
resp = self.client.get("/plugins/")
self.assert200(resp)
logging.debug(resp.json)
assert "Dummy" in resp.json
def test_detail(self):
""" Show only one plugin"""
resp = self.client.get("/plugins/Dummy")
self.assert200(resp)
logging.debug(resp.json)
assert "@id" in resp.json
assert resp.json["@id"] == "Dummy_0.1"
def test_activate(self):
""" Activate and deactivate one plugin """
resp = self.client.get("/plugins/Dummy/deactivate")
self.assert200(resp)
sleep(0.5)
resp = self.client.get("/plugins/Dummy")
self.assert200(resp)
assert "is_activated" in resp.json
assert resp.json["is_activated"] == False
resp = self.client.get("/plugins/Dummy/activate")
self.assert200(resp)
sleep(0.5)
resp = self.client.get("/plugins/Dummy")
self.assert200(resp)
assert "is_activated" in resp.json
assert resp.json["is_activated"] == True
from senpy.plugins import SenpyPlugin
plugin = SenpyPlugin("dummy")
from senpy.plugins import SentimentPlugin
from senpy.models import Response
class DummyPlugin(SentimentPlugin):
pass
def analyse(self, *args, **kwargs):