Commit eaf65f0c authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

First tests

parent a5e79bea
from senpy.plugins import SenpyPlugin
class Prueba(SenpyPlugin):
def __init__(self, **kwargs):
super(Prueba, self).__init__(name="prueba",
version="4.0",
**kwargs)
plugin = Prueba()
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
Sentiment analysis server in Python Sentiment analysis server in Python
''' '''
VERSION = "0.2.5" VERSION = "0.2.6"
import extensions import extensions
import blueprints import blueprints
......
...@@ -17,8 +17,11 @@ ...@@ -17,8 +17,11 @@
''' '''
Simple Sentiment Analysis server Simple Sentiment Analysis server
''' '''
from flask import Blueprint, render_template, request, jsonify, current_app
import json import json
import logging
logger = logging.getLogger(__name__)
from flask import Blueprint, render_template, request, jsonify, current_app
nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__) nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
...@@ -86,10 +89,12 @@ def home(entries=None): ...@@ -86,10 +89,12 @@ def home(entries=None):
algo = get_params(request).get("algorithm", None) algo = get_params(request).get("algorithm", None)
specific_params = current_app.senpy.parameters(algo) specific_params = current_app.senpy.parameters(algo)
params = get_params(request, specific_params) params = get_params(request, specific_params)
response = current_app.senpy.analyse(**params)
return jsonify(response)
except ValueError as ex: except ValueError as ex:
return ex.message return ex.message
response = current_app.senpy.analyse(**params) except Exception as ex:
return jsonify(response) return jsonify(status="400", message=ex.message)
@nif_blueprint.route("/default") @nif_blueprint.route("/default")
def default(): def default():
......
import os import os
import sys import sys
import imp import imp
import logging
logger = logging.getLogger(__name__)
from flask import current_app
from collections import defaultdict
from .plugins import SentimentPlugin, EmotionPlugin from .plugins import SentimentPlugin, EmotionPlugin
try: try:
...@@ -11,29 +12,33 @@ try: ...@@ -11,29 +12,33 @@ try:
except ImportError: except ImportError:
from flask import _request_ctx_stack as stack from flask import _request_ctx_stack as stack
from blueprints import nif_blueprint from .blueprints import nif_blueprint
from git import Repo, InvalidGitRepositoryError from git import Repo, InvalidGitRepositoryError
class Senpy(object): class Senpy(object):
""" Default Senpy extension for Flask """
def __init__(self, app=None, plugin_folder="plugins"): def __init__(self, app=None, plugin_folder="plugins"):
self.app = app self.app = app
base_folder = os.path.join(os.path.dirname(__file__), "plugins") base_folder = os.path.join(os.path.dirname(__file__), "plugins")
self.search_folders = (folder for folder in (base_folder, plugin_folder, '/tmp/plugins') self._search_folders = set()
if folder and os.path.isdir(folder)) self._outdated = True
for folder in (base_folder, plugin_folder):
self.add_folder(folder)
if app is not None: if app is not None:
self.init_app(app) self.init_app(app)
""" def init_app(self, app):
""" Initialise a flask app to add plugins to its context """
"""
Note: I'm not particularly fond of adding self.app and app.senpy, but Note: I'm not particularly fond of adding self.app and app.senpy, but
I can't think of a better way to do it. I can't think of a better way to do it.
""" """
def init_app(self, app, plugin_folder="plugins"):
app.senpy = self app.senpy = self
#app.config.setdefault('SQLITE3_DATABASE', ':memory:')
# Use the newstyle teardown_appcontext if it's available, # Use the newstyle teardown_appcontext if it's available,
# otherwise fall back to the request context # otherwise fall back to the request context
if hasattr(app, 'teardown_appcontext'): if hasattr(app, 'teardown_appcontext'):
...@@ -42,9 +47,18 @@ class Senpy(object): ...@@ -42,9 +47,18 @@ class Senpy(object):
app.teardown_request(self.teardown) app.teardown_request(self.teardown)
app.register_blueprint(nif_blueprint) app.register_blueprint(nif_blueprint)
def add_folder(self, folder):
if os.path.isdir(folder):
self._search_folders.add(folder)
self._outdated = True
return True
else:
return False
def analyse(self, **params): def analyse(self, **params):
algo = None algo = None
print("analysing with params: {}".format(params)) logger.debug("analysing with params: {}".format(params))
if "algorithm" in params: if "algorithm" in params:
algo = params["algorithm"] algo = params["algorithm"]
elif self.plugins: elif self.plugins:
...@@ -55,60 +69,68 @@ class Senpy(object): ...@@ -55,60 +69,68 @@ class Senpy(object):
resp.analysis.append(plug.jsonable()) resp.analysis.append(plug.jsonable())
return resp return resp
else: else:
return {"status": 500, "message": "No valid algorithm"} return {"status": 400, "message": "The algorithm '{}' is not valid".format(algo) }
@property @property
def default_plugin(self): def default_plugin(self):
if self.plugins: candidates = self.filter_plugins(enabled=True)
candidate = self.filter_plugins(enabled=True).keys()[0] if len(candidates)>1:
print("Default: {}".format(candidate)) candidate = candidates.keys()[0]
logger.debug("Default: {}".format(candidate))
return candidate return candidate
else: else:
return Exception("No algorithm") return None
def parameters(self, algo): def parameters(self, algo):
return getattr(self.plugins.get(algo or self.default_plugin), "params", {}) return getattr(self.plugins.get(algo or self.default_plugin), "params", {})
def enable_plugin(self, plugin): def enable_plugin(self, plugin):
self.plugins[plugin].disable() self.plugins[plugin].enable()
def disable_plugin(self, plugin): def disable_plugin(self, plugin):
self.plugins[plugin].disable() self.plugins[plugin].disable()
def reload_plugin(self, plugin): def reload_plugin(self, plugin):
print("Reloading {}".format(plugin)) logger.debug("Reloading {}".format(plugin))
plug = self.plugins[plugin] plug = self.plugins[plugin]
nplug = self._load_plugin(plug.module, plug.path) nplug = self._load_plugin(plug.module, plug.path)
del self.plugins[plugin] del self.plugins[plugin]
self.plugins[nplug.name] = nplug self.plugins[nplug.name] = nplug
def _load_plugin(self, plugin, search_folder, enabled=True): def _load_plugin(self, plugin, search_folder, enabled=True):
logger.debug("Loading plugins")
sys.path.append(search_folder) sys.path.append(search_folder)
(fp, pathname, desc) = imp.find_module(plugin) (fp, pathname, desc) = imp.find_module(plugin)
tmp = imp.load_module(plugin, fp, pathname, desc).plugin
sys.path.remove(search_folder)
tmp.path = search_folder
try: try:
repo_path = os.path.join(search_folder, plugin) tmp = imp.load_module(plugin, fp, pathname, desc).plugin
tmp.repo = Repo(repo_path) sys.path.remove(search_folder)
except InvalidGitRepositoryError: tmp.path = search_folder
tmp.repo = None try:
if not hasattr(tmp, "enabled"): repo_path = os.path.join(search_folder, plugin)
tmp.enabled = enabled tmp.repo = Repo(repo_path)
tmp.module = plugin except InvalidGitRepositoryError:
tmp.repo = None
if not hasattr(tmp, "enabled"):
tmp.enabled = enabled
tmp.module = plugin
except Exception as ex:
tmp = None
logger.debug("Exception importing {}: {}".format(plugin, ex))
return tmp return tmp
def _load_plugins(self): def _load_plugins(self):
plugins = {} plugins = {}
for search_folder in self.search_folders: for search_folder in self._search_folders:
for item in os.listdir(search_folder): for item in os.listdir(search_folder):
if os.path.isdir(os.path.join(search_folder, item)) \ if os.path.isdir(os.path.join(search_folder, item)) \
and os.path.exists( and os.path.exists(
os.path.join(search_folder, item, "__init__.py")): os.path.join(search_folder, item, "__init__.py")):
plugin = self._load_plugin(item, search_folder) plugin = self._load_plugin(item, search_folder)
plugins[plugin.name] = plugin if plugin:
plugins[plugin.name] = plugin
self._outdated = False
return plugins return plugins
def teardown(self, exception): def teardown(self, exception):
...@@ -126,30 +148,36 @@ class Senpy(object): ...@@ -126,30 +148,36 @@ class Senpy(object):
@property @property
def plugins(self): def plugins(self):
""" Return the plugins registered for a given application. """
ctx = stack.top ctx = stack.top
if ctx is not None: if ctx is not None:
if not hasattr(self, '_plugins'): if not hasattr(ctx, 'senpy_plugins') or self._outdated:
self._plugins = self._load_plugins() ctx.senpy_plugins = self._load_plugins()
return self._plugins return ctx.senpy_plugins
def filter_plugins(self, **kwargs): def filter_plugins(self, **kwargs):
""" Filter plugins by different criteria """
def matches(plug): def matches(plug):
res = all(getattr(plug, k, None)==v for (k,v) in kwargs.items()) res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
print("matching {} with {}: {}".format(plug.name, kwargs, res)) logger.debug("matching {} with {}: {}".format(plug.name,
kwargs,
res))
return res return res
if not kwargs: if not kwargs:
return self.plugins return self.plugins
else: else:
return {n:p for n,p in self.plugins.items() if matches(p)} return {n:p for n, p in self.plugins.items() if matches(p)}
def sentiment_plugins(self): def sentiment_plugins(self):
return (plugin for plugin in self.plugins if """ Return only the sentiment plugins """
isinstance(plugin, SentimentPlugin)) return {p:plugin for p, plugin in self.plugins.items() if
isinstance(plugin, SentimentPlugin)}
if __name__ == '__main__': if __name__ == '__main__':
from flask import Flask from flask import Flask
app = Flask(__name__) app = Flask(__name__)
sp = Senpy() sp = Senpy()
sp.init_app(app) sp.init_app(APP)
with app.app_context(): with APP.app_context():
sp._load_plugins() sp._load_plugins()
import logging
logger = logging.getLogger(__name__)
PARAMS = {"input": {"aliases": ["i", "input"], PARAMS = {"input": {"aliases": ["i", "input"],
"required": True, "required": True,
"help": "Input text" "help": "Input text"
...@@ -30,7 +34,7 @@ PARAMS = {"input": {"aliases": ["i", "input"], ...@@ -30,7 +34,7 @@ PARAMS = {"input": {"aliases": ["i", "input"],
class SenpyPlugin(object): class SenpyPlugin(object):
def __init__(self, name=None, version=None, extraparams=None, params=None): def __init__(self, name=None, version=None, extraparams=None, params=None):
print("Initing {}".format(name)) logger.debug("Initialising {}".format(name))
self.name = name self.name = name
self.version = version self.version = version
if params: if params:
......
'''
SENTIMENT140
=============
* http://www.sentiment140.com/api/bulkClassifyJson
* Method: POST
* Parameters: JSON Object (that is copied to the result)
* text
* query
* language
* topic
* Example response:
```json
{"data": [{"text": "I love Titanic.", "id":1234, "polarity": 4},
{"text": "I hate Titanic.", "id":4567, "polarity": 0}]}
```
'''
import requests
import json
ENDPOINT_URI = "http://www.sentiment140.com/api/bulkClassifyJson"
def analyse(texts):
parameters = {"data": []}
if isinstance(texts, list):
for text in texts:
parameters["data"].append({"text": text})
else:
parameters["data"].append({"text": texts})
res = requests.post(ENDPOINT_URI, json.dumps(parameters))
res.json()
return res.json()
def test():
print analyse("I love Titanic")
print analyse(["I love Titanic", "I hate Titanic"])
if __name__ == "__main__":
test()
import os
import logging
try:
import unittest.mock as mock
except ImportError:
import mock
from senpy.extensions import Senpy
from flask import Flask
from flask.ext.testing import TestCase
def check_dict(indic, template):
return all(item in indic.items() for item in template.items())
class Blueprints_Test(TestCase):
def create_app(self):
self.app = Flask("test_extensions")
self.senpy = Senpy()
self.senpy.init_app(self.app)
self.dir = os.path.join(os.path.dirname(__file__), "..")
self.senpy.add_folder(self.dir)
return self.app
def test_home(self):
""" Calling with no arguments should ask the user for more arguments """
resp = self.client.get("/")
self.assert200(resp)
logging.debug(resp.json)
assert resp.json["status"] == "failed"
atleast = {
"status": "failed",
"message": "Missing or invalid parameters",
}
assert check_dict(resp.json, atleast)
from senpy.plugins import SenpyPlugin
plugin = SenpyPlugin("dummy")
import os
import logging
try:
import unittest.mock as mock
except ImportError:
import mock
from senpy.extensions import Senpy
from flask import Flask
from flask.ext.testing import TestCase
class Extensions_Test(TestCase):
def create_app(self):
self.app = Flask("test_extensions")
self.senpy = Senpy()
self.senpy.init_app(self.app)
self.dir = os.path.join(os.path.dirname(__file__), "..")
self.senpy.add_folder(self.dir)
return self.app
def test_init(self):
""" Initialising the app with the extension. """
assert hasattr(self.app, "senpy")
tapp = Flask("temp app")
tsen = Senpy(tapp)
assert hasattr(tapp, "senpy")
def test_discovery(self):
""" Discovery of plugins in given folders. """
assert self.dir in self.senpy._search_folders
print self.senpy.plugins
assert "dummy" in self.senpy.plugins
def test_enabling(self):
""" Enabling a plugin """
self.senpy.enable_plugin("dummy")
assert self.senpy.plugins["dummy"].enabled
def test_disabling(self):
""" Disabling a plugin """
self.senpy.enable_plugin("dummy")
self.senpy.disable_plugin("dummy")
assert self.senpy.plugins["dummy"].enabled == False
def test_default(self):
""" Default plugin should be set """
assert self.senpy.default_plugin
assert self.senpy.default_plugin == "dummy"
def test_analyse(self):
""" Using a plugin """
with mock.patch.object(self.senpy.plugins["dummy"], "analyse") as mocked:
self.senpy.analyse(algorithm="dummy", input="tupni", output="tuptuo")
self.senpy.analyse(input="tupni", output="tuptuo")
mocked.assert_any_call(input="tupni", output="tuptuo", algorithm="dummy")
mocked.assert_any_call(input="tupni", output="tuptuo")
for plug in self.senpy.plugins:
self.senpy.disable_plugin(plug)
resp = self.senpy.analyse(input="tupni")
logging.debug("Response: {}".format(resp))
assert resp["status"] == 400
def test_filtering(self):
""" Filtering plugins """
assert len(self.senpy.filter_plugins(name="dummy"))>0
assert not len(self.senpy.filter_plugins(name="notdummy"))
assert self.senpy.filter_plugins(name="dummy", enabled=True)
self.senpy.disable_plugin("dummy")
assert not len(self.senpy.filter_plugins(name="dummy", enabled=True))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment