Commit d1006bbc authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

PEP8+Better JSON-LD support

* The API has also changed, there are new parameters to send the
context as part of the headers.
* Improved tests
* PEP8 compliance (despite the line about gevent)
parent d58137e8
...@@ -19,7 +19,7 @@ This is a helper for development. If you want to run Senpy use: ...@@ -19,7 +19,7 @@ This is a helper for development. If you want to run Senpy use:
python -m senpy python -m senpy
""" """
from gevent.monkey import patch_all; patch_all() from gevent.monkey import patch_all patch_all()
import gevent import gevent
import config import config
from flask import Flask from flask import Flask
......
...@@ -11,8 +11,7 @@ class Sentiment140Plugin(SentimentPlugin): ...@@ -11,8 +11,7 @@ class Sentiment140Plugin(SentimentPlugin):
p = params.get("prefix", None) p = params.get("prefix", None)
response = Response(prefix=p) response = Response(prefix=p)
#polarity_value = self.maxPolarityValue*int(res.json()["data"][0]["polarity"]) * 0.25 polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
polarity_value = max(-1, min(1, random.gauss(0.2,0.2)))
polarity = "marl:Neutral" polarity = "marl:Neutral"
if polarity_value > 0: if polarity_value > 0:
polarity = "marl:Positive" polarity = "marl:Positive"
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
"version": "0.1", "version": "0.1",
"extra_params": { "extra_params": {
"language": { "language": {
"@id": "lang_rand",
"aliases": ["language", "l"], "aliases": ["language", "l"],
"required": false, "required": false,
"options": ["es", "en", "auto"] "options": ["es", "en", "auto"]
......
...@@ -17,7 +17,8 @@ class Sentiment140Plugin(SentimentPlugin): ...@@ -17,7 +17,8 @@ class Sentiment140Plugin(SentimentPlugin):
p = params.get("prefix", None) p = params.get("prefix", None)
response = Response(prefix=p) response = Response(prefix=p)
polarity_value = self.maxPolarityValue*int(res.json()["data"][0]["polarity"]) * 0.25 polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
["polarity"]) * 0.25
polarity = "marl:Neutral" polarity = "marl:Neutral"
if polarity_value > 50: if polarity_value > 50:
polarity = "marl:Positive" polarity = "marl:Positive"
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
"version": "0.1", "version": "0.1",
"extra_params": { "extra_params": {
"language": { "language": {
"@id": "lang_sentiment140",
"aliases": ["language", "l"], "aliases": ["language", "l"],
"required": false, "required": false,
"options": ["es", "en", "auto"] "options": ["es", "en", "auto"]
......
...@@ -19,15 +19,18 @@ Senpy is a modular sentiment analysis server. This script runs an instance of ...@@ -19,15 +19,18 @@ Senpy is a modular sentiment analysis server. This script runs an instance of
the server. the server.
""" """
from gevent.monkey import patch_all; patch_all(thread=False)
import gevent
from flask import Flask from flask import Flask
from senpy.extensions import Senpy from senpy.extensions import Senpy
from gevent.wsgi import WSGIServer
from gevent.monkey import patch_all
import gevent
import logging import logging
import os import os
from gevent.wsgi import WSGIServer
import argparse import argparse
patch_all(thread=False)
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Run a Senpy server') parser = argparse.ArgumentParser(description='Run a Senpy server')
parser.add_argument('--level', parser.add_argument('--level',
...@@ -43,20 +46,20 @@ if __name__ == '__main__': ...@@ -43,20 +46,20 @@ if __name__ == '__main__':
help='Run the application in debug mode') help='Run the application in debug mode')
parser.add_argument('--host', parser.add_argument('--host',
type=str, type=str,
default = "127.0.0.1", default="127.0.0.1",
help='Use 0.0.0.0 to accept requests from any host.') help='Use 0.0.0.0 to accept requests from any host.')
parser.add_argument('--port', parser.add_argument('--port',
'-p', '-p',
type=int, type=int,
default = 5000, default=5000,
help='Port to listen on.') help='Port to listen on.')
parser.add_argument('--plugins-folder', parser.add_argument('--plugins-folder',
'-f', '-f',
type=str, type=str,
default = "plugins", default="plugins",
help='Where to look for plugins.') help='Where to look for plugins.')
args = parser.parse_args() args = parser.parse_args()
logging.basicConfig(level=getattr(logging,args.level)) logging.basicConfig(level=getattr(logging, args.level))
app = Flask(__name__) app = Flask(__name__)
app.debug = args.debug app.debug = args.debug
sp = Senpy(app, args.plugins_folder) sp = Senpy(app, args.plugins_folder)
......
...@@ -17,19 +17,35 @@ ...@@ -17,19 +17,35 @@
""" """
Blueprints for Senpy Blueprints for Senpy
""" """
from flask import Blueprint, request, current_app
from .models import Error, Response
import json import json
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from flask import Blueprint, request, jsonify, current_app
nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__) nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
BASIC_PARAMS = { BASIC_PARAMS = {
"algorithm": {"aliases": ["algorithm", "a", "algo"], "algorithm": {
"required": False, "aliases": ["algorithm", "a", "algo"],
}, "required": False,
},
"inHeaders": {
"aliases": ["inHeaders", "headers"],
"required": True,
"default": "0"
}
}
LIST_PARAMS = {
"params": {
"aliases": ["params", "with_params"],
"required": False,
"default": "0"
},
} }
...@@ -44,34 +60,40 @@ def get_params(req, params=BASIC_PARAMS): ...@@ -44,34 +60,40 @@ def get_params(req, params=BASIC_PARAMS):
outdict = {} outdict = {}
wrong_params = {} wrong_params = {}
for param, options in params.iteritems(): for param, options in params.iteritems():
for alias in options["aliases"]: if param[0] != "@": # Exclude json-ld properties
if alias in indict: logger.debug("Param: %s - Options: %s", param, options)
outdict[param] = indict[alias] for alias in options["aliases"]:
if param not in outdict: if alias in indict:
if options.get("required", False) and "default" not in options: outdict[param] = indict[alias]
wrong_params[param] = params[param] if param not in outdict:
if options.get("required", False) and "default" not in options:
wrong_params[param] = params[param]
else:
if "default" in options:
outdict[param] = options["default"]
else: else:
if "default" in options: if "options" in params[param] and \
outdict[param] = options["default"] outdict[param] not in params[param]["options"]:
else: wrong_params[param] = params[param]
if "options" in params[param] and outdict[param] not in params[param]["options"]:
wrong_params[param] = params[param]
if wrong_params: if wrong_params:
message = {"status": "failed", message = Error({"status": 404,
"message": "Missing or invalid parameters", "message": "Missing or invalid parameters",
"parameters": outdict, "parameters": outdict,
"errors": {param: error for param, error in wrong_params.iteritems()} "errors": {param: error for param, error in
} wrong_params.iteritems()}
})
raise ValueError(message) raise ValueError(message)
return outdict return outdict
def basic_analysis(params): def basic_analysis(params):
response = {"@context": ["http://demos.gsi.dit.upm.es/eurosentiment/static/context.jsonld", response = {"@context":
{ [("http://demos.gsi.dit.upm.es/"
"@base": "{}#".format(request.url.encode('utf-8')) "eurosentiment/static/context.jsonld"),
} {
], "@base": "{}#".format(request.url.encode('utf-8'))
}
],
"analysis": [{"@type": "marl:SentimentAnalysis"}], "analysis": [{"@type": "marl:SentimentAnalysis"}],
"entries": [] "entries": []
} }
...@@ -91,19 +113,25 @@ def home(): ...@@ -91,19 +113,25 @@ def home():
params = get_params(request) params = get_params(request)
algo = params.get("algorithm", None) algo = params.get("algorithm", None)
specific_params = current_app.senpy.parameters(algo) specific_params = current_app.senpy.parameters(algo)
logger.debug(
"Specific params: %s", json.dumps(specific_params, indent=4))
params.update(get_params(request, specific_params)) params.update(get_params(request, specific_params))
response = current_app.senpy.analyse(**params) response = current_app.senpy.analyse(**params)
return jsonify(response) in_headers = params["inHeaders"] != "0"
return response.flask(in_headers=in_headers)
except ValueError as ex: except ValueError as ex:
return jsonify(ex.message) return ex.message.flask()
except Exception as ex:
return jsonify(status="400", message=ex.message)
@nif_blueprint.route("/default") @nif_blueprint.route("/default")
def default(): def default():
return current_app.senpy.default_plugin # return current_app.senpy.default_plugin
#return plugins(action="list", plugin=current_app.senpy.default_algorithm) plug = current_app.senpy.default_plugin
if plug:
return plugins(action="list", plugin=plug.name)
else:
error = Error(status=404, message="No plugins found")
return error.flask()
@nif_blueprint.route('/plugins/', methods=['POST', 'GET']) @nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
...@@ -118,12 +146,15 @@ def plugins(plugin=None, action="list"): ...@@ -118,12 +146,15 @@ def plugins(plugin=None, action="list"):
if plugin and not plugs: if plugin and not plugs:
return "Plugin not found", 400 return "Plugin not found", 400
if action == "list": if action == "list":
with_params = request.args.get("params", "") == "1" with_params = get_params(request, LIST_PARAMS)["params"] == "1"
in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0"
if plugin: if plugin:
dic = plugs[plugin].jsonable(with_params) dic = plugs[plugin]
else: else:
dic = {plug: plugs[plug].jsonable(with_params) for plug in plugs} dic = Response(
return jsonify(dic) {plug: plugs[plug].jsonld(with_params) for plug in plugs},
frame={})
return dic.flask(in_headers=in_headers)
method = "{}_plugin".format(action) method = "{}_plugin".format(action)
if(hasattr(sp, method)): if(hasattr(sp, method)):
getattr(sp, method)(plugin) getattr(sp, method)(plugin)
......
...@@ -36,5 +36,7 @@ ...@@ -36,5 +36,7 @@
}, },
"text": { "@id": "nif:isString" }, "text": { "@id": "nif:isString" },
"wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#", "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
"xsd": "http://www.w3.org/2001/XMLSchema#" "xsd": "http://www.w3.org/2001/XMLSchema#",
"senpy": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#",
"@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#"
} }
""" """
""" """
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
from .blueprints import nif_blueprint
from git import Repo, InvalidGitRepositoryError
from functools import partial
import os import os
import fnmatch import fnmatch
import inspect import inspect
...@@ -11,15 +18,9 @@ import json ...@@ -11,15 +18,9 @@ import json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
from .blueprints import nif_blueprint
from git import Repo, InvalidGitRepositoryError
from functools import partial
class Senpy(object): class Senpy(object):
""" Default Senpy extension for Flask """ """ Default Senpy extension for Flask """
def __init__(self, app=None, plugin_folder="plugins"): def __init__(self, app=None, plugin_folder="plugins"):
...@@ -66,32 +67,45 @@ class Senpy(object): ...@@ -66,32 +67,45 @@ class Senpy(object):
if "algorithm" in params: if "algorithm" in params:
algo = params["algorithm"] algo = params["algorithm"]
elif self.plugins: elif self.plugins:
algo = self.default_plugin algo = self.default_plugin and self.default_plugin.name
if not algo:
return Error(status=404,
message=("No plugins found."
" Please install one.").format(algo))
if algo in self.plugins: if algo in self.plugins:
if self.plugins[algo].is_activated: if self.plugins[algo].is_activated:
plug = self.plugins[algo] plug = self.plugins[algo]
resp = plug.analyse(**params) resp = plug.analyse(**params)
resp.analysis.append(plug) resp.analysis.append(plug)
logger.debug("Returning analysis result: {}".format(resp))
return resp return resp
else: else:
logger.debug("Plugin not activated: {}".format(algo)) logger.debug("Plugin not activated: {}".format(algo))
return Error(status=400, message="The algorithm '{}' is not activated yet".format(algo)) return Error(status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
else: else:
logger.debug("The algorithm '{}' is not valid\nValid algorithms: {}".format(algo, self.plugins.keys())) logger.debug(("The algorithm '{}' is not valid\n"
return Error(status=400, message="The algorithm '{}' is not valid".format(algo)) "Valid algorithms: {}").format(algo,
self.plugins.keys()))
return Error(status=400,
message="The algorithm '{}' is not valid"
.format(algo))
@property @property
def default_plugin(self): def default_plugin(self):
candidates = self.filter_plugins(is_activated=True) candidates = self.filter_plugins(is_activated=True)
if len(candidates) > 0: if len(candidates) > 0:
candidate = candidates.keys()[0] candidate = candidates.values()[0]
logger.debug("Default: {}".format(candidate)) logger.debug("Default: {}".format(candidate))
return candidate return candidate
else: else:
return None return None
def parameters(self, algo): def parameters(self, algo):
return getattr(self.plugins.get(algo or self.default_plugin), "params", {}) return getattr(self.plugins.get(algo) or self.default_plugin,
"params",
{})
def activate_all(self, sync=False): def activate_all(self, sync=False):
ps = [] ps = []
...@@ -137,20 +151,22 @@ class Senpy(object): ...@@ -137,20 +151,22 @@ class Senpy(object):
def _load_plugin(root, filename): def _load_plugin(root, filename):
logger.debug("Loading plugin: {}".format(filename)) logger.debug("Loading plugin: {}".format(filename))
fpath = os.path.join(root, filename) fpath = os.path.join(root, filename)
with open(fpath,'r') as f: with open(fpath, 'r') as f:
info = json.load(f) info = json.load(f)
logger.debug("Info: {}".format(info)) logger.debug("Info: {}".format(info))
sys.path.append(root) sys.path.append(root)
module = info["module"] module = info["module"]
name = info["name"] name = info["name"]
(fp, pathname, desc) = imp.find_module(module, [root,]) (fp, pathname, desc) = imp.find_module(module, [root, ])
try: try:
tmp = imp.load_module(module, fp, pathname, desc) tmp = imp.load_module(module, fp, pathname, desc)
sys.path.remove(root) sys.path.remove(root)
candidate = None candidate = None
for _, obj in inspect.getmembers(tmp): for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp: if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug("Found plugin class: {}@{}".format(obj, inspect.getmodule(obj))) logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj))
)
candidate = obj candidate = obj
break break
if not candidate: if not candidate:
......
...@@ -2,7 +2,8 @@ import json ...@@ -2,7 +2,8 @@ import json
import os import os
from collections import defaultdict from collections import defaultdict
from pyld import jsonld from pyld import jsonld
import logging
from flask import Response as FlaskResponse
class Leaf(dict): class Leaf(dict):
...@@ -11,18 +12,17 @@ class Leaf(dict): ...@@ -11,18 +12,17 @@ class Leaf(dict):
_context = {} _context = {}
def __init__(self, def __init__(self,
id=None, *args,
context=None, **kwargs):
vocab=None,
prefix=None, id = kwargs.pop("id", None)
frame=None): context = kwargs.pop("context", self._context)
super(Leaf, self).__init__() vocab = kwargs.pop("vocab", None)
prefix = kwargs.pop("prefix", None)
frame = kwargs.pop("frame", None)
super(Leaf, self).__init__(*args, **kwargs)
if context is not None: if context is not None:
self.context = context self.context = context
elif self._context:
self.context = self._context
else:
self.context = {}
if frame is not None: if frame is not None:
self._frame = frame self._frame = frame
self._prefix = prefix self._prefix = prefix
...@@ -60,10 +60,11 @@ class Leaf(dict): ...@@ -60,10 +60,11 @@ class Leaf(dict):
def get_id(self, id): def get_id(self, id):
""" """
This is not the most elegant solution to change the @id attribute, but it Get id, dealing with prefixes
is the quickest way to have it included in the dictionary without extra
boilerplate.
""" """
# This is not the most elegant solution to change the @id attribute,
# but it is the quickest way to have it included in the dictionary
# without extra boilerplate.
if id and self._prefix and ":" not in id: if id and self._prefix and ":" not in id:
return "{}{}".format(self._prefix, id) return "{}{}".format(self._prefix, id)
else: else:
...@@ -97,7 +98,7 @@ class Leaf(dict): ...@@ -97,7 +98,7 @@ class Leaf(dict):
return context return context
def compact(self): def compact(self):
return jsonld.compact(self, self.context) return jsonld.compact(self, self.get_context(self.context))
def frame(self, frame=None, options=None): def frame(self, frame=None, options=None):
if frame is None: if frame is None:
...@@ -106,84 +107,100 @@ class Leaf(dict): ...@@ -106,84 +107,100 @@ class Leaf(dict):
options = {} options = {}
return jsonld.frame(self, frame, options) return jsonld.frame(self, frame, options)
def jsonable(self, parameters=False, frame=None, options=None, context=None): def jsonld(self, frame=None, options=None,
context=None, removeContext=None):
if removeContext is None:
removeContext = Response._context # Loop?
if frame is None: if frame is None:
frame = self._frame frame = self._frame
if options is None:
options = {}
if context is None: if context is None:
context = self._context context = self.context
return jsonld.compact(jsonld.frame(self, frame, options), context) else:
#if parameters: context = self.get_context(context)
#resp["parameters"] = self.params # For some reason, this causes errors with pyld
#elif self.extra_params: # if options is None:
#resp["extra_parameters"] = self.extra_params # options = {"expandContext": context.copy() }
#return resp js = self
if frame:
logging.debug("Framing: %s", json.dumps(self, indent=4))
def to_JSON(self): logging.debug("Framing with %s", json.dumps(frame, indent=4))
return json.dumps(self,