Commit d1006bbc authored by J. Fernando Sánchez's avatar J. Fernando Sánchez

PEP8+Better JSON-LD support

* The API has also changed, there are new parameters to send the
context as part of the headers.
* Improved tests
* PEP8 compliance (despite the line about gevent)
parent d58137e8
......@@ -19,7 +19,7 @@ This is a helper for development. If you want to run Senpy use:
python -m senpy
"""
from gevent.monkey import patch_all; patch_all()
from gevent.monkey import patch_all patch_all()
import gevent
import config
from flask import Flask
......
......@@ -11,8 +11,7 @@ class Sentiment140Plugin(SentimentPlugin):
p = params.get("prefix", None)
response = Response(prefix=p)
#polarity_value = self.maxPolarityValue*int(res.json()["data"][0]["polarity"]) * 0.25
polarity_value = max(-1, min(1, random.gauss(0.2,0.2)))
polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
polarity = "marl:Neutral"
if polarity_value > 0:
polarity = "marl:Positive"
......
......@@ -6,6 +6,7 @@
"version": "0.1",
"extra_params": {
"language": {
"@id": "lang_rand",
"aliases": ["language", "l"],
"required": false,
"options": ["es", "en", "auto"]
......
......@@ -17,7 +17,8 @@ class Sentiment140Plugin(SentimentPlugin):
p = params.get("prefix", None)
response = Response(prefix=p)
polarity_value = self.maxPolarityValue*int(res.json()["data"][0]["polarity"]) * 0.25
polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
["polarity"]) * 0.25
polarity = "marl:Neutral"
if polarity_value > 50:
polarity = "marl:Positive"
......
......@@ -6,6 +6,7 @@
"version": "0.1",
"extra_params": {
"language": {
"@id": "lang_sentiment140",
"aliases": ["language", "l"],
"required": false,
"options": ["es", "en", "auto"]
......
......@@ -19,15 +19,18 @@ Senpy is a modular sentiment analysis server. This script runs an instance of
the server.
"""
from gevent.monkey import patch_all; patch_all(thread=False)
import gevent
from flask import Flask
from senpy.extensions import Senpy
from gevent.wsgi import WSGIServer
from gevent.monkey import patch_all
import gevent
import logging
import os
from gevent.wsgi import WSGIServer
import argparse
patch_all(thread=False)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Run a Senpy server')
parser.add_argument('--level',
......@@ -43,20 +46,20 @@ if __name__ == '__main__':
help='Run the application in debug mode')
parser.add_argument('--host',
type=str,
default = "127.0.0.1",
default="127.0.0.1",
help='Use 0.0.0.0 to accept requests from any host.')
parser.add_argument('--port',
'-p',
type=int,
default = 5000,
default=5000,
help='Port to listen on.')
parser.add_argument('--plugins-folder',
'-f',
type=str,
default = "plugins",
default="plugins",
help='Where to look for plugins.')
args = parser.parse_args()
logging.basicConfig(level=getattr(logging,args.level))
logging.basicConfig(level=getattr(logging, args.level))
app = Flask(__name__)
app.debug = args.debug
sp = Senpy(app, args.plugins_folder)
......
......@@ -17,19 +17,35 @@
"""
Blueprints for Senpy
"""
from flask import Blueprint, request, current_app
from .models import Error, Response
import json
import logging
logger = logging.getLogger(__name__)
from flask import Blueprint, request, jsonify, current_app
nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
BASIC_PARAMS = {
"algorithm": {"aliases": ["algorithm", "a", "algo"],
"required": False,
},
"algorithm": {
"aliases": ["algorithm", "a", "algo"],
"required": False,
},
"inHeaders": {
"aliases": ["inHeaders", "headers"],
"required": True,
"default": "0"
}
}
LIST_PARAMS = {
"params": {
"aliases": ["params", "with_params"],
"required": False,
"default": "0"
},
}
......@@ -44,34 +60,40 @@ def get_params(req, params=BASIC_PARAMS):
outdict = {}
wrong_params = {}
for param, options in params.iteritems():
for alias in options["aliases"]:
if alias in indict:
outdict[param] = indict[alias]
if param not in outdict:
if options.get("required", False) and "default" not in options:
wrong_params[param] = params[param]
if param[0] != "@": # Exclude json-ld properties
logger.debug("Param: %s - Options: %s", param, options)
for alias in options["aliases"]:
if alias in indict:
outdict[param] = indict[alias]
if param not in outdict:
if options.get("required", False) and "default" not in options:
wrong_params[param] = params[param]
else:
if "default" in options:
outdict[param] = options["default"]
else:
if "default" in options:
outdict[param] = options["default"]
else:
if "options" in params[param] and outdict[param] not in params[param]["options"]:
wrong_params[param] = params[param]
if "options" in params[param] and \
outdict[param] not in params[param]["options"]:
wrong_params[param] = params[param]
if wrong_params:
message = {"status": "failed",
"message": "Missing or invalid parameters",
"parameters": outdict,
"errors": {param: error for param, error in wrong_params.iteritems()}
}
message = Error({"status": 404,
"message": "Missing or invalid parameters",
"parameters": outdict,
"errors": {param: error for param, error in
wrong_params.iteritems()}
})
raise ValueError(message)
return outdict
def basic_analysis(params):
response = {"@context": ["http://demos.gsi.dit.upm.es/eurosentiment/static/context.jsonld",
{
"@base": "{}#".format(request.url.encode('utf-8'))
}
],
response = {"@context":
[("http://demos.gsi.dit.upm.es/"
"eurosentiment/static/context.jsonld"),
{
"@base": "{}#".format(request.url.encode('utf-8'))
}
],
"analysis": [{"@type": "marl:SentimentAnalysis"}],
"entries": []
}
......@@ -91,19 +113,25 @@ def home():
params = get_params(request)
algo = params.get("algorithm", None)
specific_params = current_app.senpy.parameters(algo)
logger.debug(
"Specific params: %s", json.dumps(specific_params, indent=4))
params.update(get_params(request, specific_params))
response = current_app.senpy.analyse(**params)
return jsonify(response)
in_headers = params["inHeaders"] != "0"
return response.flask(in_headers=in_headers)
except ValueError as ex:
return jsonify(ex.message)
except Exception as ex:
return jsonify(status="400", message=ex.message)
return ex.message.flask()
@nif_blueprint.route("/default")
def default():
return current_app.senpy.default_plugin
#return plugins(action="list", plugin=current_app.senpy.default_algorithm)
# return current_app.senpy.default_plugin
plug = current_app.senpy.default_plugin
if plug:
return plugins(action="list", plugin=plug.name)
else:
error = Error(status=404, message="No plugins found")
return error.flask()
@nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
......@@ -118,12 +146,15 @@ def plugins(plugin=None, action="list"):
if plugin and not plugs:
return "Plugin not found", 400
if action == "list":
with_params = request.args.get("params", "") == "1"
with_params = get_params(request, LIST_PARAMS)["params"] == "1"
in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0"
if plugin:
dic = plugs[plugin].jsonable(with_params)
dic = plugs[plugin]
else:
dic = {plug: plugs[plug].jsonable(with_params) for plug in plugs}
return jsonify(dic)
dic = Response(
{plug: plugs[plug].jsonld(with_params) for plug in plugs},
frame={})
return dic.flask(in_headers=in_headers)
method = "{}_plugin".format(action)
if(hasattr(sp, method)):
getattr(sp, method)(plugin)
......
......@@ -36,5 +36,7 @@
},
"text": { "@id": "nif:isString" },
"wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
"xsd": "http://www.w3.org/2001/XMLSchema#"
"xsd": "http://www.w3.org/2001/XMLSchema#",
"senpy": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#",
"@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#"
}
"""
"""
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
from .blueprints import nif_blueprint
from git import Repo, InvalidGitRepositoryError
from functools import partial
import os
import fnmatch
import inspect
......@@ -11,15 +18,9 @@ import json
logger = logging.getLogger(__name__)
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
from .blueprints import nif_blueprint
from git import Repo, InvalidGitRepositoryError
from functools import partial
class Senpy(object):
""" Default Senpy extension for Flask """
def __init__(self, app=None, plugin_folder="plugins"):
......@@ -66,32 +67,45 @@ class Senpy(object):
if "algorithm" in params:
algo = params["algorithm"]
elif self.plugins:
algo = self.default_plugin
algo = self.default_plugin and self.default_plugin.name
if not algo:
return Error(status=404,
message=("No plugins found."
" Please install one.").format(algo))
if algo in self.plugins:
if self.plugins[algo].is_activated:
plug = self.plugins[algo]
resp = plug.analyse(**params)
resp.analysis.append(plug)
logger.debug("Returning analysis result: {}".format(resp))
return resp
else:
logger.debug("Plugin not activated: {}".format(algo))
return Error(status=400, message="The algorithm '{}' is not activated yet".format(algo))
return Error(status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
else:
logger.debug("The algorithm '{}' is not valid\nValid algorithms: {}".format(algo, self.plugins.keys()))
return Error(status=400, message="The algorithm '{}' is not valid".format(algo))
logger.debug(("The algorithm '{}' is not valid\n"
"Valid algorithms: {}").format(algo,
self.plugins.keys()))
return Error(status=400,
message="The algorithm '{}' is not valid"
.format(algo))
@property
def default_plugin(self):
candidates = self.filter_plugins(is_activated=True)
if len(candidates) > 0:
candidate = candidates.keys()[0]
candidate = candidates.values()[0]
logger.debug("Default: {}".format(candidate))
return candidate
else:
return None
def parameters(self, algo):
return getattr(self.plugins.get(algo or self.default_plugin), "params", {})
return getattr(self.plugins.get(algo) or self.default_plugin,
"params",
{})
def activate_all(self, sync=False):
ps = []
......@@ -137,20 +151,22 @@ class Senpy(object):
def _load_plugin(root, filename):
logger.debug("Loading plugin: {}".format(filename))
fpath = os.path.join(root, filename)
with open(fpath,'r') as f:
with open(fpath, 'r') as f:
info = json.load(f)
logger.debug("Info: {}".format(info))
sys.path.append(root)
module = info["module"]
name = info["name"]
(fp, pathname, desc) = imp.find_module(module, [root,])
(fp, pathname, desc) = imp.find_module(module, [root, ])
try:
tmp = imp.load_module(module, fp, pathname, desc)
sys.path.remove(root)
candidate = None
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug("Found plugin class: {}@{}".format(obj, inspect.getmodule(obj)))
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj))
)
candidate = obj
break
if not candidate:
......
......@@ -2,7 +2,8 @@ import json
import os
from collections import defaultdict
from pyld import jsonld
import logging
from flask import Response as FlaskResponse
class Leaf(dict):
......@@ -11,18 +12,17 @@ class Leaf(dict):
_context = {}
def __init__(self,
id=None,
context=None,
vocab=None,
prefix=None,
frame=None):
super(Leaf, self).__init__()
*args,
**kwargs):
id = kwargs.pop("id", None)
context = kwargs.pop("context", self._context)
vocab = kwargs.pop("vocab", None)
prefix = kwargs.pop("prefix", None)
frame = kwargs.pop("frame", None)
super(Leaf, self).__init__(*args, **kwargs)
if context is not None:
self.context = context
elif self._context:
self.context = self._context
else:
self.context = {}
if frame is not None:
self._frame = frame
self._prefix = prefix
......@@ -60,10 +60,11 @@ class Leaf(dict):
def get_id(self, id):
"""
This is not the most elegant solution to change the @id attribute, but it
is the quickest way to have it included in the dictionary without extra
boilerplate.
Get id, dealing with prefixes
"""
# This is not the most elegant solution to change the @id attribute,
# but it is the quickest way to have it included in the dictionary
# without extra boilerplate.
if id and self._prefix and ":" not in id:
return "{}{}".format(self._prefix, id)
else:
......@@ -97,7 +98,7 @@ class Leaf(dict):
return context
def compact(self):
return jsonld.compact(self, self.context)
return jsonld.compact(self, self.get_context(self.context))
def frame(self, frame=None, options=None):
if frame is None:
......@@ -106,84 +107,100 @@ class Leaf(dict):
options = {}
return jsonld.frame(self, frame, options)
def jsonable(self, parameters=False, frame=None, options=None, context=None):
def jsonld(self, frame=None, options=None,
context=None, removeContext=None):
if removeContext is None:
removeContext = Response._context # Loop?
if frame is None:
frame = self._frame
if options is None:
options = {}
if context is None:
context = self._context
return jsonld.compact(jsonld.frame(self, frame, options), context)
#if parameters:
#resp["parameters"] = self.params
#elif self.extra_params:
#resp["extra_parameters"] = self.extra_params
#return resp
def to_JSON(self):
return json.dumps(self,
context = self.context
else:
context = self.get_context(context)
# For some reason, this causes errors with pyld
# if options is None:
# options = {"expandContext": context.copy() }
js = self
if frame:
logging.debug("Framing: %s", json.dumps(self, indent=4))
logging.debug("Framing with %s", json.dumps(frame, indent=4))
js = jsonld.frame(js, frame, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
logging.debug("Compacting with %s", json.dumps(context, indent=4))
js = jsonld.compact(js, context, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
if removeContext == context:
del js["@context"]
return js
def to_JSON(self, removeContext=None):
return json.dumps(self.jsonld(removeContext=removeContext),
default=lambda o: o.__dict__,
sort_keys=True, indent=4)
def flask(self,
in_headers=False,
url="http://demos.gsi.dit.upm.es/senpy/senpy.jsonld"):
"""
Return the values and error to be used in flask
"""
js = self.jsonld()
headers = None
if in_headers:
ctx = js["@context"]
headers = {
"Link": ('<%s>;'
'rel="http://www.w3.org/ns/json-ld#context";'
' type="application/ld+json"' % url)
}
del js["@context"]
return FlaskResponse(json.dumps(js),
status=self.get("status", 200),
headers=headers,
mimetype="application/json")
class Response(Leaf):
_frame = { "@context": {
"analysis": {
"@container": "@set",
"@id": "prov:wasInformedBy"
},
"date": {
"@id": "dc:date",
"@type": "xsd:dateTime"
},
"dc": "http://purl.org/dc/terms/",
"dc:subject": {
"@type": "@id"
},
"emotions": {
"@container": "@set",
"@id": "onyx:hasEmotionSet"
},
"entries": {
"@container": "@set",
"@id": "prov:generated"
},
"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
"nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
"onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#",
"opinions": {
"@container": "@set",
"@id": "marl:hasOpinion"
},
"prov": "http://www.w3.org/ns/prov#",
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
"strings": {
"@container": "@set",
"@reverse": "nif:hasContext"
},
"wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
"xsd": "http://www.w3.org/2001/XMLSchema#"
},
"analysis": {},
"entries": {}
_context = Leaf.get_context("{}/context.jsonld".format(
os.path.dirname(os.path.realpath(__file__))))
_frame = {
"@context": _context,
"analysis": {
"@explicit": True,
"maxPolarityValue": {},
"minPolarityValue": {},
"name": {},
"version": {},
},
"entries": {}
}
def __init__(self, context=None, *args, **kwargs):
def __init__(self, *args, **kwargs):
context = kwargs.pop("context", None)
frame = kwargs.pop("frame", None)
if context is None:
context = "{}/context.jsonld".format(os.path.dirname(
os.path.realpath(__file__)))
super(Response, self).__init__(*args, context=context, **kwargs)
self.analysis = []
self.entries = []
context = self._context
self.context = context
super(Response, self).__init__(
*args, context=context, frame=frame, **kwargs)
if self._frame is not None and "entries" in self._frame:
self.analysis = []
self.entries = []
def jsonld(self, frame=None, options=None, context=None, removeContext={}):
return super(Response, self).jsonld(frame,
options,
context,
removeContext)
class Entry(Leaf):
_context = {
"@vocab": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#"
"@vocab": ("http://persistence.uni-leipzig.org/"
"nlp2rdf/ontologies/nif-core#")
}
def __init__(self, text=None, emotion_sets=None, opinions=None, **kwargs):
super(Entry, self).__init__(**kwargs)
if text:
......@@ -191,10 +208,12 @@ class Entry(Leaf):
self.emotionSets = emotion_sets if emotion_sets else []
self.opinions = opinions if opinions else []
class Opinion(Leaf):
_context = {
"@vocab": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"
}
def __init__(self, polarityValue=None, hasPolarity=None, *args, **kwargs):
super(Opinion, self).__init__(*args,
**kwargs)
......@@ -206,6 +225,7 @@ class Opinion(Leaf):
class EmotionSet(Leaf):
_context = {}
def __init__(self, emotions=None, *args, **kwargs):
if not emotions:
emotions = []
......@@ -214,10 +234,16 @@ class EmotionSet(Leaf):
**kwargs)
self.emotions = emotions or []
class Emotion(Leaf):
_context = {}
class Error(Leaf):
class Error(Response):
# A better pattern would be this:
# http://flask.pocoo.org/docs/0.10/patterns/apierrors/
_frame = {}
_context = {}
def __init__(self, *args, **kwargs):
super(Error, self).__init__(*args)
self.update(kwargs)
super(Error, self).__init__(*args, **kwargs)
import logging
import ConfigParser
from .models import Leaf
from .models import Response, Leaf
logger = logging.getLogger(__name__)
PARAMS = {"input": {"aliases": ["i", "input"],
"required": True,
"help": "Input text"
},
"informat": {"aliases": ["f", "informat"],
"required": False,