Commit 14c9f618 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez

Python 3 compatible

There are also some slight changes to the JSON schemas and the use of
JSON-LD.
parent a79df7a3
......@@ -3,3 +3,4 @@
*egg-info
dist
README.html
__pycache__
\ No newline at end of file
......@@ -2,6 +2,10 @@ Flask>=0.10.1
gunicorn>=19.0.0
requests>=2.4.1
GitPython>=0.3.2.RC1
gevent>=1.0.1
gevent>=1.1rc4
PyLD>=0.6.5
Flask-Testing>=0.4.2
six
future
jsonschema
jsonref
......@@ -17,8 +17,8 @@
"""
Blueprints for Senpy
"""
from flask import Blueprint, request, current_app, Flask, redirect, url_for, render_template
from .models import Error, Response, Leaf
from flask import Blueprint, request, current_app, render_template
from .models import Error, Response
from future.utils import iteritems
import json
......@@ -27,6 +27,7 @@ import logging
logger = logging.getLogger(__name__)
nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__)
BASIC_PARAMS = {
"algorithm": {
......@@ -40,15 +41,6 @@ BASIC_PARAMS = {
}
}
LIST_PARAMS = {
"params": {
"aliases": ["params", "with_params"],
"required": False,
"default": "0"
},
}
def get_params(req, params=BASIC_PARAMS):
if req.method == 'POST':
indict = req.form
......@@ -76,12 +68,11 @@ def get_params(req, params=BASIC_PARAMS):
outdict[param] not in params[param]["options"]:
wrong_params[param] = params[param]
if wrong_params:
message = Error({"status": 404,
"message": "Missing or invalid parameters",
"parameters": outdict,
"errors": {param: error for param, error in
iteritems(wrong_params)}
})
message = Error(status=404,
message="Missing or invalid parameters",
parameters=outdict,
errors={param: error for param, error in
iteritems(wrong_params)})
raise Error(message=message)
return outdict
......@@ -107,12 +98,12 @@ def basic_analysis(params):
return response
@nif_blueprint.route('/')
@demo_blueprint.route('/')
def index():
return render_template("index.html")
@nif_blueprint.route('/api', methods=['POST', 'GET'])
@nif_blueprint.route('/', methods=['POST', 'GET'])
def api():
try:
params = get_params(request)
......@@ -128,7 +119,7 @@ def api():
return ex.message.flask()
@nif_blueprint.route("/api/default")
@nif_blueprint.route("/default")
def default():
# return current_app.senpy.default_plugin
plug = current_app.senpy.default_plugin
......@@ -139,9 +130,9 @@ def default():
return error.flask()
@nif_blueprint.route('/api/plugins/', methods=['POST', 'GET'])
@nif_blueprint.route('/api/plugins/<plugin>', methods=['POST', 'GET'])
@nif_blueprint.route('/api/plugins/<plugin>/<action>', methods=['POST', 'GET'])
@nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
@nif_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
def plugins(plugin=None, action="list"):
filt = {}
sp = current_app.senpy
......@@ -151,21 +142,19 @@ def plugins(plugin=None, action="list"):
if plugin and not plugs:
return "Plugin not found", 400
if action == "list":
with_params = get_params(request, LIST_PARAMS)["params"] == "1"
in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0"
if plugin:
dic = plugs[plugin]
else:
dic = Response(
{plug: plugs[plug].jsonld(with_params) for plug in plugs},
frame={})
{plug: plugs[plug].serializable() for plug in plugs})
return dic.flask(in_headers=in_headers)
method = "{}_plugin".format(action)
if(hasattr(sp, method)):
getattr(sp, method)(plugin)
return Leaf(message="Ok").flask()
return Response(message="Ok").flask()
else:
return Error("action '{}' not allowed".format(action)).flask()
return Error(message="action '{}' not allowed".format(action)).flask()
if __name__ == '__main__':
......
{
"dc": "http://purl.org/dc/terms/",
"dc:subject": {
"@type": "@id"
},
"xsd": "http://www.w3.org/2001/XMLSchema#",
"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
"nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
"onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#",
"emotions": {
"@container": "@set",
"@id": "onyx:hasEmotionSet"
},
"opinions": {
"@container": "@set",
"@id": "marl:hasOpinion"
},
"prov": "http://www.w3.org/ns/prov#",
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
"analysis": {
"@container": "@set",
"@id": "prov:wasInformedBy"
},
"entries": {
"@container": "@set",
"@id": "prov:generated"
},
"strings": {
"@container": "@set",
"@reverse": "nif:hasContext"
},
"date":
{
"@id": "dc:date",
"@type": "xsd:dateTime"
},
"text": { "@id": "nif:isString" },
"wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"senpy": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#",
"@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#"
}
......@@ -8,7 +8,7 @@ monkey.patch_all()
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
from .blueprints import nif_blueprint
from .blueprints import nif_blueprint, demo_blueprint
from git import Repo, InvalidGitRepositoryError
from functools import partial
......@@ -57,7 +57,8 @@ class Senpy(object):
app.teardown_appcontext(self.teardown)
else:
app.teardown_request(self.teardown)
app.register_blueprint(nif_blueprint)
app.register_blueprint(nif_blueprint, url_prefix="/api")
app.register_blueprint(demo_blueprint, url_prefix="/")
def add_folder(self, folder):
logger.debug("Adding folder: %s", folder)
......
'''
Senpy Models.
This implementation should mirror the JSON schema definition.
For compatibility with Py3 and for easier debugging, this new version drops introspection
and adds all arguments to the models.
'''
from __future__ import print_function
from six import string_types
import time
import copy
import json
import os
import logging
import jsonref
import jsonschema
from collections import defaultdict
from pyld import jsonld
from flask import Response as FlaskResponse
class Response(object):
DEFINITIONS_FILE = 'definitions.json'
CONTEXT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
@property
def context(self):
if not hasattr(self, '_context'):
self._context = None
return self._context
def get_schema_path(schema_file, absolute=False):
if absolute:
return os.path.realpath(schema_file)
else:
return os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', schema_file)
def read_schema(schema_file, absolute=False):
schema_path = get_schema_path(schema_file, absolute)
schema_uri = 'file://{}'.format(schema_path)
return jsonref.load(open(schema_path), base_uri=schema_uri)
base_schema = read_schema(DEFINITIONS_FILE)
logging.debug(base_schema)
class Context(dict):
@staticmethod
def get_context(context):
if isinstance(context, list):
def load(context):
logging.debug('Loading context: {}'.format(context))
if not context:
return context
elif isinstance(context, list):
contexts = []
for c in context:
contexts.append(Response.get_context(c))
contexts.append(Context.load(c))
return contexts
elif isinstance(context, dict):
return context
return Context(context)
elif isinstance(context, string_types):
try:
with open(context) as f:
return json.loads(f.read())
return Context(json.loads(f.read()))
except IOError:
return context
else:
raise AttributeError('Please, provide a valid context')
def jsonld(self, frame=None, options=None,
context=None, removeContext=None):
if removeContext is None:
removeContext = Response._context # Loop?
if frame is None:
frame = self._frame
if context is None:
context = self.context
else:
context = self.get_context(context)
# For some reason, this causes errors with pyld
# if options is None:
# options = {"expandContext": context.copy() }
js = self
if frame:
logging.debug("Framing: %s", json.dumps(self, indent=4))
logging.debug("Framing with %s", json.dumps(frame, indent=4))
js = jsonld.frame(js, frame, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
logging.debug("Compacting with %s", json.dumps(context, indent=4))
js = jsonld.compact(js, context, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
if removeContext == context:
del js["@context"]
return js
base_context = Context.load(CONTEXT_PATH)
def to_JSON(self, removeContext=None):
return json.dumps(self.jsonld(removeContext=removeContext),
default=lambda o: o.__dict__,
sort_keys=True, indent=4)
class SenpyMixin(object):
context = base_context
def flask(self,
in_headers=False,
......@@ -73,46 +73,166 @@ class Response(object):
"""
Return the values and error to be used in flask
"""
js = self.jsonld()
headers = None
if in_headers:
ctx = js["@context"]
headers = {
"Link": ('<%s>;'
'rel="http://www.w3.org/ns/json-ld#context";'
' type="application/ld+json"' % url)
}
del js["@context"]
return FlaskResponse(json.dumps(js, indent=4),
status=self.get("status", 200),
return FlaskResponse(self.to_JSON(with_context=not in_headers),
status=getattr(self, "status", 200),
headers=headers,
mimetype="application/json")
class Entry(JSONLD):
pass
class Sentiment(JSONLD):
pass
def serializable(self):
def ser_or_down(item):
if hasattr(item, 'serializable'):
return item.serializable()
elif isinstance(item, dict):
temp = dict()
for kp in item:
vp = item[kp]
temp[kp] = ser_or_down(vp)
return temp
elif isinstance(item, list):
return list(ser_or_down(i) for i in item)
else:
return item
return ser_or_down(self._plain_dict())
class EmotionSet(JSONLD):
pass
def jsonld(self, context=None, with_context=False):
ser = self.serializable()
if with_context:
ser["@context"] = self.context
class Emotion(JSONLD):
pass
return ser
class Suggestion(JSONLD):
pass
def to_JSON(self, *args, **kwargs):
js = json.dumps(self.jsonld(*args, **kwargs), indent=4,
sort_keys=True)
return js
class SenpyModel(SenpyMixin, dict):
class Error(BaseException, JSONLD):
# A better pattern would be this:
# htp://flask.pocoo.org/docs/0.10/patterns/apierrors/
_frame = {}
_context = {}
schema = base_schema
prefix = None
def __init__(self, *args, **kwargs):
self.message = kwargs.get('message', None)
super(Error, self).__init__(*args)
temp = dict(*args, **kwargs)
reqs = self.schema.get('required', [])
for i in reqs:
if i not in temp:
prop = self.schema['properties'][i]
if 'default' in prop:
temp[i] = copy.deepcopy(prop['default'])
if 'context' in temp:
context = temp['context']
del temp['context']
self.__dict__['context'] = Context.load(context)
super(SenpyModel, self).__init__(temp)
def _get_key(self, key):
key = key.replace("__", ":", 1)
return key
def __setitem__(self, key, value):
dict.__setitem__(self, key, value)
def __delitem__(self, key):
dict.__delitem__(self, key)
def __getattr__(self, key):
try:
return self.__getitem__(self._get_key(key))
except KeyError:
raise AttributeError(key)
def __setattr__(self, key, value):
self.__setitem__(self._get_key(key), value)
def __delattr__(self, key):
self.__delitem__(self._get_key(key))
def validate(self, obj=None):
if not obj:
obj = self
if hasattr(obj, "jsonld"):
obj = obj.jsonld()
jsonschema.validate(obj, self.schema)
@classmethod
def from_base(cls, name):
subschema = base_schema[name]
return warlock.model_factory(subschema, base_class=cls)
def _plain_dict(self):
d = { k: v for (k,v) in self.items() if k[0] != "_"}
if hasattr(self, "id"):
d["@id"] = self.id
return d
@property
def id(self):
if not hasattr(self, '_id'):
self.__dict__["_id"] = '_:{}_{}'.format(type(self).__name__, time.time())
return self._id
@id.setter
def id(self, value):
self._id = value
class Response(SenpyModel):
schema = read_schema('response.json')
class Results(SenpyModel):
schema = read_schema('results.json')
def jsonld(self, context=None, with_context=True):
return super(Results, self).jsonld(context, with_context)
class Entry(SenpyModel):
schema = read_schema('entry.json')
class Sentiment(SenpyModel):
schema = read_schema('sentiment.json')
class Analysis(SenpyModel):
schema = read_schema('analysis.json')
class EmotionSet(SenpyModel):
schema = read_schema('emotionSet.json')
class Suggestion(SenpyModel):
schema = read_schema('suggestion.json')
class PluginModel(SenpyModel):
schema = read_schema('plugin.json')
class Plugins(SenpyModel):
schema = read_schema('plugins.json')
class Error(SenpyMixin, BaseException ):
def __init__(self, message, status=500, params=None, errors=None, *args, **kwargs):
self.message = message
self.status = status
self.params = params or {}
self.errors = errors or ""
def _plain_dict(self):
return self.__dict__
def __str__(self):
return str(self.jsonld())
......@@ -5,7 +5,7 @@ import inspect
import os.path
import shelve
import logging
from .models import Response, Leaf
from .models import Response, PluginModel, Error
logger = logging.getLogger(__name__)
......@@ -58,36 +58,21 @@ PARAMS = {
}
class SenpyPlugin(Leaf):
_context = Leaf.get_context(Response._context)
_frame = {"@context": _context,
"name": {},
"extra_params": {"@container": "@index"},
"@explicit": True,
"version": {},
"repo": None,
"is_activated": {},
"params": None,
}
class SenpyPlugin(PluginModel):
def __init__(self, info=None):
if not info:
raise Error(message=("You need to provide configuration"
"information for the plugin."))
logger.debug("Initialising {}".format(info))
super(SenpyPlugin, self).__init__()
self.name = info["name"]
self.version = info["version"]
self.id = "{}_{}".format(self.name, self.version)
self.params = info.get("params", PARAMS.copy())
if "@id" not in self.params:
self.params["@id"] = "params_%s" % self.id
self.extra_params = info.get("extra_params", {})
self.params.update(self.extra_params.copy())
if "@id" not in self.extra_params:
self.extra_params["@id"] = "extra_params_%s" % self.id
self.is_activated = False
self._info = info
super(SenpyPlugin, self).__init__()
def get_folder(self):
return os.path.dirname(inspect.getfile(self.__class__))
......@@ -102,13 +87,6 @@ class SenpyPlugin(Leaf):
def deactivate(self):
pass
def jsonld(self, parameters=False, *args, **kwargs):
nframe = kwargs.pop("frame", self._frame)
if parameters:
nframe = nframe.copy()
nframe["params"] = {}
return super(SenpyPlugin, self).jsonld(frame=nframe, *args, **kwargs)
@property
def id(self):
return "{}_{}".format(self.name, self.version)
......@@ -123,6 +101,7 @@ class SentimentPlugin(SenpyPlugin):
super(SentimentPlugin, self).__init__(info, *args, **kwargs)
self.minPolarityValue = float(info.get("minPolarityValue", 0))
self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
self["@type"] = "marl:SentimentAnalysis"
class EmotionPlugin(SenpyPlugin):
......@@ -131,6 +110,7 @@ class EmotionPlugin(SenpyPlugin):
resp = super(EmotionPlugin, self).__init__(info, *args, **kwargs)
self.minEmotionValue = float(info.get("minEmotionValue", 0))
self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
self["@type"] = "onyx:EmotionAnalysis"
class ShelfMixin(object):
......@@ -145,6 +125,11 @@ class ShelfMixin(object):
def sh(self):
if os.path.isfile(self.shelf_file):
os.remove(self.shelf_file)
self.close()
def __del__(self):
self.close()
self.deactivate()
@property
def shelf_file(self):
......
......@@ -2,7 +2,7 @@ import json
import random
from senpy.plugins import SentimentPlugin
from senpy.models import Response, Opinion, Entry
from senpy.models import Results, Sentiment, Entry
class Sentiment140Plugin(SentimentPlugin):
......@@ -10,22 +10,33 @@ class Sentiment140Plugin(SentimentPlugin):
lang = params.get("language", "auto")
p = params.get("prefix", None)
response = Response(prefix=p)
response = Results(prefix=p)
polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
polarity = "marl:Neutral"
if polarity_value > 0:
polarity = "marl:Positive"
elif polarity_value < 0:
polarity = "marl:Negative"
entry = Entry(id="Entry0",
text=params["input"],
prefix=p)
opinion = Opinion(id="Opinion0",
prefix=p,
hasPolarity=polarity,
polarityValue=polarity_value)
opinion["prov:wasGeneratedBy"] = self.id
entry.opinions.append(opinion)
entry = Entry({"id":":Entry0",
"nif:isString": params["input"]})
sentiment = Sentiment({"id": ":Sentiment0",
"marl:hasPolarity": polarity,
"marl:polarityValue": polarity_value})
sentiment["prov:wasGeneratedBy"] = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
response.entries.append(entry)
return response
......@@ -2,7 +2,7 @@ import requests
import json
from senpy.plugins import SentimentPlugin
from senpy.models import Response, Opinion, Entry
from senpy.models import Results, Sentiment, Entry
class Sentiment140Plugin(SentimentPlugin):
......@@ -16,7 +16,7 @@ class Sentiment140Plugin(SentimentPlugin):
)
p = params.get("prefix", None)
response = Response(prefix=p)
response = Results(prefix=p)
polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
["polarity"]) * 0.25
polarity = "marl:Neutral"
......@@ -25,15 +25,16 @@ class Sentiment140Plugin(SentimentPlugin):
polarity = "marl:Positive"
elif polarity_value < neutral_value:
polarity = "marl:Negative"
entry = Entry(id="Entry0",
text=params["input"],
prefix=p)
opinion = Opinion(id="Opinion0",
prefix=p,
hasPolarity=polarity,
polarityValue=polarity_value)
opinion["prov:wasGeneratedBy"] = self.id
entry.opinions.append(opinion)
nif__isString=params["input"])
sentiment = Sentiment(id="Sentiment0",
prefix=p,
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
sentiment.prov__wasGeneratedBy = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
response.entries.append(entry)
return response
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "definitions.json#/Analysis"
}
{
"@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy#",
"dc": "http://dublincore.org/2012/06/14/dcelements#",
"me": "http://www.mixedemotions-project.eu/ns/model#",
"prov": "http://www.w3.org/ns/prov#",
"nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
"onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#",
"wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"topics": {
"@id": "dc:subject"
},
"entities": {
"@id": "me:hasEntities"
},
"suggestions": {
"@id": "me:hasSuggestions"
},
"emotions": {
"@id": "onyx:hasEmotionSet"
},
"sentiments": {
"@id": "marl:hasOpinion"
},
"entries": {
"@id": "prov:used"
},
"analysis": {
"@id": "prov:wasGeneratedBy"
}
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"Results": {
"title": "Results",
"description": "The results of an analysis",
"type": "object",
"properties": {