Commit 14c9f618 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Python 3 compatible

There are also some slight changes to the JSON schemas and the use of
JSON-LD.
parent a79df7a3
...@@ -3,3 +3,4 @@ ...@@ -3,3 +3,4 @@
*egg-info *egg-info
dist dist
README.html README.html
__pycache__
\ No newline at end of file
...@@ -2,6 +2,10 @@ Flask>=0.10.1 ...@@ -2,6 +2,10 @@ Flask>=0.10.1
gunicorn>=19.0.0 gunicorn>=19.0.0
requests>=2.4.1 requests>=2.4.1
GitPython>=0.3.2.RC1 GitPython>=0.3.2.RC1
gevent>=1.0.1 gevent>=1.1rc4
PyLD>=0.6.5 PyLD>=0.6.5
Flask-Testing>=0.4.2 Flask-Testing>=0.4.2
six
future
jsonschema
jsonref
...@@ -17,8 +17,8 @@ ...@@ -17,8 +17,8 @@
""" """
Blueprints for Senpy Blueprints for Senpy
""" """
from flask import Blueprint, request, current_app, Flask, redirect, url_for, render_template from flask import Blueprint, request, current_app, render_template
from .models import Error, Response, Leaf from .models import Error, Response
from future.utils import iteritems from future.utils import iteritems
import json import json
...@@ -27,6 +27,7 @@ import logging ...@@ -27,6 +27,7 @@ import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__) nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__)
BASIC_PARAMS = { BASIC_PARAMS = {
"algorithm": { "algorithm": {
...@@ -40,15 +41,6 @@ BASIC_PARAMS = { ...@@ -40,15 +41,6 @@ BASIC_PARAMS = {
} }
} }
LIST_PARAMS = {
"params": {
"aliases": ["params", "with_params"],
"required": False,
"default": "0"
},
}
def get_params(req, params=BASIC_PARAMS): def get_params(req, params=BASIC_PARAMS):
if req.method == 'POST': if req.method == 'POST':
indict = req.form indict = req.form
...@@ -76,12 +68,11 @@ def get_params(req, params=BASIC_PARAMS): ...@@ -76,12 +68,11 @@ def get_params(req, params=BASIC_PARAMS):
outdict[param] not in params[param]["options"]: outdict[param] not in params[param]["options"]:
wrong_params[param] = params[param] wrong_params[param] = params[param]
if wrong_params: if wrong_params:
message = Error({"status": 404, message = Error(status=404,
"message": "Missing or invalid parameters", message="Missing or invalid parameters",
"parameters": outdict, parameters=outdict,
"errors": {param: error for param, error in errors={param: error for param, error in
iteritems(wrong_params)} iteritems(wrong_params)})
})
raise Error(message=message) raise Error(message=message)
return outdict return outdict
...@@ -107,12 +98,12 @@ def basic_analysis(params): ...@@ -107,12 +98,12 @@ def basic_analysis(params):
return response return response
@nif_blueprint.route('/') @demo_blueprint.route('/')
def index(): def index():
return render_template("index.html") return render_template("index.html")
@nif_blueprint.route('/api', methods=['POST', 'GET']) @nif_blueprint.route('/', methods=['POST', 'GET'])
def api(): def api():
try: try:
params = get_params(request) params = get_params(request)
...@@ -128,7 +119,7 @@ def api(): ...@@ -128,7 +119,7 @@ def api():
return ex.message.flask() return ex.message.flask()
@nif_blueprint.route("/api/default") @nif_blueprint.route("/default")
def default(): def default():
# return current_app.senpy.default_plugin # return current_app.senpy.default_plugin
plug = current_app.senpy.default_plugin plug = current_app.senpy.default_plugin
...@@ -139,9 +130,9 @@ def default(): ...@@ -139,9 +130,9 @@ def default():
return error.flask() return error.flask()
@nif_blueprint.route('/api/plugins/', methods=['POST', 'GET']) @nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
@nif_blueprint.route('/api/plugins/<plugin>', methods=['POST', 'GET']) @nif_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@nif_blueprint.route('/api/plugins/<plugin>/<action>', methods=['POST', 'GET']) @nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
def plugins(plugin=None, action="list"): def plugins(plugin=None, action="list"):
filt = {} filt = {}
sp = current_app.senpy sp = current_app.senpy
...@@ -151,21 +142,19 @@ def plugins(plugin=None, action="list"): ...@@ -151,21 +142,19 @@ def plugins(plugin=None, action="list"):
if plugin and not plugs: if plugin and not plugs:
return "Plugin not found", 400 return "Plugin not found", 400
if action == "list": if action == "list":
with_params = get_params(request, LIST_PARAMS)["params"] == "1"
in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0" in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0"
if plugin: if plugin:
dic = plugs[plugin] dic = plugs[plugin]
else: else:
dic = Response( dic = Response(
{plug: plugs[plug].jsonld(with_params) for plug in plugs}, {plug: plugs[plug].serializable() for plug in plugs})
frame={})
return dic.flask(in_headers=in_headers) return dic.flask(in_headers=in_headers)
method = "{}_plugin".format(action) method = "{}_plugin".format(action)
if(hasattr(sp, method)): if(hasattr(sp, method)):
getattr(sp, method)(plugin) getattr(sp, method)(plugin)
return Leaf(message="Ok").flask() return Response(message="Ok").flask()
else: else:
return Error("action '{}' not allowed".format(action)).flask() return Error(message="action '{}' not allowed".format(action)).flask()
if __name__ == '__main__': if __name__ == '__main__':
......
{
"dc": "http://purl.org/dc/terms/",
"dc:subject": {
"@type": "@id"
},
"xsd": "http://www.w3.org/2001/XMLSchema#",
"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
"nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
"onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#",
"emotions": {
"@container": "@set",
"@id": "onyx:hasEmotionSet"
},
"opinions": {
"@container": "@set",
"@id": "marl:hasOpinion"
},
"prov": "http://www.w3.org/ns/prov#",
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
"analysis": {
"@container": "@set",
"@id": "prov:wasInformedBy"
},
"entries": {
"@container": "@set",
"@id": "prov:generated"
},
"strings": {
"@container": "@set",
"@reverse": "nif:hasContext"
},
"date":
{
"@id": "dc:date",
"@type": "xsd:dateTime"
},
"text": { "@id": "nif:isString" },
"wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"senpy": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#",
"@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#"
}
...@@ -8,7 +8,7 @@ monkey.patch_all() ...@@ -8,7 +8,7 @@ monkey.patch_all()
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error from .models import Error
from .blueprints import nif_blueprint from .blueprints import nif_blueprint, demo_blueprint
from git import Repo, InvalidGitRepositoryError from git import Repo, InvalidGitRepositoryError
from functools import partial from functools import partial
...@@ -57,7 +57,8 @@ class Senpy(object): ...@@ -57,7 +57,8 @@ class Senpy(object):
app.teardown_appcontext(self.teardown) app.teardown_appcontext(self.teardown)
else: else:
app.teardown_request(self.teardown) app.teardown_request(self.teardown)
app.register_blueprint(nif_blueprint) app.register_blueprint(nif_blueprint, url_prefix="/api")
app.register_blueprint(demo_blueprint, url_prefix="/")
def add_folder(self, folder): def add_folder(self, folder):
logger.debug("Adding folder: %s", folder) logger.debug("Adding folder: %s", folder)
......
'''
Senpy Models.
This implementation should mirror the JSON schema definition.
For compatibility with Py3 and for easier debugging, this new version drops introspection
and adds all arguments to the models.
'''
from __future__ import print_function from __future__ import print_function
from six import string_types from six import string_types
import time
import copy
import json import json
import os import os
import logging import logging
import jsonref
import jsonschema
from collections import defaultdict
from pyld import jsonld
from flask import Response as FlaskResponse from flask import Response as FlaskResponse
class Response(object): DEFINITIONS_FILE = 'definitions.json'
CONTEXT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
@property def get_schema_path(schema_file, absolute=False):
def context(self): if absolute:
if not hasattr(self, '_context'): return os.path.realpath(schema_file)
self._context = None else:
return self._context return os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', schema_file)
def read_schema(schema_file, absolute=False):
schema_path = get_schema_path(schema_file, absolute)
schema_uri = 'file://{}'.format(schema_path)
return jsonref.load(open(schema_path), base_uri=schema_uri)
base_schema = read_schema(DEFINITIONS_FILE)
logging.debug(base_schema)
class Context(dict):
@staticmethod @staticmethod
def get_context(context): def load(context):
if isinstance(context, list): logging.debug('Loading context: {}'.format(context))
if not context:
return context
elif isinstance(context, list):
contexts = [] contexts = []
for c in context: for c in context:
contexts.append(Response.get_context(c)) contexts.append(Context.load(c))
return contexts return contexts
elif isinstance(context, dict): elif isinstance(context, dict):
return context return Context(context)
elif isinstance(context, string_types): elif isinstance(context, string_types):
try: try:
with open(context) as f: with open(context) as f:
return json.loads(f.read()) return Context(json.loads(f.read()))
except IOError: except IOError:
return context return context
else: else:
raise AttributeError('Please, provide a valid context') raise AttributeError('Please, provide a valid context')
def jsonld(self, frame=None, options=None, base_context = Context.load(CONTEXT_PATH)
context=None, removeContext=None):
if removeContext is None:
removeContext = Response._context # Loop?
if frame is None:
frame = self._frame
if context is None:
context = self.context
else:
context = self.get_context(context)
# For some reason, this causes errors with pyld
# if options is None:
# options = {"expandContext": context.copy() }
js = self
if frame:
logging.debug("Framing: %s", json.dumps(self, indent=4))
logging.debug("Framing with %s", json.dumps(frame, indent=4))
js = jsonld.frame(js, frame, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
logging.debug("Compacting with %s", json.dumps(context, indent=4))
js = jsonld.compact(js, context, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
if removeContext == context:
del js["@context"]
return js
def to_JSON(self, removeContext=None): class SenpyMixin(object):
return json.dumps(self.jsonld(removeContext=removeContext), context = base_context
default=lambda o: o.__dict__,
sort_keys=True, indent=4)
def flask(self, def flask(self,
in_headers=False, in_headers=False,
...@@ -73,46 +73,166 @@ class Response(object): ...@@ -73,46 +73,166 @@ class Response(object):
""" """
Return the values and error to be used in flask Return the values and error to be used in flask
""" """
js = self.jsonld()
headers = None headers = None
if in_headers: if in_headers:
ctx = js["@context"]
headers = { headers = {
"Link": ('<%s>;' "Link": ('<%s>;'
'rel="http://www.w3.org/ns/json-ld#context";' 'rel="http://www.w3.org/ns/json-ld#context";'
' type="application/ld+json"' % url) ' type="application/ld+json"' % url)
} }
del js["@context"] return FlaskResponse(self.to_JSON(with_context=not in_headers),
return FlaskResponse(json.dumps(js, indent=4), status=getattr(self, "status", 200),
status=self.get("status", 200),
headers=headers, headers=headers,
mimetype="application/json") mimetype="application/json")
class Entry(JSONLD):
pass def serializable(self):
def ser_or_down(item):
if hasattr(item, 'serializable'):
return item.serializable()
elif isinstance(item, dict):
temp = dict()
for kp in item:
vp = item[kp]
temp[kp] = ser_or_down(vp)
return temp
elif isinstance(item, list):
return list(ser_or_down(i) for i in item)
else:
return item
return ser_or_down(self._plain_dict())
class Sentiment(JSONLD): def jsonld(self, context=None, with_context=False):
pass ser = self.serializable()
if with_context:
ser["@context"] = self.context
class EmotionSet(JSONLD): return ser
pass
class Emotion(JSONLD): def to_JSON(self, *args, **kwargs):
pass js = json.dumps(self.jsonld(*args, **kwargs), indent=4,
sort_keys=True)
return js
class Suggestion(JSONLD): class SenpyModel(SenpyMixin, dict):
pass
class Error(BaseException, JSONLD): schema = base_schema
# A better pattern would be this: prefix = None
# htp://flask.pocoo.org/docs/0.10/patterns/apierrors/
_frame = {}
_context = {}
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.message = kwargs.get('message', None) temp = dict(*args, **kwargs)
super(Error, self).__init__(*args)
reqs = self.schema.get('required', [])
for i in reqs:
if i not in temp:
prop = self.schema['properties'][i]
if 'default' in prop:
temp[i] = copy.deepcopy(prop['default'])
if 'context' in temp:
context = temp['context']
del temp['context']
self.__dict__['context'] = Context.load(context)
super(SenpyModel, self).__init__(temp)
def _get_key(self, key):
key = key.replace("__", ":", 1)
return key
def __setitem__(self, key, value):
dict.__setitem__(self, key, value)
def __delitem__(self, key):
dict.__delitem__(self, key)
def __getattr__(self, key):
try:
return self.__getitem__(self._get_key(key))
except KeyError:
raise AttributeError(key)
def __setattr__(self, key, value):
self.__setitem__(self._get_key(key), value)
def __delattr__(self, key):
self.__delitem__(self._get_key(key))
def validate(self, obj=None):
if not obj:
obj = self
if hasattr(obj, "jsonld"):
obj = obj.jsonld()
jsonschema.validate(obj, self.schema)
@classmethod
def from_base(cls, name):
subschema = base_schema[name]
return warlock.model_factory(subschema, base_class=cls)
def _plain_dict(self):
d = { k: v for (k,v) in self.items() if k[0] != "_"}
if hasattr(self, "id"):
d["@id"] = self.id
return d
@property
def id(self):
if not hasattr(self, '_id'):
self.__dict__["_id"] = '_:{}_{}'.format(type(self).__name__, time.time())
return self._id
@id.setter
def id(self, value):
self._id = value
class Response(SenpyModel):
schema = read_schema('response.json')
class Results(SenpyModel):
schema = read_schema('results.json')
def jsonld(self, context=None, with_context=True):
return super(Results, self).jsonld(context, with_context)
class Entry(SenpyModel):
schema = read_schema('entry.json')
class Sentiment(SenpyModel):
schema = read_schema('sentiment.json')
class Analysis(SenpyModel):
schema = read_schema('analysis.json')
class EmotionSet(SenpyModel):
schema = read_schema('emotionSet.json')
class Suggestion(SenpyModel):
schema = read_schema('suggestion.json')
class PluginModel(SenpyModel):
schema = read_schema('plugin.json')
class Plugins(SenpyModel):
schema = read_schema('plugins.json')
class Error(SenpyMixin, BaseException ):
def __init__(self, message, status=500, params=None, errors=None, *args, **kwargs):
self.message = message
self.status = status
self.params = params or {}
self.errors = errors or ""
def _plain_dict(self):
return self.__dict__
def __str__(self):
return str(self.jsonld())
...@@ -5,7 +5,7 @@ import inspect ...@@ -5,7 +5,7 @@ import inspect
import os.path import os.path
import shelve import shelve
import logging import logging
from .models import Response, Leaf from .models import Response, PluginModel, Error
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -58,36 +58,21 @@ PARAMS = { ...@@ -58,36 +58,21 @@ PARAMS = {
} }
class SenpyPlugin(Leaf): class SenpyPlugin(PluginModel):
_context = Leaf.get_context(Response._context)
_frame = {"@context": _context,
"name": {},
"extra_params": {"@container": "@index"},
"@explicit": True,
"version": {},
"repo": None,
"is_activated": {},
"params": None,