Commit b8339e39 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Improved request handling

Also:
 * Shelve -> Pickle to avoid weird db problems
 * Serving schemas and contexts
parent 407d17b2
...@@ -2,5 +2,6 @@ ...@@ -2,5 +2,6 @@
.* .*
*egg-info *egg-info
dist dist
build
README.html README.html
__pycache__ __pycache__
\ No newline at end of file
...@@ -17,17 +17,18 @@ ...@@ -17,17 +17,18 @@
""" """
Blueprints for Senpy Blueprints for Senpy
""" """
from flask import Blueprint, request, current_app, render_template from flask import Blueprint, request, current_app, render_template, url_for, jsonify
from .models import Error, Response, Plugins from .models import Error, Response, Plugins, read_schema
from future.utils import iteritems from future.utils import iteritems
from functools import wraps
import json import json
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__) api_blueprint = Blueprint("api", __name__)
demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__) demo_blueprint = Blueprint("demo", __name__)
API_PARAMS = { API_PARAMS = {
"algorithm": { "algorithm": {
...@@ -47,7 +48,7 @@ API_PARAMS = { ...@@ -47,7 +48,7 @@ API_PARAMS = {
}, },
} }
BASIC_PARAMS = { NIF_PARAMS = {
"algorithm": { "algorithm": {
"aliases": ["algorithm", "a", "algo"], "aliases": ["algorithm", "a", "algo"],
"required": False, "required": False,
...@@ -104,7 +105,7 @@ BASIC_PARAMS = { ...@@ -104,7 +105,7 @@ BASIC_PARAMS = {
}, },
} }
def get_params(req, params=BASIC_PARAMS): def update_params(req, params=NIF_PARAMS):
if req.method == 'POST': if req.method == 'POST':
indict = req.form indict = req.form
elif req.method == 'GET': elif req.method == 'GET':
...@@ -136,64 +137,73 @@ def get_params(req, params=BASIC_PARAMS): ...@@ -136,64 +137,73 @@ def get_params(req, params=BASIC_PARAMS):
parameters=outdict, parameters=outdict,
errors={param: error for param, error in errors={param: error for param, error in
iteritems(wrong_params)}) iteritems(wrong_params)})
raise Error(message=message) raise message
if hasattr(request, 'params'):
request.params.update(outdict)
else:
request.params = outdict
return outdict return outdict
def basic_analysis(params):
response = {"@context":
[("http://demos.gsi.dit.upm.es/"
"eurosentiment/static/context.jsonld"),
{
"@base": "{}#".format(request.url.encode('utf-8'))
}
],
"analysis": [{"@type": "marl:SentimentAnalysis"}],
"entries": []
}
if "language" in params:
response["language"] = params["language"]
for idx, sentence in enumerate(params["input"].split(".")):
response["entries"].append({
"@id": "Sentence{}".format(idx),
"nif:isString": sentence
})
return response
@demo_blueprint.route('/') @demo_blueprint.route('/')
def index(): def index():
return render_template("index.html") return render_template("index.html")
@api_blueprint.route('/contexts/<entity>.jsonld')
def context(entity="context"):
return jsonify({"@context": Response.context})
@nif_blueprint.route('/', methods=['POST', 'GET']) @api_blueprint.route('/schemas/<schema>')
def api(): def schema(schema="definitions"):
try: try:
params = get_params(request) return jsonify(read_schema(schema))
algo = params.get("algorithm", None) except Exception: # Should be FileNotFoundError, but it's missing from py2
specific_params = current_app.senpy.parameters(algo) return Error(message="Schema not found", status=404).flask()
logger.debug(
"Specific params: %s", json.dumps(specific_params, indent=4)) def basic_api(f):
params.update(get_params(request, specific_params)) @wraps(f)
response = current_app.senpy.analyse(**params) def decorated_function(*args, **kwargs):
in_headers = params["inHeaders"] != "0" print('Getting request:')
prefix = params["prefix"] print(request)
return response.flask(in_headers=in_headers, prefix=prefix) update_params(request, params=API_PARAMS)
except Error as ex: print('Params: %s' % request.params)
return ex.message.flask() try:
response = f(*args, **kwargs)
except Error as ex:
@nif_blueprint.route('/plugins/', methods=['POST', 'GET']) response = ex
in_headers = request.params["inHeaders"] != "0"
prefix = request.params["prefix"]
headers = {'X-ORIGINAL-PARAMS': request.params}
return response.flask(in_headers=in_headers,
prefix=prefix,
headers=headers,
context_uri=url_for('api.context', entity=type(response).__name__,
_external=True))
return decorated_function
@api_blueprint.route('/', methods=['POST', 'GET'])
@basic_api
def api():
algo = request.params.get("algorithm", None)
specific_params = current_app.senpy.parameters(algo)
update_params(request, params=NIF_PARAMS)
logger.debug("Specific params: %s", json.dumps(specific_params, indent=4))
update_params(request, specific_params)
response = current_app.senpy.analyse(**request.params)
return response
@api_blueprint.route('/plugins/', methods=['POST', 'GET'])
@basic_api
def plugins(): def plugins():
in_headers = get_params(request, API_PARAMS)["inHeaders"] != "0"
sp = current_app.senpy sp = current_app.senpy
dic = Plugins(plugins=list(sp.plugins.values())) dic = Plugins(plugins=list(sp.plugins.values()))
return dic.flask(in_headers=in_headers) return dic
@nif_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET']) @api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET']) @api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
@basic_api
def plugin(plugin=None, action="list"): def plugin(plugin=None, action="list"):
params = get_params(request, API_PARAMS)
filt = {} filt = {}
sp = current_app.senpy sp = current_app.senpy
plugs = sp.filter_plugins(name=plugin) plugs = sp.filter_plugins(name=plugin)
...@@ -203,21 +213,19 @@ def plugin(plugin=None, action="list"): ...@@ -203,21 +213,19 @@ def plugin(plugin=None, action="list"):
elif plugin in sp.plugins: elif plugin in sp.plugins:
response = sp.plugins[plugin] response = sp.plugins[plugin]
else: else:
return Error(message="Plugin not found", status=404).flask() return Error(message="Plugin not found", status=404)
if action == "list": if action == "list":
in_headers = params["inHeaders"] != "0" return response
prefix = params['prefix']
return response.flask(in_headers=in_headers, prefix=prefix)
method = "{}_plugin".format(action) method = "{}_plugin".format(action)
if(hasattr(sp, method)): if(hasattr(sp, method)):
getattr(sp, method)(plugin) getattr(sp, method)(plugin)
return Response(message="Ok").flask() return Response(message="Ok")
else: else:
return Error(message="action '{}' not allowed".format(action)).flask() return Error(message="action '{}' not allowed".format(action))
if __name__ == '__main__': if __name__ == '__main__':
import config import config
app.register_blueprint(nif_blueprint) app.register_blueprint(api_blueprint)
app.debug = config.DEBUG app.debug = config.DEBUG
app.run(host='0.0.0.0', port=5000) app.run(host='0.0.0.0', port=5000)
...@@ -8,7 +8,7 @@ monkey.patch_all() ...@@ -8,7 +8,7 @@ monkey.patch_all()
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error from .models import Error
from .blueprints import nif_blueprint, demo_blueprint from .blueprints import api_blueprint, demo_blueprint
from git import Repo, InvalidGitRepositoryError from git import Repo, InvalidGitRepositoryError
from functools import partial from functools import partial
...@@ -58,7 +58,7 @@ class Senpy(object): ...@@ -58,7 +58,7 @@ class Senpy(object):
app.teardown_appcontext(self.teardown) app.teardown_appcontext(self.teardown)
else: else:
app.teardown_request(self.teardown) app.teardown_request(self.teardown)
app.register_blueprint(nif_blueprint, url_prefix="/api") app.register_blueprint(api_blueprint, url_prefix="/api")
app.register_blueprint(demo_blueprint, url_prefix="/") app.register_blueprint(demo_blueprint, url_prefix="/")
def add_folder(self, folder): def add_folder(self, folder):
...@@ -77,28 +77,30 @@ class Senpy(object): ...@@ -77,28 +77,30 @@ class Senpy(object):
elif self.plugins: elif self.plugins:
algo = self.default_plugin and self.default_plugin.name algo = self.default_plugin and self.default_plugin.name
if not algo: if not algo:
return Error(status=404, raise Error(status=404,
message=("No plugins found." message=("No plugins found."
" Please install one.").format(algo)) " Please install one.").format(algo))
if algo in self.plugins: if algo not in self.plugins:
if self.plugins[algo].is_activated:
plug = self.plugins[algo]
resp = plug.analyse(**params)
resp.analysis.append(plug)
logger.debug("Returning analysis result: {}".format(resp))
return resp
else:
logger.debug("Plugin not activated: {}".format(algo))
return Error(status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
else:
logger.debug(("The algorithm '{}' is not valid\n" logger.debug(("The algorithm '{}' is not valid\n"
"Valid algorithms: {}").format(algo, "Valid algorithms: {}").format(algo,
self.plugins.keys())) self.plugins.keys()))
return Error(status=404, raise Error(status=404,
message="The algorithm '{}' is not valid" message="The algorithm '{}' is not valid"
.format(algo)) .format(algo))
if not self.plugins[algo].is_activated:
logger.debug("Plugin not activated: {}".format(algo))
raise Error(status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
plug = self.plugins[algo]
try:
resp = plug.analyse(**params)
resp.analysis.append(plug)
logger.debug("Returning analysis result: {}".format(resp))
except Exception as ex:
resp = Error(message=str(ex), status=500)
return resp
@property @property
def default_plugin(self): def default_plugin(self):
......
...@@ -63,28 +63,31 @@ class Context(dict): ...@@ -63,28 +63,31 @@ class Context(dict):
base_context = Context.load(CONTEXT_PATH) base_context = Context.load(CONTEXT_PATH)
class SenpyMixin(object): class SenpyMixin(object):
context = base_context context = base_context["@context"]
def flask(self, def flask(self,
in_headers=False, in_headers=False,
url="http://demos.gsi.dit.upm.es/senpy/senpy.jsonld", headers=None,
prefix=None): prefix=None,
**kwargs):
""" """
Return the values and error to be used in flask. Return the values and error to be used in flask.
So far, it returns a fixed context. We should store/generate different So far, it returns a fixed context. We should store/generate different
contexts if the plugin adds more aliases. contexts if the plugin adds more aliases.
""" """
headers = None headers = headers or {}
kwargs["with_context"] = True
js = self.jsonld(**kwargs)
if in_headers: if in_headers:
headers = { url = js["@context"]
del js["@context"]
headers.update({
"Link": ('<%s>;' "Link": ('<%s>;'
'rel="http://www.w3.org/ns/json-ld#context";' 'rel="http://www.w3.org/ns/json-ld#context";'
' type="application/ld+json"' % url) ' type="application/ld+json"' % url)
} })
return FlaskResponse(self.to_JSON(with_context=not in_headers, return FlaskResponse(json.dumps(js, indent=2, sort_keys=True),
prefix=prefix),
status=getattr(self, "status", 200), status=getattr(self, "status", 200),
headers=headers, headers=headers,
mimetype="application/json") mimetype="application/json")
...@@ -107,15 +110,27 @@ class SenpyMixin(object): ...@@ -107,15 +110,27 @@ class SenpyMixin(object):
return ser_or_down(self._plain_dict()) return ser_or_down(self._plain_dict())
def jsonld(self, context=None, prefix=None, with_context=False): def jsonld(self, prefix=None, with_context=True, context_uri=None):
ser = self.serializable() ser = self.serializable()
if with_context: if with_context:
ser["@context"] = self.context.copy() context = []
if context_uri:
context = context_uri
else:
context = self.context.copy()
if prefix: if prefix:
ser["@context"]["@base"] = prefix # This sets @base for the document, which will be used in
# all relative URIs will. For example, if a uri is "Example" and
# prefix =s "http://example.com", the absolute URI after expanding
# with JSON-LD will be "http://example.com/Example"
prefix_context = {"@base": prefix}
if isinstance(context, list):
context.append(prefix_context)
else:
context = [context, prefix_context]
ser["@context"] = context
return ser return ser
...@@ -184,11 +199,6 @@ class SenpyModel(SenpyMixin, dict): ...@@ -184,11 +199,6 @@ class SenpyModel(SenpyMixin, dict):
self.__delitem__(self._get_key(key)) self.__delitem__(self._get_key(key))
@classmethod
def from_base(cls, name):
subschema = base_schema[name]
return warlock.model_factory(subschema, base_class=cls)
def _plain_dict(self): def _plain_dict(self):
d = { k: v for (k,v) in self.items() if k[0] != "_"} d = { k: v for (k,v) in self.items() if k[0] != "_"}
d["@id"] = d.pop('id') d["@id"] = d.pop('id')
......
...@@ -3,7 +3,7 @@ standard_library.install_aliases() ...@@ -3,7 +3,7 @@ standard_library.install_aliases()
import inspect import inspect
import os.path import os.path
import shelve import pickle
import logging import logging
from .models import Response, PluginModel, Error from .models import Response, PluginModel, Error
...@@ -18,10 +18,7 @@ class SenpyPlugin(PluginModel): ...@@ -18,10 +18,7 @@ class SenpyPlugin(PluginModel):
logger.debug("Initialising {}".format(info)) logger.debug("Initialising {}".format(info))
super(SenpyPlugin, self).__init__(info) super(SenpyPlugin, self).__init__(info)
self.id = '{}_{}'.format(self.name, self.version) self.id = '{}_{}'.format(self.name, self.version)
self.params = info.get("extra_params", {})
self._info = info self._info = info
if "@id" not in self.params:
self.params["@id"] = "params_%s" % self.id
self.is_activated = False self.is_activated = False
def get_folder(self): def get_folder(self):
...@@ -64,28 +61,34 @@ class ShelfMixin(object): ...@@ -64,28 +61,34 @@ class ShelfMixin(object):
@property @property
def sh(self): def sh(self):
if not hasattr(self, '_sh') or self._sh is None: if not hasattr(self, '_sh') or self._sh is None:
self._sh = shelve.open(self.shelf_file, writeback=True) self.__dict__['_sh'] = {}
if os.path.isfile(self.shelf_file):
self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
return self._sh return self._sh
@sh.deleter @sh.deleter
def sh(self): def sh(self):
if os.path.isfile(self.shelf_file): if os.path.isfile(self.shelf_file):
os.remove(self.shelf_file) os.remove(self.shelf_file)
self.close() del self.__dict__['_sh']
self.save()
def __del__(self): def __del__(self):
self.close() self.save()
self.deactivate() super(ShelfMixin, self).__del__()
@property @property
def shelf_file(self): def shelf_file(self):
if not hasattr(self, '_shelf_file') or not self._shelf_file: if not hasattr(self, '_shelf_file') or not self._shelf_file:
if hasattr(self, '_info') and 'shelf_file' in self._info: if hasattr(self, '_info') and 'shelf_file' in self._info:
self._shelf_file = self._info['shelf_file'] self.__dict__['_shelf_file'] = self._info['shelf_file']
else: else:
self._shelf_file = os.path.join(self.get_folder(), self.name + '.db') self._shelf_file = os.path.join(self.get_folder(), self.name + '.p')
return self._shelf_file return self._shelf_file
def close(self): def save(self):
self.sh.close() logger.debug('closing pickle')
del(self._sh) if hasattr(self, '_sh') and self._sh is not None:
with open(self.shelf_file, 'wb') as f:
pickle.dump(self._sh, f)
del(self.__dict__['_sh'])
...@@ -9,8 +9,7 @@ class Sentiment140Plugin(SentimentPlugin): ...@@ -9,8 +9,7 @@ class Sentiment140Plugin(SentimentPlugin):
def analyse(self, **params): def analyse(self, **params):
lang = params.get("language", "auto") lang = params.get("language", "auto")
p = params.get("prefix", None) response = Results()
response = Results(prefix=p)
polarity_value = max(-1, min(1, random.gauss(0.2, 0.2))) polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
polarity = "marl:Neutral" polarity = "marl:Neutral"
if polarity_value > 0: if polarity_value > 0:
......
{ {
"@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy#", "@context": {
"dc": "http://dublincore.org/2012/06/14/dcelements#", "@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy#",
"me": "http://www.mixedemotions-project.eu/ns/model#", "dc": "http://dublincore.org/2012/06/14/dcelements#",
"prov": "http://www.w3.org/ns/prov#", "me": "http://www.mixedemotions-project.eu/ns/model#",
"nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#", "prov": "http://www.w3.org/ns/prov#",
"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#", "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
"onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#", "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
"wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#", "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#",
"xsd": "http://www.w3.org/2001/XMLSchema#", "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
"topics": { "xsd": "http://www.w3.org/2001/XMLSchema#",
"@id": "dc:subject" "topics": {
}, "@id": "dc:subject"
"entities": { },
"@id": "me:hasEntities" "entities": {
}, "@id": "me:hasEntities"
"suggestions": { },
"@id": "me:hasSuggestions" "suggestions": {
}, "@id": "me:hasSuggestions"
"emotions": { },
"@id": "onyx:hasEmotionSet" "emotions": {
}, "@id": "onyx:hasEmotionSet"
"sentiments": { },
"@id": "marl:hasOpinion" "sentiments": {
}, "@id": "marl:hasOpinion"
"entries": { },
"@id": "prov:used" "entries": {
}, "@id": "prov:used"
"analysis": { },
"@id": "prov:wasGeneratedBy" "analysis": {
"@id": "prov:wasGeneratedBy"
} }
}
} }
...@@ -152,10 +152,14 @@ ...@@ -152,10 +152,14 @@
}, },
"Plugin": { "Plugin": {
"type": "object", "type": "object",
"required": ["@id"], "required": ["@id", "extra_params"],
"properties": { "properties": {
"@id": { "@id": {
"type": "string"