Commit a243f68b authored by J. Fernando Sánchez's avatar J. Fernando Sánchez

WIP simpler pipeline

parent fca0ac00
from future.utils import iteritems from future.utils import iteritems
from .models import Error from .models import Error, Results, Entry, from_string
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
API_PARAMS = { API_PARAMS = {
"algorithm": { "algorithm": {
"aliases": ["algorithm", "a", "algo"], "aliases": ["algorithms", "a", "algo"],
"required": False, "required": False,
}, "description": ("Algorithms that will be used to process the request."
"outformat": { "It may be a list of comma-separated names."),
"@id": "outformat",
"aliases": ["outformat", "o"],
"default": "json-ld",
"required": True,
"options": ["json-ld", "turtle"],
}, },
"expanded-jsonld": { "expanded-jsonld": {
"@id": "expanded-jsonld", "@id": "expanded-jsonld",
"aliases": ["expanded", "expanded-jsonld"], "aliases": ["expanded"],
"required": True, "required": True,
"default": 0 "default": 0
}, },
"emotionModel": { "with_parameters": {
"@id": "emotionModel", "aliases": ['withparameters',
"aliases": ["emotionModel", "emoModel"], 'with-parameters'],
"required": False "options": "boolean",
"default": False,
"required": True
}, },
"plugin_type": { "plugin_type": {
"@id": "pluginType", "@id": "pluginType",
"description": 'What kind of plugins to list', "description": 'What kind of plugins to list',
"aliases": ["pluginType", "plugin_type"], "aliases": ["pluginType"],
"required": True, "required": True,
"default": "analysisPlugin" "default": "analysisPlugin"
}, },
"conversion": { "outformat": {
"@id": "conversion", "@id": "outformat",
"description": "How to show the elements that have (not) been converted", "aliases": ["o"],
"default": "json-ld",
"required": True, "required": True,
"options": ["filtered", "nested", "full"], "options": ["json-ld", "turtle"],
"default": "full"
}, },
"help": { "help": {
"@id": "help", "@id": "help",
"description": "Show additional help to know more about the possible parameters", "description": "Show additional help to know more about the possible parameters",
"aliases": ["help", "h"], "aliases": ["h"],
"required": True, "required": True,
"options": ["True", "False"], "options": "boolean",
"default": "False" "default": False
},
"emotionModel": {
"@id": "emotionModel",
"aliases": ["emoModel"],
"required": False
},
"conversion": {
"@id": "conversion",
"description": "How to show the elements that have (not) been converted",
"required": True,
"options": ["filtered", "nested", "full"],
"default": "full"
} }
} }
WEB_PARAMS = { WEB_PARAMS = {
"inHeaders": { "inHeaders": {
"aliases": ["inHeaders", "headers"], "aliases": ["headers"],
"required": True, "required": True,
"default": "0" "default": False,
"options": "boolean"
}, },
} }
CLI_PARAMS = { CLI_PARAMS = {
"plugin_folder": { "plugin_folder": {
"aliases": ["plugin_folder", "folder"], "aliases": ["folder"],
"required": True, "required": True,
"default": "." "default": "."
}, },
...@@ -69,64 +79,71 @@ CLI_PARAMS = { ...@@ -69,64 +79,71 @@ CLI_PARAMS = {
NIF_PARAMS = { NIF_PARAMS = {
"input": { "input": {
"@id": "input", "@id": "input",
"aliases": ["i", "input"], "aliases": ["i"],
"required": True, "required": True,
"help": "Input text" "help": "Input text"
}, },
"informat": {
"@id": "informat",
"aliases": ["f", "informat"],
"required": False,
"default": "text",
"options": ["turtle", "text", "json-ld"],
},
"intype": { "intype": {
"@id": "intype", "@id": "intype",
"aliases": ["intype", "t"], "aliases": ["t"],
"required": False, "required": False,
"default": "direct", "default": "direct",
"options": ["direct", "url", "file"], "options": ["direct", "url", "file"],
}, },
"informat": {
"@id": "informat",
"aliases": ["f"],
"required": False,
"default": "text",
"options": ["turtle", "text", "json-ld"],
},
"language": { "language": {
"@id": "language", "@id": "language",
"aliases": ["language", "l"], "aliases": ["l"],
"required": False, "required": False,
}, },
"prefix": { "prefix": {
"@id": "prefix", "@id": "prefix",
"aliases": ["prefix", "p"], "aliases": ["p"],
"required": True, "required": True,
"default": "", "default": "",
}, },
"urischeme": { "urischeme": {
"@id": "urischeme", "@id": "urischeme",
"aliases": ["urischeme", "u"], "aliases": ["u"],
"required": False, "required": False,
"default": "RFC5147String", "default": "RFC5147String",
"options": "RFC5147String" "options": "RFC5147String"
}, }
} }
def parse_params(indict, spec=NIF_PARAMS): def parse_params(indict, *specs):
logger.debug("Parsing: {}\n{}".format(indict, spec)) if not specs:
specs = [NIF_PARAMS]
logger.debug("Parsing: {}\n{}".format(indict, specs))
outdict = indict.copy() outdict = indict.copy()
wrong_params = {} wrong_params = {}
for param, options in iteritems(spec): for spec in specs:
if param[0] != "@": # Exclude json-ld properties for param, options in iteritems(spec):
for alias in options.get("aliases", []): if param[0] != "@": # Exclude json-ld properties
if alias in indict: for alias in options.get("aliases", []):
outdict[param] = indict[alias] # Replace each alias with the correct name of the parameter
if param not in outdict: if alias in indict and alias is not param:
if options.get("required", False) and "default" not in options: outdict[param] = indict[alias]
wrong_params[param] = spec[param] del indict[alias]
else: continue
if "default" in options: if param not in outdict:
outdict[param] = options["default"] if options.get("required", False) and "default" not in options:
else: wrong_params[param] = spec[param]
if "options" in spec[param] and \ else:
outdict[param] not in spec[param]["options"]: if "default" in options:
wrong_params[param] = spec[param] outdict[param] = options["default"]
elif "options" in spec[param]:
if spec[param]["options"] == "boolean":
outdict[param] = outdict[param] in [None, True, 'true', '1']
elif outdict[param] not in spec[param]["options"]:
wrong_params[param] = spec[param]
if wrong_params: if wrong_params:
logger.debug("Error parsing: %s", wrong_params) logger.debug("Error parsing: %s", wrong_params)
message = Error( message = Error(
...@@ -136,4 +153,30 @@ def parse_params(indict, spec=NIF_PARAMS): ...@@ -136,4 +153,30 @@ def parse_params(indict, spec=NIF_PARAMS):
errors={param: error errors={param: error
for param, error in iteritems(wrong_params)}) for param, error in iteritems(wrong_params)})
raise message raise message
if 'algorithm' in outdict and isinstance(outdict['algorithm'], str):
outdict['algorithm'] = outdict['algorithm'].split(',')
return outdict return outdict
def get_extra_params(request, plugin=None):
params = request.parameters.copy()
if plugin:
extra_params = parse_params(params, plugin.get('extra_params', {}))
params.update(extra_params)
return params
def parse_call(params):
'''Return a results object based on the parameters used in a call/request.
'''
params = parse_params(params, NIF_PARAMS)
if params['informat'] == 'text':
results = Results()
entry = Entry(nif__isString=params['input'])
results.entries.append(entry)
elif params['informat'] == 'json-ld':
results = from_string(params['input'], cls=Results)
else:
raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
results.parameters = params
return results
...@@ -19,8 +19,8 @@ Blueprints for Senpy ...@@ -19,8 +19,8 @@ Blueprints for Senpy
""" """
from flask import (Blueprint, request, current_app, render_template, url_for, from flask import (Blueprint, request, current_app, render_template, url_for,
jsonify) jsonify)
from .models import Error, Response, Plugins, read_schema from .models import Error, Response, Help, Plugins, read_schema
from .api import WEB_PARAMS, API_PARAMS, CLI_PARAMS, NIF_PARAMS, parse_params from . import api
from .version import __version__ from .version import __version__
from functools import wraps from functools import wraps
...@@ -43,6 +43,7 @@ def get_params(req): ...@@ -43,6 +43,7 @@ def get_params(req):
raise Error(message="Invalid data") raise Error(message="Invalid data")
return indict return indict
@demo_blueprint.route('/') @demo_blueprint.route('/')
def index(): def index():
return render_template("index.html", version=__version__) return render_template("index.html", version=__version__)
...@@ -75,20 +76,16 @@ def basic_api(f): ...@@ -75,20 +76,16 @@ def basic_api(f):
def decorated_function(*args, **kwargs): def decorated_function(*args, **kwargs):
raw_params = get_params(request) raw_params = get_params(request)
headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)} headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)}
# Get defaults
web_params = parse_params({}, spec=WEB_PARAMS)
api_params = parse_params({}, spec=API_PARAMS)
outformat = 'json-ld' outformat = 'json-ld'
try: try:
print('Getting request:') print('Getting request:')
print(request) print(request)
web_params = parse_params(raw_params, spec=WEB_PARAMS) params = api.parse_params(raw_params, api.WEB_PARAMS, api.API_PARAMS)
api_params = parse_params(raw_params, spec=API_PARAMS) if hasattr(request, 'parameters'):
if hasattr(request, 'params'): request.parameters.update(params)
request.params.update(api_params)
else: else:
request.params = api_params request.parameters = params
response = f(*args, **kwargs) response = f(*args, **kwargs)
except Error as ex: except Error as ex:
response = ex response = ex
...@@ -96,14 +93,14 @@ def basic_api(f): ...@@ -96,14 +93,14 @@ def basic_api(f):
if current_app.debug: if current_app.debug:
raise raise
in_headers = web_params['inHeaders'] != "0" in_headers = params['inHeaders']
expanded = api_params['expanded-jsonld'] expanded = params['expanded-jsonld']
outformat = api_params['outformat'] outformat = params['outformat']
return response.flask( return response.flask(
in_headers=in_headers, in_headers=in_headers,
headers=headers, headers=headers,
prefix=url_for('.api', _external=True), prefix=url_for('.api_root', _external=True),
context_uri=url_for('api.context', context_uri=url_for('api.context',
entity=type(response).__name__, entity=type(response).__name__,
_external=True), _external=True),
...@@ -115,14 +112,14 @@ def basic_api(f): ...@@ -115,14 +112,14 @@ def basic_api(f):
@api_blueprint.route('/', methods=['POST', 'GET']) @api_blueprint.route('/', methods=['POST', 'GET'])
@basic_api @basic_api
def api(): def api_root():
phelp = request.params.get('help') if request.parameters['help']:
if phelp == "True": dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
dic = dict(API_PARAMS, **NIF_PARAMS) response = Help(parameters=dic)
response = Response(dic)
return response return response
else: else:
response = current_app.senpy.analyse(**request.params) req = api.parse_call(request.parameters)
response = current_app.senpy.analyse(req)
return response return response
...@@ -130,7 +127,7 @@ def api(): ...@@ -130,7 +127,7 @@ def api():
@basic_api @basic_api
def plugins(): def plugins():
sp = current_app.senpy sp = current_app.senpy
ptype = request.params.get('plugin_type') ptype = request.parameters.get('plugin_type')
plugins = sp.filter_plugins(plugin_type=ptype) plugins = sp.filter_plugins(plugin_type=ptype)
dic = Plugins(plugins=list(plugins.values())) dic = Plugins(plugins=list(plugins.values()))
return dic return dic
......
import sys import sys
from .models import Error from .models import Error
from .api import parse_params, CLI_PARAMS
from .extensions import Senpy from .extensions import Senpy
from . import api
def argv_to_dict(argv): def argv_to_dict(argv):
...@@ -13,27 +13,25 @@ def argv_to_dict(argv): ...@@ -13,27 +13,25 @@ def argv_to_dict(argv):
if argv[i][0] == '-': if argv[i][0] == '-':
key = argv[i].strip('-') key = argv[i].strip('-')
value = argv[i + 1] if len(argv) > i + 1 else None value = argv[i + 1] if len(argv) > i + 1 else None
if value and value[0] == '-': if not value or value[0] == '-':
cli_dict[key] = "" cli_dict[key] = True
else: else:
cli_dict[key] = value cli_dict[key] = value
return cli_dict return cli_dict
def parse_cli(argv):
cli_dict = argv_to_dict(argv)
cli_params = parse_params(cli_dict, spec=CLI_PARAMS)
return cli_params, cli_dict
def main_function(argv): def main_function(argv):
'''This is the method for unit testing '''This is the method for unit testing
''' '''
cli_params, cli_dict = parse_cli(argv) params = api.parse_params(argv_to_dict(argv),
plugin_folder = cli_params['plugin_folder'] api.CLI_PARAMS,
api.API_PARAMS,
api.NIF_PARAMS)
plugin_folder = params['plugin_folder']
sp = Senpy(default_plugins=False, plugin_folder=plugin_folder) sp = Senpy(default_plugins=False, plugin_folder=plugin_folder)
sp.activate_all(sync=True) sp.activate_all(sync=True)
res = sp.analyse(**cli_dict) request = api.parse_call(params)
res = sp.analyse(request)
return res return res
......
...@@ -5,11 +5,10 @@ It orchestrates plugin (de)activation and analysis. ...@@ -5,11 +5,10 @@ It orchestrates plugin (de)activation and analysis.
from future import standard_library from future import standard_library
standard_library.install_aliases() standard_library.install_aliases()
from . import plugins from . import plugins, api
from .plugins import SenpyPlugin from .plugins import SenpyPlugin
from .models import Error, Entry, Results, from_string from .models import Error
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
from .api import API_PARAMS, NIF_PARAMS, parse_params
from threading import Thread from threading import Thread
...@@ -72,22 +71,20 @@ class Senpy(object): ...@@ -72,22 +71,20 @@ class Senpy(object):
else: else:
logger.debug("Not a folder: %s", folder) logger.debug("Not a folder: %s", folder)
def _find_plugins(self, params): def _get_plugins(self, request):
if not self.analysis_plugins: if not self.analysis_plugins:
raise Error( raise Error(
status=404, status=404,
message=("No plugins found." message=("No plugins found."
" Please install one.")) " Please install one."))
api_params = parse_params(params, spec=API_PARAMS) algos = request.parameters.get('algorithm', None)
algos = None if not algos:
if "algorithm" in api_params and api_params["algorithm"]: if self.default_plugin:
algos = api_params["algorithm"].split(',') algos = [self.default_plugin.name, ]
elif self.default_plugin: else:
algos = [self.default_plugin.name, ] raise Error(
else: status=404,
raise Error( message="No default plugin found, and None provided")
status=404,
message="No default plugin found, and None provided")
plugins = list() plugins = list()
for algo in algos: for algo in algos:
...@@ -108,66 +105,46 @@ class Senpy(object): ...@@ -108,66 +105,46 @@ class Senpy(object):
plugins.append(self.plugins[algo]) plugins.append(self.plugins[algo])
return plugins return plugins
def _get_params(self, params, plugin=None): def _process_entries(self, entries, req, plugins):
nif_params = parse_params(params, spec=NIF_PARAMS)
if plugin:
extra_params = plugin.get('extra_params', {})
specific_params = parse_params(params, spec=extra_params)
nif_params.update(specific_params)
return nif_params
def _get_entries(self, params):
if params['informat'] == 'text':
results = Results()
entry = Entry(text=params['input'])
results.entries.append(entry)
elif params['informat'] == 'json-ld':
results = from_string(params['input'], cls=Results)
else:
raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
return results
def _process_entries(self, entries, plugins, nif_params):
if not plugins: if not plugins:
for i in entries: for i in entries:
yield i yield i
return return
plugin = plugins[0] plugin = plugins[0]
specific_params = self._get_params(nif_params, plugin) specific_params = api.get_extra_params(req, plugin)
req.analysis.append({'plugin': plugin,
'parameters': specific_params})
results = plugin.analyse_entries(entries, specific_params) results = plugin.analyse_entries(entries, specific_params)
for i in self._process_entries(results, plugins[1:], nif_params): for i in self._process_entries(results, req, plugins[1:]):
yield i yield i
def _process_response(self, resp, plugins, nif_params): def analyse(self, request):
entries = resp.entries
resp.entries = []
for plug in plugins:
resp.analysis.append(plug.id)
for i in self._process_entries(entries, plugins, nif_params):
resp.entries.append(i)
return resp
def analyse(self, **api_params):
""" """
Main method that analyses a request, either from CLI or HTTP. Main method that analyses a request, either from CLI or HTTP.
It uses a dictionary of parameters, provided by the user. It takes a processed request, provided by the user, as returned
by api.parse_call().
""" """
logger.debug("analysing with params: {}".format(api_params)) logger.debug("analysing request: {}".format(request))
plugins = self._find_plugins(api_params)
nif_params = self._get_params(api_params)
resp = self._get_entries(nif_params)
if 'with_parameters' in api_params:
resp.parameters = nif_params
try: try:
resp = self._process_response(resp, plugins, nif_params) entries = request.entries
self.convert_emotions(resp, plugins, nif_params) request.entries = []
logger.debug("Returning analysis result: {}".format(resp)) plugins = self._get_plugins(request)
results = request
for i in self._process_entries(entries, results, plugins):
results.entries.append(i)
self.convert_emotions(results)
if 'with_parameters' not in results.parameters:
del results.parameters
logger.debug("Returning analysis result: {}".format(results))
except (Error, Exception) as ex: except (Error, Exception) as ex:
if not isinstance(ex, Error): if not isinstance(ex, Error):
ex = Error(message=str(ex), status=500) msg = "Error during analysis: {} \n\t{}".format(ex,
traceback.format_exc())
ex = Error(message=msg, status=500)
logger.exception('Error returning analysis result') logger.exception('Error returning analysis result')
raise ex raise ex
return resp