Commit 4ba30304 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez

New schema for parameters

* Improve extra requirement handling
* New mechanism to handle parameters beforehand in chained
  calls, and the ability to get help on available parameters in chained
  calls (through `?help`).
* Redefined Analysis, to reflect the new ontology
* Add parameters as an entity in the schema
* Update examples to include analyses and parameters
* Add processing plugins, with an interface similar to analysis plugins
* Update tests
* Avoid duplication in split plugin

Closes #51

Squashed commit of the following:

commit d145a852
commit 6a106978
commit ca69bddc
commit aa35e62a
parent 41aa142c
......@@ -6,13 +6,9 @@
],
"entries": [
{
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:beginIndex": 0,
"nif:endIndex": 40,
"nif:isString": "My favourite actress is Natalie Portman"
"text": "An entry should have a nif:isString key"
}
]
}
......@@ -3,10 +3,21 @@
"@id": "me:Result1",
"@type": "results",
"analysis": [
"me:SAnalysis1",
"me:SgAnalysis1",
"me:EmotionAnalysis1",
"me:NER1"
{
"@id": "_:SAnalysis1_Activity",
"@type": "marl:SentimentAnalysis",
"prov:wasAssociatedWith": "me:SAnalysis1"
},
{
"@id": "_:EmotionAnalysis1_Activity",
"@type": "onyx:EmotionAnalysis",
"prov:wasAssociatedWith": "me:EmotionAnalysis1"
},
{
"@id": "_:NER1_Activity",
"@type": "me:NER",
"prov:wasAssociatedWith": "me:NER1"
}
],
"entries": [
{
......@@ -23,7 +34,7 @@
"nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
"prov:wasGeneratedBy": "_:NER1_Activity"
},
{
"@id": "http://micro.blog/status1#char=25,37",
......@@ -31,7 +42,7 @@
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
"prov:wasGeneratedBy": "_:NER1_Activity"
}
],
"suggestions": [
......@@ -40,7 +51,7 @@
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
"prov:wasGeneratedBy": "_:SgAnalysis1_Activity"
}
],
"sentiments": [
......@@ -51,14 +62,14 @@
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
"prov:wasGeneratedBy": "_:SgAnalysis1_Activity"
}
],
"emotions": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"prov:wasGeneratedBy": "_:EmotionAnalysis1_Activity",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
......
{
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"me:SAnalysis1",
"me:SgAnalysis1",
"me:EmotionAnalysis1",
"me:NER1",
{
"@type": "analysis",
"@id": "anonymous"
}
],
"entries": [
{
"@id": "http://micro.blog/status1",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
{
"@id": "http://micro.blog/status1#char=5,13",
"nif:beginIndex": 5,
"nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
},
{
"@id": "http://micro.blog/status1#char=25,37",
"nif:beginIndex": 25,
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
}
],
"suggestions": [
{
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [
{
"@id": "http://micro.blog/status1#char=80,97",
"nif:beginIndex": 80,
"nif:endIndex": 97,
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
}
],
"emotions": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
},
{
"onyx:hasEmotionCategory": "wna:excitement"
}
]
}
]
}
]
}
{
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "http://example.com#NIFExample",
"@type": "results",
"analysis": [
],
"entries": [
{
"@id": "http://example.org#char=0,40",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:beginIndex": 0,
"nif:endIndex": 40,
"nif:isString": "My favourite actress is Natalie Portman"
}
]
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [ ],
"entries": [
{
"@id": "http://example.org#char=0,40",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:beginIndex": 0,
"nif:endIndex": 40,
"nif:isString": "My favourite actress is Natalie Portman"
}
]
}
{
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
{
"@id": "me:SAnalysis1",
"@type": "marl:SentimentAnalysis",
"marl:maxPolarityValue": 1,
"marl:minPolarityValue": 0
},
{
"@id": "me:SgAnalysis1",
"@type": "me:SuggestionAnalysis"
},
{
"@id": "me:EmotionAnalysis1",
"@type": "me:EmotionAnalysis"
},
{
"@id": "me:NER1",
"@type": "me:NER"
}
],
"entries": [
{
"@id": "http://micro.blog/status1",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
{
"@id": "http://micro.blog/status1#char=5,13",
"nif:beginIndex": 5,
"nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
"@id": "_:SAnalysis1_Activity",
"@type": "marl:SentimentAnalysis",
"prov:wasAssociatedWith": "me:SentimentAnalysis",
"prov:used": [
{
"name": "marl:maxPolarityValue",
"prov:value": "1"
},
{
"name": "marl:minPolarityValue",
"prov:value": "0"
}
]
},
{
"@id": "http://micro.blog/status1#char=25,37",
"nif:beginIndex": 25,
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
}
],
"suggestions": [
"@id": "_:SgAnalysis1_Activity",
"prov:wasAssociatedWith": "me:SgAnalysis1",
"@type": "me:SuggestionAnalysis"
},
{
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [
"@id": "_:EmotionAnalysis1_Activity",
"@type": "me:EmotionAnalysis",
"prov:wasAssociatedWith": "me:EmotionAnalysis1"
},
{
"@id": "http://micro.blog/status1#char=80,97",
"nif:beginIndex": 80,
"nif:endIndex": 97,
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
"@id": "_:NER1_Activity",
"@type": "me:NER",
"prov:wasAssociatedWith": "me:EmotionNER1"
}
],
"emotions": [
],
"entries": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
},
{
"onyx:hasEmotionCategory": "wna:excitement"
}
]
"@id": "http://micro.blog/status1",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
{
"@id": "http://micro.blog/status1#char=5,13",
"nif:beginIndex": 5,
"nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
},
{
"@id": "http://micro.blog/status1#char=25,37",
"nif:beginIndex": 25,
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
}
],
"suggestions": [
{
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [
{
"@id": "http://micro.blog/status1#char=80,97",
"nif:beginIndex": 80,
"nif:endIndex": 97,
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
}
],
"emotions": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
},
{
"onyx:hasEmotionCategory": "wna:excitement"
}
]
}
]
}
]
}
]
]
}
......@@ -4,8 +4,9 @@
"@type": "results",
"analysis": [
{
"@id": "me:EmotionAnalysis1",
"@type": "onyx:EmotionAnalysis"
"@id": "me:EmotionAnalysis1_Activity",
"@type": "me:EmotionAnalysis1",
"prov:wasAssociatedWith": "me:EmotionAnalysis1"
}
],
"entries": [
......@@ -26,7 +27,7 @@
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EmotionAnalysis1",
"prov:wasGeneratedBy": "_:EmotionAnalysis1_Activity",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
......
......@@ -4,8 +4,9 @@
"@type": "results",
"analysis": [
{
"@id": "me:NER1",
"@type": "me:NERAnalysis"
"@id": "_:NER1_Activity",
"@type": "me:NERAnalysis",
"prov:wasAssociatedWith": "me:NER1"
}
],
"entries": [
......
......@@ -9,9 +9,15 @@
"@type": "results",
"analysis": [
{
"@id": "me:HesamsAnalysis",
"@id": "me:HesamsAnalysis_Activity",
"@type": "onyx:EmotionAnalysis",
"onyx:usesEmotionModel": "emovoc:pad-dimensions"
"prov:wasAssociatedWith": "me:HesamsAnalysis",
"prov:used": [
{
"name": "emotion-model",
"prov:value": "emovoc:pad-dimensions"
}
]
}
],
"entries": [
......@@ -32,7 +38,7 @@
{
"@id": "Entry1#char=0,21",
"nif:anchorOf": "This is a test string",
"prov:wasGeneratedBy": "me:HesamAnalysis",
"prov:wasGeneratedBy": "_:HesamAnalysis_Activity",
"onyx:hasEmotion": [
{
"emovoc:pleasure": 0.5,
......
......@@ -4,10 +4,9 @@
"@type": "results",
"analysis": [
{
"@id": "me:SAnalysis1",
"@id": "_:SAnalysis1_Activity",
"@type": "marl:SentimentAnalysis",
"marl:maxPolarityValue": 1,
"marl:minPolarityValue": 0
"prov:wasAssociatedWith": "me:SAnalysis1"
}
],
"entries": [
......@@ -30,7 +29,7 @@
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
"prov:wasGeneratedBy": "_:SAnalysis1_Activity"
}
],
"emotionSets": [
......
......@@ -3,7 +3,11 @@
"@id": "me:Result1",
"@type": "results",
"analysis": [
"me:SgAnalysis1"
{
"@id": "_:SgAnalysis1_Activity",
"@type": "me:SuggestionAnalysis",
"prov:wasAssociatedWith": "me:SgAnalysis1"
}
],
"entries": [
{
......@@ -12,7 +16,6 @@
"nif:RFC5147String",
"nif:Context"
],
"prov:wasGeneratedBy": "me:SAnalysis1",
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
],
......@@ -22,7 +25,7 @@
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
"prov:wasGeneratedBy": "_:SgAnalysis1_Activity"
}
],
"sentiments": [
......
......@@ -8,7 +8,8 @@ boolean = [True, False]
API_PARAMS = {
"algorithm": {
"aliases": ["algorithms", "a", "algo"],
"required": False,
"required": True,
"default": 'default',
"description": ("Algorithms that will be used to process the request."
"It may be a list of comma-separated names."),
},
......@@ -41,6 +42,15 @@ API_PARAMS = {
"options": boolean,
"default": False
},
"verbose": {
"@id": "verbose",
"description": ("Show all help, including the common API parameters, or "
"only plugin-related info"),
"aliases": ["v"],
"required": True,
"options": boolean,
"default": True
},
"emotionModel": {
"@id": "emotionModel",
"aliases": ["emoModel"],
......@@ -168,8 +178,7 @@ def parse_params(indict, *specs):
outdict[param] = options["default"]
elif options.get("required", False):
wrong_params[param] = spec[param]
continue
if "options" in options:
elif "options" in options:
if options["options"] == boolean:
outdict[param] = str(outdict[param]).lower() in ['true', '1']
elif outdict[param] not in options["options"]:
......@@ -182,29 +191,116 @@ def parse_params(indict, *specs):
parameters=outdict,
errors=wrong_params)
raise message
if 'algorithm' in outdict and not isinstance(outdict['algorithm'], tuple):
outdict['algorithm'] = tuple(outdict['algorithm'].split(','))
return outdict
def parse_extra_params(request, plugins=None):
plugins = plugins or []
params = request.parameters.copy()
def get_all_params(plugins, *specs):
'''Return a list of parameters for a given set of specifications and plugins.'''
dic = {}
for s in specs:
dic.update(s)
dic.update(get_extra_params(plugins))
return dic
def get_extra_params(plugins):
'''Get a list of possible parameters given a list of plugins'''
params = {}
extra_params = {}
for plugin in plugins:
if plugin:
extra_params = parse_params(params, plugin.get('extra_params', {}))
for k, v in extra_params.items():
if k not in BUILTIN_PARAMS:
if k in params: # Set by another plugin
del params[k]
else:
params[k] = v
params['{}.{}'.format(plugin.name, k)] = v
this_params = plugin.get('extra_params', {})
for k, v in this_params.items():
if k not in extra_params:
extra_params[k] = {}
extra_params[k][plugin.name] = v
for k, v in extra_params.items(): # Resolve conflicts
if len(v) == 1: # Add the extra options that do not collide
params[k] = list(v.values())[0]
else:
required = False
aliases = None
options = None
default = None
nodefault = False # Set when defaults are not compatible
for plugin, opt in v.items():
params['{}.{}'.format(plugin, k)] = opt
required = required or opt.get('required', False)
newaliases = set(opt.get('aliases', []))
if aliases is None:
aliases = newaliases
else:
aliases = aliases & newaliases
if 'options' in opt:
newoptions = set(opt['options'])
options = newoptions if options is None else options & newoptions
if 'default' in opt:
newdefault = opt['default']
if newdefault:
if default is None and not nodefault:
default = newdefault
elif newdefault != default:
nodefault = True
default = None
# Check for incompatibilities
if options != set():
params[k] = {
'default': default,
'aliases': list(aliases),
'required': required,
'options': list(options)
}
return params
def parse_analysis(params, plugins):
'''
Parse the given parameters individually for each plugin, and get a list of the parameters that
belong to each of the plugins. Each item can then be used in the plugin.analyse_entries method.
'''
analysis_list = []
for i, plugin in enumerate(plugins):
if not plugin:
continue
this_params = filter_params(params, plugin, i)
parsed = parse_params(this_params, plugin.get('extra_params', {}))
analysis = plugin.activity(parsed)
analysis_list.append(analysis)
return analysis_list
def filter_params(params, plugin, ith=-1):
'''
Get the values within params that apply to a plugin.
More specific names override more general names, in this order:
<index_order>.parameter > <plugin.name>.parameter > parameter
Example:
>>> filter_params({'0.hello': True, 'hello': False}, Plugin(), 0)
{ '0.hello': True, 'hello': True}
'''
thisparams = {}
if ith >= 0:
ith = '{}.'.format(ith)
else:
ith = ""
for k, v in params.items():
if ith and k.startswith(str(ith)):
thisparams[k[len(ith):]] = v
elif k.startswith(plugin.name):
thisparams[k[len(plugin.name) + 1:]] = v
elif k not in thisparams:
thisparams[k] = v
return thisparams
def parse_call(params):
'''Return a results object based on the parameters used in a call/request.
'''
Return a results object based on the parameters used in a call/request.
'''
params = parse_params(params, NIF_PARAMS)
if params['informat'] == 'text':
......
......@@ -188,17 +188,27 @@ def basic_api(f):
@api_blueprint.route('/<path:plugin>', methods=['POST', 'GET'])
@basic_api
def api_root(plugin):
if plugin:
if request.parameters['algorithm'] != api.API_PARAMS['algorithm']['default']:
raise Error('You cannot specify the algorithm with a parameter and a URL variable.'
' Please, remove one of them')
request.parameters['algorithm'] = tuple(plugin.replace('+', '/').split('/'))
plugin = request.parameters['algorithm']
sp = current_app.senpy
plugins = sp.get_plugins(plugin)
if request.parameters['help']:
dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
response = Help(valid_parameters=dic)
apis = []
if request.parameters['verbose']:
apis.append(api.BUILTIN_PARAMS)
allparameters = api.get_all_params(plugins, *apis)
response = Help(valid_parameters=allparameters)
return response
req = api.parse_call(request.parameters)
if plugin:
plugin = plugin.replace('+', '/')
plugin = plugin.split('/')
req.parameters['algorithm'] = tuple(plugin)
results = current_app.senpy.analyse(req)
results.analysis = set(i.id for i in results.analysis)
analysis = api.parse_analysis(req.parameters, plugins)
results = current_app.senpy.analyse(req, analysis)
return results
......
......@@ -31,10 +31,10 @@ def main_function(argv):
default_plugins = params.get('default-plugins', False)
sp = Senpy(default_plugins=default_plugins, plugin_folder=plugin_folder)
request = api.parse_call(params)
algos = request.parameters.get('algorithm', None)
algos = sp.get_plugins(request.parameters.get('algorithm', None))
if algos:
for algo in algos:
sp.activate_plugin(algo)
sp.activate_plugin(algo.name)
else:
sp.activate_all()
res = sp.analyse(request)
......
......@@ -78,27 +78,47 @@ class Senpy(object):