Commit 8a516d92 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Multiple changes in the API, schemas and UI

Check out the CHANGELOG.md file for more information
parent 4ba30304
This is a collection of plugins that exemplify certain aspects of plugin development with senpy.
The first series of plugins the `basic` ones.
The first series of plugins are the `basic` ones.
Their starting point is a classification function defined in `basic.py`.
They all include testing and running them as a script will run all tests.
In ascending order of customization, the plugins are:
......@@ -19,5 +19,5 @@ In rest of the plugins show advanced topics:
All of the plugins in this folder include a set of test cases and they are periodically tested with the latest version of senpy.
Additioanlly, for an example of stand-alone plugin that can be tested and deployed with docker, take a look at: lab.cluster.gsi.dit.upm.es/senpy/plugin-example
Additioanlly, for an example of stand-alone plugin that can be tested and deployed with docker, take a look at: lab.gsi.upm.es/senpy/plugin-example
bbm
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-
emoticons = {
'pos': [':)', ':]', '=)', ':D'],
......@@ -7,17 +7,19 @@ emoticons = {
}
emojis = {
'pos': ['😁', '😂', '😃', '😄', '😆', '😅', '😄' '😍'],
'neg': ['😢', '😡', '😠', '😞', '😖', '😔', '😓', '😒']
'pos': [u'😁', u'😂', u'😃', u'😄', u'😆', u'😅', u'😄', u'😍'],
'neg': [u'😢', u'😡', u'😠', u'😞', u'😖', u'😔', u'😓', u'😒']
}
def get_polarity(text, dictionaries=[emoticons, emojis]):
polarity = 'marl:Neutral'
print('Input for get_polarity', text)
for dictionary in dictionaries:
for label, values in dictionary.items():
for emoticon in values:
if emoticon and emoticon in text:
polarity = label
break
print('Polarity', polarity)
return polarity
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-
from senpy import easy_test, models, plugins
......@@ -18,13 +18,13 @@ class BasicAnalyseEntry(plugins.SentimentPlugin):
'default': 'marl:Neutral'
}
def analyse_entry(self, entry, params):
def analyse_entry(self, entry, activity):
polarity = basic.get_polarity(entry.text)
polarity = self.mappings.get(polarity, self.mappings['default'])
s = models.Sentiment(marl__hasPolarity=polarity)
s.prov(self)
s.prov(activity)
entry.sentiments.append(s)
yield entry
......
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-
from senpy import easy_test, SentimentBox
......@@ -12,15 +12,13 @@ class BasicBox(SentimentBox):
author = '@balkian'
version = '0.1'
mappings = {
'pos': 'marl:Positive',
'neg': 'marl:Negative',
'default': 'marl:Neutral'
}
def predict_one(self, input):
output = basic.get_polarity(input)
return self.mappings.get(output, self.mappings['default'])
def predict_one(self, features, **kwargs):
output = basic.get_polarity(features[0])
if output == 'pos':
return [1, 0, 0]
if output == 'neg':
return [0, 0, 1]
return [0, 1, 0]
test_cases = [{
'input': 'Hello :)',
......
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-
from senpy import easy_test, SentimentBox, MappingMixin
from senpy import easy_test, SentimentBox
import basic
class Basic(MappingMixin, SentimentBox):
class Basic(SentimentBox):
'''Provides sentiment annotation using a lexicon'''
author = '@balkian'
version = '0.1'
mappings = {
'pos': 'marl:Positive',
'neg': 'marl:Negative',
'default': 'marl:Neutral'
}
def predict_one(self, input):
return basic.get_polarity(input)
def predict_one(self, features, **kwargs):
output = basic.get_polarity(features[0])
if output == 'pos':
return [1, 0, 0]
if output == 'neu':
return [0, 1, 0]
return [0, 0, 1]
test_cases = [{
'input': 'Hello :)',
'input': u'Hello :)',
'polarity': 'marl:Positive'
}, {
'input': 'So sad :(',
'input': u'So sad :(',
'polarity': 'marl:Negative'
}, {
'input': 'Yay! Emojis 😁',
'input': u'Yay! Emojis 😁',
'polarity': 'marl:Positive'
}, {
'input': 'But no emoticons 😢',
'input': u'But no emoticons 😢',
'polarity': 'marl:Negative'
}]
......
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-
from senpy import easy_test, models, plugins
......@@ -16,7 +16,7 @@ class Dictionary(plugins.SentimentPlugin):
mappings = {'pos': 'marl:Positive', 'neg': 'marl:Negative'}
def analyse_entry(self, entry, params):
def analyse_entry(self, entry, *args, **kwargs):
polarity = basic.get_polarity(entry.text, self.dictionaries)
if polarity in self.mappings:
polarity = self.mappings[polarity]
......
......@@ -6,12 +6,13 @@ from senpy.models import EmotionSet, Emotion, Entry
class EmoRand(EmotionPlugin):
'''A sample plugin that returns a random emotion annotation'''
name = 'emotion-random'
author = '@balkian'
version = '0.1'
url = "https://github.com/gsi-upm/senpy-plugins-community"
onyx__usesEmotionModel = "emoml:big6"
def analyse_entry(self, entry, params):
def analyse_entry(self, entry, activity):
category = "emoml:big6happiness"
number = max(-1, min(1, random.gauss(0, 0.5)))
if number > 0:
......@@ -19,7 +20,7 @@ class EmoRand(EmotionPlugin):
emotionSet = EmotionSet()
emotion = Emotion({"onyx:hasEmotionCategory": category})
emotionSet.onyx__hasEmotion.append(emotion)
emotionSet.prov__wasGeneratedBy = self.id
emotionSet.prov(activity)
entry.emotions.append(emotionSet)
yield entry
......@@ -27,6 +28,6 @@ class EmoRand(EmotionPlugin):
params = dict()
results = list()
for i in range(100):
res = next(self.analyse_entry(Entry(nif__isString="Hello"), params))
res = next(self.analyse_entry(Entry(nif__isString="Hello"), self.activity(params)))
res.validate()
results.append(res.emotions[0]['onyx:hasEmotion'][0]['onyx:hasEmotionCategory'])
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-
from senpy import easy_test, models, plugins
......@@ -25,7 +25,8 @@ class ParameterizedDictionary(plugins.SentimentPlugin):
}
}
def analyse_entry(self, entry, params):
def analyse_entry(self, entry, activity):
params = activity.params
positive_words = params['positive-words'].split(',')
negative_words = params['negative-words'].split(',')
dictionary = {
......@@ -35,7 +36,7 @@ class ParameterizedDictionary(plugins.SentimentPlugin):
polarity = basic.get_polarity(entry.text, [dictionary])
s = models.Sentiment(marl__hasPolarity=polarity)
s.prov(self)
s.prov(activity)
entry.sentiments.append(s)
yield entry
......
......@@ -2,15 +2,16 @@ import random
from senpy import SentimentPlugin, Sentiment, Entry
class Rand(SentimentPlugin):
class RandSent(SentimentPlugin):
'''A sample plugin that returns a random sentiment annotation'''
name = 'sentiment-random'
author = "@balkian"
version = '0.1'
url = "https://github.com/gsi-upm/senpy-plugins-community"
marl__maxPolarityValue = '1'
marl__minPolarityValue = "-1"
def analyse_entry(self, entry, params):
def analyse_entry(self, entry, activity):
polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
polarity = "marl:Neutral"
if polarity_value > 0:
......@@ -19,7 +20,7 @@ class Rand(SentimentPlugin):
polarity = "marl:Negative"
sentiment = Sentiment(marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
sentiment.prov(self)
sentiment.prov(activity)
entry.sentiments.append(sentiment)
yield entry
......@@ -28,8 +29,9 @@ class Rand(SentimentPlugin):
params = dict()
results = list()
for i in range(50):
activity = self.activity(params)
res = next(self.analyse_entry(Entry(nif__isString="Hello"),
params))
activity))
res.validate()
results.append(res.sentiments[0]['marl:hasPolarity'])
assert 'marl:Positive' in results
......
from senpy import SentimentBox, MappingMixin, easy_test
from senpy import SentimentBox, easy_test
from mypipeline import pipeline
class PipelineSentiment(MappingMixin, SentimentBox):
'''
This is a pipeline plugin that wraps a classifier defined in another module
(mypipeline).
'''
class PipelineSentiment(SentimentBox):
'''This is a pipeline plugin that wraps a classifier defined in another module
(mypipeline).'''
author = '@balkian'
version = 0.1
maxPolarityValue = 1
minPolarityValue = -1
mappings = {
1: 'marl:Positive',
-1: 'marl:Negative'
}
def predict_one(self, input):
return pipeline.predict([input, ])[0]
def predict_one(self, features, **kwargs):
if pipeline.predict(features) > 0:
return [1, 0, 0]
return [0, 0, 1]
test_cases = [
{
......
......@@ -15,8 +15,6 @@ spec:
- name: senpy-latest
image: $IMAGEWTAG
imagePullPolicy: Always
args:
- "--default-plugins"
resources:
limits:
memory: "512Mi"
......
......@@ -12,3 +12,10 @@ spec:
backend:
serviceName: senpy-latest
servicePort: 5000
- host: latest.senpy.gsi.upm.es
http:
paths:
- path: /
backend:
serviceName: senpy-latest
servicePort: 5000
......@@ -40,8 +40,14 @@ def main():
'-l',
metavar='logging_level',
type=str,
default="WARN",
default="INFO",
help='Logging level')
parser.add_argument(
'--log-format',
metavar='log_format',
type=str,
default='%(asctime)s %(levelname)-10s %(name)-30s \t %(message)s',
help='Logging format')
parser.add_argument(
'--debug',
'-d',
......@@ -49,10 +55,10 @@ def main():
default=False,
help='Run the application in debug mode')
parser.add_argument(
'--default-plugins',
'--no-default-plugins',
action='store_true',
default=False,
help='Load the default plugins')
help='Do not load the default plugins')
parser.add_argument(
'--host',
type=str,
......@@ -68,7 +74,7 @@ def main():
'--plugins-folder',
'-f',
type=str,
default='.',
action='append',
help='Where to look for plugins.')
parser.add_argument(
'--only-install',
......@@ -100,10 +106,10 @@ def main():
default=None,
help='Where to look for data. It be set with the SENPY_DATA environment variable as well.')
parser.add_argument(
'--threaded',
action='store_false',
default=True,
help='Run a threaded server')
'--no-threaded',
action='store_true',
default=False,
help='Run a single-threaded server')
parser.add_argument(
'--no-deps',
'-n',
......@@ -123,30 +129,42 @@ def main():
default=False,
help='Do not exit if some plugins fail to activate')
args = parser.parse_args()
print('Senpy version {}'.format(senpy.__version__))
print(sys.version)
if args.version:
print('Senpy version {}'.format(senpy.__version__))
print(sys.version)
exit(1)
rl = logging.getLogger()
rl.setLevel(getattr(logging, args.level))
logger_handler = rl.handlers[0]
# First, generic formatter:
logger_handler.setFormatter(logging.Formatter(args.log_format))
app = Flask(__name__)
app.debug = args.debug
sp = Senpy(app, args.plugins_folder,
default_plugins=args.default_plugins,
sp = Senpy(app,
plugin_folder=None,
default_plugins=not args.no_default_plugins,
data_folder=args.data_folder)
folders = list(args.plugins_folder) if args.plugins_folder else []
if not folders:
folders.append(".")
for p in folders:
sp.add_folder(p)
plugins = sp.plugins(plugin_type=None, is_activated=False)
maxname = max(len(x.name) for x in plugins)
maxversion = max(len(str(x.version)) for x in plugins)
print('Found {} plugins:'.format(len(plugins)))
for plugin in plugins:
import inspect
fpath = inspect.getfile(plugin.__class__)
print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
plugin.version,
fpath,
maxname=maxname,
maxversion=maxversion))
if args.only_list:
plugins = sp.plugins()
maxname = max(len(x.name) for x in plugins)
maxversion = max(len(x.version) for x in plugins)
print('Found {} plugins:'.format(len(plugins)))
for plugin in plugins:
import inspect
fpath = inspect.getfile(plugin.__class__)
print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
plugin.version,
fpath,
maxname=maxname,
maxversion=maxversion))
return
if not args.no_deps:
sp.install_deps()
......@@ -160,10 +178,13 @@ def main():
print('Senpy version {}'.format(senpy.__version__))
print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
args.port))
app.run(args.host,
args.port,
threaded=args.threaded,
debug=app.debug)
try:
app.run(args.host,
args.port,
threaded=not args.no_threaded,
debug=app.debug)
except KeyboardInterrupt:
print('Bye!')
sp.deactivate_all()
......
......@@ -5,24 +5,31 @@ logger = logging.getLogger(__name__)
boolean = [True, False]
processors = {
'string_to_tuple': lambda p: p if isinstance(p, (tuple, list)) else tuple(p.split(','))
}
API_PARAMS = {
"algorithm": {
"aliases": ["algorithms", "a", "algo"],
"required": True,
"default": 'default',
"processor": 'string_to_tuple',
"description": ("Algorithms that will be used to process the request."
"It may be a list of comma-separated names."),
},
"expanded-jsonld": {
"@id": "expanded-jsonld",
"aliases": ["expanded"],
"description": "use JSON-LD expansion to get full URIs",
"aliases": ["expanded", "expanded_jsonld"],
"options": boolean,
"required": True,
"default": False
},
"with_parameters": {
"with-parameters": {
"aliases": ['withparameters',
'with-parameters'],
'with_parameters'],
"description": "include initial parameters in the response",
"options": boolean,
"default": False,
"required": True
......@@ -31,9 +38,67 @@ API_PARAMS = {
"@id": "outformat",
"aliases": ["o"],
"default": "json-ld",
"description": """The data can be semantically formatted (JSON-LD, turtle or n-triples),
given as a list of comma-separated fields (see the fields option) or constructed from a Jinja2
template (see the template option).""",
"required": True,
"options": ["json-ld", "turtle", "ntriples"],
},
"template": {
"@id": "template",
"required": False,
"description": """Jinja2 template for the result. The input data for the template will
be the results as a dictionary.
For example:
Consider the results before templating:
```
[{
"@type": "entry",
"onyx:hasEmotionSet": [],
"nif:isString": "testing the template",
"marl:hasOpinion": [
{
"@type": "sentiment",
"marl:hasPolarity": "marl:Positive"
}
]
}]
```
And the template:
```
{% for entry in entries %}
{{ entry["nif:isString"] | upper }},{{entry.sentiments[0]["marl:hasPolarity"].split(":")[1]}}
{% endfor %}
```
The final result would be:
```
TESTING THE TEMPLATE,Positive
```
"""
},
"fields": {
"@id": "fields",
"required": False,
"description": """A jmespath selector, that can be used to extract a new dictionary, array or value
from the results.
jmespath is a powerful query language for json and/or dictionaries.
It allows you to change the structure (and data) of your objects through queries.
e.g., the following expression gets a list of `[emotion label, intensity]` for each entry:
`entries[]."onyx:hasEmotionSet"[]."onyx:hasEmotion"[]["onyx:hasEmotionCategory","onyx:hasEmotionIntensity"]`
For more information, see: https://jmespath.org
"""
},
"help": {
"@id": "help",
"description": "Show additional help to know more about the possible parameters",
......@@ -44,21 +109,39 @@ API_PARAMS = {
},
"verbose": {
"@id": "verbose",
"description": ("Show all help, including the common API parameters, or "
"only plugin-related info"),
"description": "Show all properties in the result",
"aliases": ["v"],
"required": True,
"options": boolean,
"default": True
"default": False
},
"aliases": {
"@id": "aliases",
"description": "Replace JSON properties with their aliases",
"aliases": [],
"required": True,
"options": boolean,
"default": False
},
"emotionModel": {
"emotion-model": {
"@id": "emotionModel",
"aliases": ["emoModel"],
"description": """Emotion model to use in the response.
Senpy will try to convert the output to this model automatically.
Examples: `wna:liking` and `emoml:big6`.
""",
"aliases": ["emoModel", "emotionModel"],
"required": False
},
"conversion": {
"@id": "conversion",
"description": "How to show the elements that have (not) been converted",
"description": """How to show the elements that have (not) been converted.
* full: converted and original elements will appear side-by-side
* filtered: only converted elements will be shown
* nested: converted elements will be shown, and they will include a link to the original element
(using `prov:wasGeneratedBy`).
""",
"required": True,
"options": ["filtered", "nested", "full"],
"default": "full"
......@@ -68,9 +151,10 @@ API_PARAMS = {
EVAL_PARAMS = {
"algorithm": {
"aliases": ["plug", "p", "plugins", "algorithms", 'algo', 'a', 'plugin'],
"description": "Plugins to be evaluated",
"description": "Plugins to evaluate",
"required": True,
"help": "See activated plugins in /plugins"
"help": "See activated plugins in /plugins",
"processor": API_PARAMS['algorithm']['processor']
},
"dataset": {
"aliases": ["datasets", "data", "d"],
......@@ -81,18 +165,19 @@ EVAL_PARAMS = {
}