Multiple changes in the API, schemas and UI

Check out the CHANGELOG.md file for more information

example-plugins/README.md

 This is a collection of plugins that exemplify certain aspects of plugin development with senpy.
 
-The first series of plugins the `basic` ones.
+The first series of plugins are the `basic` ones.
 Their starting point is a classification function defined in `basic.py`.
 They all include testing and running them as a script will run all tests.
 In ascending order of customization, the plugins are:
@@ -19,5 +19,5 @@ In rest of the plugins show advanced topics:
 
 All of the plugins in this folder include a set of test cases and they are periodically tested with the latest version of senpy.
 
-Additioanlly, for an example of stand-alone plugin that can be tested and deployed with docker, take a look at: lab.cluster.gsi.dit.upm.es/senpy/plugin-example
+Additioanlly, for an example of stand-alone plugin that can be tested and deployed with docker, take a look at: lab.gsi.upm.es/senpy/plugin-example
  bbm

example-plugins/basic.py

 #!/usr/local/bin/python
-# coding: utf-8
+# -*- coding: utf-8 -*-
 
 emoticons = {
     'pos': [':)', ':]', '=)', ':D'],
@@ -7,17 +7,19 @@ emoticons = {
 }
 
 emojis = {
-    'pos': ['😁', '😂', '😃', '😄', '😆', '😅', '😄' '😍'],
-    'neg': ['😢', '😡', '😠', '😞', '😖', '😔', '😓', '😒']
+    'pos': [u'😁', u'😂', u'😃', u'😄', u'😆', u'😅', u'😄', u'😍'],
+    'neg': [u'😢', u'😡', u'😠', u'😞', u'😖', u'😔', u'😓', u'😒']
 }
 
 
 def get_polarity(text, dictionaries=[emoticons, emojis]):
     polarity = 'marl:Neutral'
+    print('Input for get_polarity', text)
     for dictionary in dictionaries:
         for label, values in dictionary.items():
             for emoticon in values:
                 if emoticon and emoticon in text:
                     polarity = label
                     break
+    print('Polarity', polarity)
     return polarity

example-plugins/basic_analyse_entry_plugin.py

 #!/usr/local/bin/python
-# coding: utf-8
+# -*- coding: utf-8 -*-
 
 from senpy import easy_test, models, plugins
 
@@ -18,13 +18,13 @@ class BasicAnalyseEntry(plugins.SentimentPlugin):
         'default': 'marl:Neutral'
     }
 
-    def analyse_entry(self, entry, params):
+    def analyse_entry(self, entry, activity):
         polarity = basic.get_polarity(entry.text)
 
         polarity = self.mappings.get(polarity, self.mappings['default'])
 
         s = models.Sentiment(marl__hasPolarity=polarity)
-        s.prov(self)
+        s.prov(activity)
         entry.sentiments.append(s)
         yield entry
 

example-plugins/basic_box_plugin.py

 #!/usr/local/bin/python
-# coding: utf-8
+# -*- coding: utf-8 -*-
 
 from senpy import easy_test, SentimentBox
 
@@ -12,15 +12,13 @@ class BasicBox(SentimentBox):
     author = '@balkian'
     version = '0.1'
 
-    mappings = {
-        'pos': 'marl:Positive',
-        'neg': 'marl:Negative',
-        'default': 'marl:Neutral'
-    }
-
-    def predict_one(self, input):
-        output = basic.get_polarity(input)
-        return self.mappings.get(output, self.mappings['default'])
+    def predict_one(self, features, **kwargs):
+        output = basic.get_polarity(features[0])
+        if output == 'pos':
+            return [1, 0, 0]
+        if output == 'neg':
+            return [0, 0, 1]
+        return [0, 1, 0]
 
     test_cases = [{
         'input': 'Hello :)',

example-plugins/basic_plugin.py

 #!/usr/local/bin/python
-# coding: utf-8
+# -*- coding: utf-8 -*-
 
-from senpy import easy_test, SentimentBox, MappingMixin
+from senpy import easy_test, SentimentBox
 
 import basic
 
 
-class Basic(MappingMixin, SentimentBox):
+class Basic(SentimentBox):
     '''Provides sentiment annotation using a lexicon'''
 
     author = '@balkian'
     version = '0.1'
 
-    mappings = {
-        'pos': 'marl:Positive',
-        'neg': 'marl:Negative',
-        'default': 'marl:Neutral'
-    }
-
-    def predict_one(self, input):
-        return basic.get_polarity(input)
+    def predict_one(self, features, **kwargs):
+        output = basic.get_polarity(features[0])
+        if output == 'pos':
+            return [1, 0, 0]
+        if output == 'neu':
+            return [0, 1, 0]
+        return [0, 0, 1]
 
     test_cases = [{
-        'input': 'Hello :)',
+        'input': u'Hello :)',
         'polarity': 'marl:Positive'
     }, {
-        'input': 'So sad :(',
+        'input': u'So sad :(',
         'polarity': 'marl:Negative'
     }, {
-        'input': 'Yay! Emojis  😁',
+        'input': u'Yay! Emojis  😁',
         'polarity': 'marl:Positive'
     }, {
-        'input': 'But no emoticons 😢',
+        'input': u'But no emoticons 😢',
         'polarity': 'marl:Negative'
     }]
 

example-plugins/configurable_plugin.py

 #!/usr/local/bin/python
-# coding: utf-8
+# -*- coding: utf-8 -*-
 
 from senpy import easy_test, models, plugins
 
@@ -16,7 +16,7 @@ class Dictionary(plugins.SentimentPlugin):
 
     mappings = {'pos': 'marl:Positive', 'neg': 'marl:Negative'}
 
-    def analyse_entry(self, entry, params):
+    def analyse_entry(self, entry, *args, **kwargs):
         polarity = basic.get_polarity(entry.text, self.dictionaries)
         if polarity in self.mappings:
             polarity = self.mappings[polarity]

senpy/plugins/example/emorand_plugin.py → example-plugins/emorand_plugin.py

@@ -6,12 +6,13 @@ from senpy.models import EmotionSet, Emotion, Entry
 
 class EmoRand(EmotionPlugin):
     '''A sample plugin that returns a random emotion annotation'''
+    name = 'emotion-random'
     author = '@balkian'
     version = '0.1'
     url = "https://github.com/gsi-upm/senpy-plugins-community"
     onyx__usesEmotionModel = "emoml:big6"
 
-    def analyse_entry(self, entry, params):
+    def analyse_entry(self, entry, activity):
         category = "emoml:big6happiness"
         number = max(-1, min(1, random.gauss(0, 0.5)))
         if number > 0:
@@ -19,7 +20,7 @@ class EmoRand(EmotionPlugin):
         emotionSet = EmotionSet()
         emotion = Emotion({"onyx:hasEmotionCategory": category})
         emotionSet.onyx__hasEmotion.append(emotion)
-        emotionSet.prov__wasGeneratedBy = self.id
+        emotionSet.prov(activity)
         entry.emotions.append(emotionSet)
         yield entry
 
@@ -27,6 +28,6 @@ class EmoRand(EmotionPlugin):
         params = dict()
         results = list()
         for i in range(100):
-            res = next(self.analyse_entry(Entry(nif__isString="Hello"), params))
+            res = next(self.analyse_entry(Entry(nif__isString="Hello"), self.activity(params)))
             res.validate()
             results.append(res.emotions[0]['onyx:hasEmotion'][0]['onyx:hasEmotionCategory'])

example-plugins/parameterized_plugin.py

 #!/usr/local/bin/python
-# coding: utf-8
+# -*- coding: utf-8 -*-
 
 from senpy import easy_test, models, plugins
 
@@ -25,7 +25,8 @@ class ParameterizedDictionary(plugins.SentimentPlugin):
         }
     }
 
-    def analyse_entry(self, entry, params):
+    def analyse_entry(self, entry, activity):
+        params = activity.params
         positive_words = params['positive-words'].split(',')
         negative_words = params['negative-words'].split(',')
         dictionary = {
@@ -35,7 +36,7 @@ class ParameterizedDictionary(plugins.SentimentPlugin):
         polarity = basic.get_polarity(entry.text, [dictionary])
 
         s = models.Sentiment(marl__hasPolarity=polarity)
-        s.prov(self)
+        s.prov(activity)
         entry.sentiments.append(s)
         yield entry
 

senpy/plugins/example/rand_plugin.py → example-plugins/rand_plugin.py

@@ -2,15 +2,16 @@ import random
 from senpy import SentimentPlugin, Sentiment, Entry
 
 
-class Rand(SentimentPlugin):
+class RandSent(SentimentPlugin):
     '''A sample plugin that returns a random sentiment annotation'''
+    name = 'sentiment-random'
     author = "@balkian"
     version = '0.1'
     url = "https://github.com/gsi-upm/senpy-plugins-community"
     marl__maxPolarityValue = '1'
     marl__minPolarityValue = "-1"
 
-    def analyse_entry(self, entry, params):
+    def analyse_entry(self, entry, activity):
         polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
         polarity = "marl:Neutral"
         if polarity_value > 0:
@@ -19,7 +20,7 @@ class Rand(SentimentPlugin):
             polarity = "marl:Negative"
         sentiment = Sentiment(marl__hasPolarity=polarity,
                               marl__polarityValue=polarity_value)
-        sentiment.prov(self)
+        sentiment.prov(activity)
         entry.sentiments.append(sentiment)
         yield entry
 
@@ -28,8 +29,9 @@ class Rand(SentimentPlugin):
         params = dict()
         results = list()
         for i in range(50):
+            activity = self.activity(params)
             res = next(self.analyse_entry(Entry(nif__isString="Hello"),
-                                          params))
+                                          activity))
             res.validate()
             results.append(res.sentiments[0]['marl:hasPolarity'])
         assert 'marl:Positive' in results

example-plugins/sklearn/pipeline_plugin.py

-from senpy import SentimentBox, MappingMixin, easy_test
+from senpy import SentimentBox, easy_test
 
 from mypipeline import pipeline
 
 
-class PipelineSentiment(MappingMixin, SentimentBox):
-    '''
-    This is a pipeline plugin that wraps a classifier defined in another module
-    (mypipeline).
-    '''
+class PipelineSentiment(SentimentBox):
+    '''This is a pipeline plugin that wraps a classifier defined in another module
+(mypipeline).'''
     author = '@balkian'
     version = 0.1
     maxPolarityValue = 1
     minPolarityValue = -1
 
-    mappings = {
-        1: 'marl:Positive',
-        -1: 'marl:Negative'
-    }
-
-    def predict_one(self, input):
-        return pipeline.predict([input, ])[0]
+    def predict_one(self, features, **kwargs):
+        if pipeline.predict(features) > 0:
+            return [1, 0, 0]
+        return [0, 0, 1]
 
     test_cases = [
         {

extra-requirements.txt

-gsitk
+gsitk>0.1.9.1

k8s/senpy-deployment.yaml

@@ -15,8 +15,6 @@ spec:
       - name: senpy-latest
         image: $IMAGEWTAG
         imagePullPolicy: Always
-        args:
-          - "--default-plugins"
         resources:
           limits:
             memory: "512Mi"

k8s/senpy-ingress.yaml

@@ -12,3 +12,10 @@ spec:
         backend:
           serviceName: senpy-latest
           servicePort: 5000
+  - host: latest.senpy.gsi.upm.es
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: senpy-latest
+          servicePort: 5000

requirements.txt

@@ -11,5 +11,6 @@ rdflib
 rdflib-jsonld
 numpy
 scipy
-scikit-learn
+scikit-learn>=0.20
 responses
+jmespath

senpy/__main__.py

@@ -40,8 +40,14 @@ def main():
         '-l',
         metavar='logging_level',
         type=str,
-        default="WARN",
+        default="INFO",
         help='Logging level')
+    parser.add_argument(
+        '--log-format',
+        metavar='log_format',
+        type=str,
+        default='%(asctime)s %(levelname)-10s %(name)-30s \t %(message)s',
+        help='Logging format')
     parser.add_argument(
         '--debug',
         '-d',
@@ -49,10 +55,10 @@ def main():
         default=False,
         help='Run the application in debug mode')
     parser.add_argument(
-        '--default-plugins',
+        '--no-default-plugins',
         action='store_true',
         default=False,
-        help='Load the default plugins')
+        help='Do not load the default plugins')
     parser.add_argument(
         '--host',
         type=str,
@@ -68,7 +74,7 @@ def main():
         '--plugins-folder',
         '-f',
         type=str,
-        default='.',
+        action='append',
         help='Where to look for plugins.')
     parser.add_argument(
         '--only-install',
@@ -100,10 +106,10 @@ def main():
         default=None,
         help='Where to look for data. It be set with the SENPY_DATA environment variable as well.')
     parser.add_argument(
-        '--threaded',
-        action='store_false',
-        default=True,
-        help='Run a threaded server')
+        '--no-threaded',
+        action='store_true',
+        default=False,
+        help='Run a single-threaded server')
     parser.add_argument(
         '--no-deps',
         '-n',
@@ -123,30 +129,42 @@ def main():
         default=False,
         help='Do not exit if some plugins fail to activate')
     args = parser.parse_args()
+    print('Senpy version {}'.format(senpy.__version__))
+    print(sys.version)
     if args.version:
-        print('Senpy version {}'.format(senpy.__version__))
-        print(sys.version)
         exit(1)
     rl = logging.getLogger()
     rl.setLevel(getattr(logging, args.level))
+    logger_handler = rl.handlers[0]
+
+    # First, generic formatter:
+    logger_handler.setFormatter(logging.Formatter(args.log_format))
+
     app = Flask(__name__)
     app.debug = args.debug
-    sp = Senpy(app, args.plugins_folder,
-               default_plugins=args.default_plugins,
+    sp = Senpy(app,
+               plugin_folder=None,
+               default_plugins=not args.no_default_plugins,
                data_folder=args.data_folder)
+    folders = list(args.plugins_folder) if args.plugins_folder else []
+    if not folders:
+        folders.append(".")
+    for p in folders:
+        sp.add_folder(p)
+
+    plugins = sp.plugins(plugin_type=None, is_activated=False)
+    maxname = max(len(x.name) for x in plugins)
+    maxversion = max(len(str(x.version)) for x in plugins)
+    print('Found {} plugins:'.format(len(plugins)))
+    for plugin in plugins:
+        import inspect
+        fpath = inspect.getfile(plugin.__class__)
+        print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
+                                                                  plugin.version,
+                                                                  fpath,
+                                                                  maxname=maxname,
+                                                                  maxversion=maxversion))
     if args.only_list:
-        plugins = sp.plugins()
-        maxname = max(len(x.name) for x in plugins)
-        maxversion = max(len(x.version) for x in plugins)
-        print('Found {} plugins:'.format(len(plugins)))
-        for plugin in plugins:
-            import inspect
-            fpath = inspect.getfile(plugin.__class__)
-            print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
-                                                                      plugin.version,
-                                                                      fpath,
-                                                                      maxname=maxname,
-                                                                      maxversion=maxversion))
         return
     if not args.no_deps:
         sp.install_deps()
@@ -160,10 +178,13 @@ def main():
     print('Senpy version {}'.format(senpy.__version__))
     print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
                                                             args.port))
-    app.run(args.host,
-            args.port,
-            threaded=args.threaded,
-            debug=app.debug)
+    try:
+        app.run(args.host,
+                args.port,
+                threaded=not args.no_threaded,
+                debug=app.debug)
+    except KeyboardInterrupt:
+        print('Bye!')
     sp.deactivate_all()
 
 

senpy/api.py

@@ -5,24 +5,31 @@ logger = logging.getLogger(__name__)
 
 boolean = [True, False]
 
+processors = {
+    'string_to_tuple': lambda p: p if isinstance(p, (tuple, list)) else tuple(p.split(','))
+}
+
 API_PARAMS = {
     "algorithm": {
         "aliases": ["algorithms", "a", "algo"],
         "required": True,
         "default": 'default',
+        "processor": 'string_to_tuple',
         "description": ("Algorithms that will be used to process the request."
                         "It may be a list of comma-separated names."),
     },
     "expanded-jsonld": {
         "@id": "expanded-jsonld",
-        "aliases": ["expanded"],
+        "description": "use JSON-LD expansion to get full URIs",
+        "aliases": ["expanded", "expanded_jsonld"],
         "options": boolean,
         "required": True,
         "default": False
     },
-    "with_parameters": {
+    "with-parameters": {
         "aliases": ['withparameters',
-                    'with-parameters'],
+                    'with_parameters'],
+        "description": "include initial parameters in the response",
         "options": boolean,
         "default": False,
         "required": True
@@ -31,9 +38,67 @@ API_PARAMS = {
         "@id": "outformat",
         "aliases": ["o"],
         "default": "json-ld",
+        "description": """The data can be semantically formatted (JSON-LD, turtle or n-triples),
+given as a list of comma-separated fields (see the fields option) or constructed from a Jinja2
+template (see the template option).""",
         "required": True,
         "options": ["json-ld", "turtle", "ntriples"],
     },
+    "template": {
+        "@id": "template",
+        "required": False,
+        "description": """Jinja2 template for the result. The input data for the template will
+be the results as a dictionary.
+For example:
+
+Consider the results before templating:
+
+```
+[{
+    "@type": "entry",
+    "onyx:hasEmotionSet": [],
+    "nif:isString": "testing the template",
+    "marl:hasOpinion": [
+        {
+            "@type": "sentiment",
+            "marl:hasPolarity": "marl:Positive"
+        }
+    ]
+}]
+```
+
+
+And the template:
+
+```
+{% for entry in entries %}
+{{ entry["nif:isString"] | upper }},{{entry.sentiments[0]["marl:hasPolarity"].split(":")[1]}}
+{% endfor %}
+```
+
+The final result would be:
+
+```
+TESTING THE TEMPLATE,Positive
+```
+"""
+
+    },
+    "fields": {
+        "@id": "fields",
+        "required": False,
+        "description": """A jmespath selector, that can be used to extract a new dictionary, array or value
+from the results.
+jmespath is a powerful query language for json and/or dictionaries.
+It allows you to change the structure (and data) of your objects through queries.
+
+e.g., the following expression gets a list of `[emotion label, intensity]` for each entry:
+`entries[]."onyx:hasEmotionSet"[]."onyx:hasEmotion"[]["onyx:hasEmotionCategory","onyx:hasEmotionIntensity"]`
+
+For more information, see: https://jmespath.org
+
+"""
+    },
     "help": {
         "@id": "help",
         "description": "Show additional help to know more about the possible parameters",
@@ -44,21 +109,39 @@ API_PARAMS = {
     },
     "verbose": {
         "@id": "verbose",
-        "description": ("Show all help, including the common API parameters, or "
-                        "only plugin-related info"),
+        "description": "Show all properties in the result",
         "aliases": ["v"],
         "required": True,
         "options": boolean,
-        "default": True
+        "default": False
+    },
+    "aliases": {
+        "@id": "aliases",
+        "description": "Replace JSON properties with their aliases",
+        "aliases": [],
+        "required": True,
+        "options": boolean,
+        "default": False
     },
-    "emotionModel": {
+    "emotion-model": {
         "@id": "emotionModel",
-        "aliases": ["emoModel"],
+        "description": """Emotion model to use in the response.
+Senpy will try to convert the output to this model automatically.
+
+Examples: `wna:liking` and `emoml:big6`.
+        """,
+        "aliases": ["emoModel", "emotionModel"],
         "required": False
     },
     "conversion": {
         "@id": "conversion",
-        "description": "How to show the elements that have (not) been converted",
+        "description": """How to show the elements that have (not) been converted.
+
+* full: converted and original elements will appear side-by-side
+* filtered: only converted elements will be shown
+* nested: converted elements will be shown, and they will include a link to the original element
+(using `prov:wasGeneratedBy`).
+""",
         "required": True,
         "options": ["filtered", "nested", "full"],
         "default": "full"
@@ -68,9 +151,10 @@ API_PARAMS = {
 EVAL_PARAMS = {
     "algorithm": {
         "aliases": ["plug", "p", "plugins", "algorithms", 'algo', 'a', 'plugin'],
-        "description": "Plugins to be evaluated",
+        "description": "Plugins to evaluate",
         "required": True,
-        "help": "See activated plugins in /plugins"
+        "help": "See activated plugins in /plugins",
+        "processor": API_PARAMS['algorithm']['processor']
     },
     "dataset": {
         "aliases": ["datasets", "data", "d"],
@@ -81,18 +165,19 @@ EVAL_PARAMS = {
 }