Added tests for all "discoverable" plugins

Closes #39

senpy/extensions.py

@@ -15,25 +15,12 @@ from threading import Thread
 
 import os
 import copy
-import fnmatch
-import inspect
-import sys
-import importlib
 import logging
 import traceback
-import yaml
-import subprocess
 
 logger = logging.getLogger(__name__)
 
 
-def log_subprocess_output(process):
-    for line in iter(process.stdout.readline, b''):
-        logger.info('%r', line)
-    for line in iter(process.stderr.readline, b''):
-        logger.error('%r', line)
-
-
 class Senpy(object):
     """ Default Senpy extension for Flask """
 
@@ -330,84 +317,6 @@ class Senpy(object):
             th.start()
             return th
 
-    @classmethod
-    def validate_info(cls, info):
-        return all(x in info for x in ('name', 'module', 'description', 'version'))
-
-    def install_deps(self):
-        for i in self.plugins.values():
-            self._install_deps(i)
-
-    @classmethod
-    def _install_deps(cls, info=None):
-        requirements = info.get('requirements', [])
-        if requirements:
-            pip_args = ['pip']
-            pip_args.append('install')
-            pip_args.append('--use-wheel')
-            for req in requirements:
-                pip_args.append(req)
-            logger.info('Installing requirements: ' + str(requirements))
-            process = subprocess.Popen(pip_args,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE)
-            log_subprocess_output(process)
-            exitcode = process.wait()
-            if exitcode != 0:
-                raise Error("Dependencies not properly installed")
-
-    @classmethod
-    def _load_module(cls, name, root):
-        sys.path.append(root)
-        tmp = importlib.import_module(name)
-        sys.path.remove(root)
-        return tmp
-
-    @classmethod
-    def _load_plugin_from_info(cls, info, root):
-        if not cls.validate_info(info):
-            logger.warn('The module info is not valid.\n\t{}'.format(info))
-            return None, None
-        module = info["module"]
-        name = info["name"]
-
-        cls._install_deps(info)
-        tmp = cls._load_module(module, root)
-
-        candidate = None
-        for _, obj in inspect.getmembers(tmp):
-            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
-                logger.debug(("Found plugin class:"
-                              " {}@{}").format(obj, inspect.getmodule(obj)))
-                candidate = obj
-                break
-        if not candidate:
-            logger.debug("No valid plugin for: {}".format(module))
-            return
-        module = candidate(info=info)
-        return name, module
-
-    @classmethod
-    def _load_plugin(cls, root, filename):
-        fpath = os.path.join(root, filename)
-        logger.debug("Loading plugin: {}".format(fpath))
-        with open(fpath, 'r') as f:
-            info = yaml.load(f)
-        logger.debug("Info: {}".format(info))
-        return cls._load_plugin_from_info(info, root)
-
-    def _load_plugins(self):
-        plugins = {}
-        for search_folder in self._search_folders:
-            for root, dirnames, filenames in os.walk(search_folder):
-                for filename in fnmatch.filter(filenames, '*.senpy'):
-                    name, plugin = self._load_plugin(root, filename)
-                    if plugin and name:
-                        plugins[name] = plugin
-
-        self._outdated = False
-        return plugins
-
     def teardown(self, exception):
         pass
 
@@ -415,7 +324,8 @@ class Senpy(object):
     def plugins(self):
         """ Return the plugins registered for a given application.  """
         if self._outdated:
-            self._plugin_list = self._load_plugins()
+            self._plugin_list = plugins.load_plugins(self._search_folders)
+            self._outdated = False
         return self._plugin_list
 
     def filter_plugins(self, **kwargs):

senpy/models.py

@@ -223,12 +223,6 @@ class BaseModel(SenpyMixin, dict):
         key = key.replace("__", ":", 1)
         return key
 
-    def __getitem__(self, key):
-        return dict.__getitem__(self, self._get_key(key))
-
-    def __setitem__(self, key, value):
-        dict.__setitem__(self, self._get_key(key), value)
-
     def __delitem__(self, key):
         dict.__delitem__(self, key)
 

senpy/plugins/__init__.py

 from future import standard_library
 standard_library.install_aliases()
 
-import inspect
 import os.path
 import os
 import pickle
 import logging
 import tempfile
 import copy
+
+import fnmatch
+import inspect
+import sys
+import subprocess
+import importlib
+import yaml
+
 from .. import models
 from ..api import API_PARAMS
 
@@ -69,6 +76,8 @@ class Plugin(models.Plugin):
             if not isinstance(exp, list):
                 exp = [exp]
             check_template(res, exp)
+            for r in res:
+                r.validate()
 
 
 SenpyPlugin = Plugin
@@ -193,3 +202,84 @@ def pfilter(plugins, **kwargs):
     if kwargs:
         candidates = filter(matches, candidates)
     return {p.name: p for p in candidates}
+
+
+def validate_info(info):
+    return all(x in info for x in ('name', 'module', 'description', 'version'))
+
+
+def load_module(name, root):
+    sys.path.append(root)
+    tmp = importlib.import_module(name)
+    sys.path.remove(root)
+    return tmp
+
+
+def log_subprocess_output(process):
+    for line in iter(process.stdout.readline, b''):
+        logger.info('%r', line)
+    for line in iter(process.stderr.readline, b''):
+        logger.error('%r', line)
+
+
+def install_deps(*plugins):
+    for info in plugins:
+        requirements = info.get('requirements', [])
+        if requirements:
+            pip_args = ['pip']
+            pip_args.append('install')
+            pip_args.append('--use-wheel')
+            for req in requirements:
+                pip_args.append(req)
+            logger.info('Installing requirements: ' + str(requirements))
+            process = subprocess.Popen(pip_args,
+                                       stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE)
+            log_subprocess_output(process)
+            exitcode = process.wait()
+            if exitcode != 0:
+                raise models.Error("Dependencies not properly installed")
+
+
+def load_plugin_from_info(info, root, validator=validate_info):
+    if not validator(info):
+        logger.warn('The module info is not valid.\n\t{}'.format(info))
+        return None, None
+    module = info["module"]
+    name = info["name"]
+
+    install_deps(info)
+    tmp = load_module(module, root)
+
+    candidate = None
+    for _, obj in inspect.getmembers(tmp):
+        if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
+            logger.debug(("Found plugin class:"
+                          " {}@{}").format(obj, inspect.getmodule(obj)))
+            candidate = obj
+            break
+    if not candidate:
+        logger.debug("No valid plugin for: {}".format(module))
+        return
+    module = candidate(info=info)
+    return name, module
+
+
+def load_plugin(root, filename):
+    fpath = os.path.join(root, filename)
+    logger.debug("Loading plugin: {}".format(fpath))
+    with open(fpath, 'r') as f:
+        info = yaml.load(f)
+    logger.debug("Info: {}".format(info))
+    return load_plugin_from_info(info, root)
+
+
+def load_plugins(folders, loader=load_plugin):
+    plugins = {}
+    for search_folder in folders:
+        for root, dirnames, filenames in os.walk(search_folder):
+            for filename in fnmatch.filter(filenames, '*.senpy'):
+                name, plugin = loader(root, filename)
+                if plugin and name:
+                    plugins[name] = plugin
+    return plugins

senpy/plugins/example/emoRand/emoRand.py

@@ -21,6 +21,6 @@ class RmoRandPlugin(EmotionPlugin):
         params = dict()
         results = list()
         for i in range(100):
-            res = next(self.analyse_entry(Entry(text="Hello"), params))
+            res = next(self.analyse_entry(Entry(nif__isString="Hello"), params))
             res.validate()
             results.append(res.emotions[0]['onyx:hasEmotion'][0]['onyx:hasEmotionCategory'])

senpy/plugins/example/rand/rand.py

@@ -27,7 +27,7 @@ class RandPlugin(SentimentPlugin):
         params = dict()
         results = list()
         for i in range(100):
-            res = next(self.analyse_entry(Entry(text="Hello"), params))
+            res = next(self.analyse_entry(Entry(nif__isString="Hello"), params))
             res.validate()
             results.append(res.sentiments[0]['marl:hasPolarity'])
         assert 'marl:Positive' in results

senpy/plugins/sentiment/sentiment140/sentiment140.py

@@ -12,7 +12,7 @@ class Sentiment140Plugin(SentimentPlugin):
                             json.dumps({
                                 "language": lang,
                                 "data": [{
-                                    "text": entry.text
+                                    "text": entry.nif__isString
                                 }]
                             }))
         p = params.get("prefix", None)
@@ -38,11 +38,11 @@ class Sentiment140Plugin(SentimentPlugin):
     test_cases = [
         {
             'entry': {
-                'text': 'I love Titanic'
+                'nif:isString': 'I love Titanic'
             },
             'params': {},
             'expected': {
-                "text": "I love Titanic",
+                "nif:isString": "I love Titanic",
                 'sentiments': [
                     {
                         'marl:hasPolarity': 'marl:Positive',

senpy/schemas/context.jsonld

@@ -20,6 +20,9 @@
       "@id": "me:hasSuggestions",
       "@container": "@set"
     },
+    "onyx:hasEmotion": {
+      "@container": "@set"
+    },
     "emotions": {
       "@id": "onyx:hasEmotionSet",
       "@container": "@set"

tests/test_extensions.py

@@ -10,6 +10,7 @@ except ImportError:
 
 from functools import partial
 from senpy.extensions import Senpy
+from senpy import plugins
 from senpy.models import Error, Results, Entry, EmotionSet, Emotion, Plugin
 from flask import Flask
 from unittest import TestCase
@@ -18,7 +19,7 @@ from unittest import TestCase
 class ExtensionsTest(TestCase):
     def setUp(self):
         self.app = Flask('test_extensions')
-        self.dir = os.path.join(os.path.dirname(__file__))
+        self.dir = os.path.dirname(__file__)
         self.senpy = Senpy(plugin_folder=self.dir,
                            app=self.app,
                            default_plugins=False)
@@ -38,8 +39,8 @@ class ExtensionsTest(TestCase):
         print(self.senpy.plugins)
         assert "Dummy" in self.senpy.plugins
 
-    def test_enabling(self):
-        """ Enabling a plugin """
+    def test_installing(self):
+        """ Installing a plugin """
         info = {
             'name': 'TestPip',
             'module': 'dummy',
@@ -48,14 +49,13 @@ class ExtensionsTest(TestCase):
             'version': 0
         }
         root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
-        name, module = self.senpy._load_plugin_from_info(info, root=root)
+        name, module = plugins.load_plugin_from_info(info, root=root)
         assert name == 'TestPip'
         assert module
         import noop
         dir(noop)
-        self.senpy.install_deps()
 
-    def test_installing(self):
+    def test_enabling(self):
         """ Enabling a plugin """
         self.senpy.activate_all(sync=True)
         assert len(self.senpy.plugins) >= 3
@@ -72,7 +72,7 @@ class ExtensionsTest(TestCase):
         }
         root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
         with self.assertRaises(Error):
-            name, module = self.senpy._load_plugin_from_info(info, root=root)
+            name, module = plugins.load_plugin_from_info(info, root=root)
 
     def test_disabling(self):
         """ Disabling a plugin """
@@ -173,7 +173,7 @@ class ExtensionsTest(TestCase):
             'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
         })
         eSet1 = EmotionSet()
-        eSet1.prov__wasGeneratedBy = plugin['id']
+        eSet1.prov__wasGeneratedBy = plugin['@id']
         eSet1['onyx:hasEmotion'].append(Emotion({
             'emoml:arousal': 1,
             'emoml:potency': 0,

tests/test_plugins.py

@@ -7,11 +7,11 @@ import tempfile
 
 from unittest import TestCase
 from senpy.models import Results, Entry, EmotionSet, Emotion
-from senpy.plugins import SentimentPlugin, ShelfMixin
+from senpy import plugins
 from senpy.plugins.conversion.emotion.centroids import CentroidConversion
 
 
-class ShelfDummyPlugin(SentimentPlugin, ShelfMixin):
+class ShelfDummyPlugin(plugins.SentimentPlugin, plugins.ShelfMixin):
     def activate(self, *args, **kwargs):
         if 'counter' not in self.sh:
             self.sh['counter'] = 0
@@ -202,3 +202,22 @@ class PluginsTest(TestCase):
         e["Y-dimension"] = 0.3
         res = c._backwards_conversion(e)
         assert res["onyx:hasEmotionCategory"] == "c2"
+
+
+def make_mini_test(plugin):
+    def mini_test(self):
+        plugin.test()
+    return mini_test
+
+
+def add_tests():
+    root = os.path.dirname(__file__)
+    plugs = plugins.load_plugins(os.path.join(root, ".."))
+    for k, v in plugs.items():
+        t_method = make_mini_test(v)
+        t_method.__name__ = 'test_plugin_{}'.format(k)
+        setattr(PluginsTest, t_method.__name__, t_method)
+        del t_method
+
+
+add_tests()