extensions.py 7.73 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
3
4
5
6
7
8
9
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
from .blueprints import nif_blueprint

from git import Repo, InvalidGitRepositoryError
from functools import partial

10
import os
11
12
import fnmatch
import inspect
13
import sys
14
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
15
import logging
16
17
import gevent
import json
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
18
19

logger = logging.getLogger(__name__)
20

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21

22
class Senpy(object):
23

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
24
    """ Default Senpy extension for Flask """
25
26
27
28
29

    def __init__(self, app=None, plugin_folder="plugins"):
        self.app = app
        base_folder = os.path.join(os.path.dirname(__file__), "plugins")

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
30
31
32
33
34
        self._search_folders = set()
        self._outdated = True

        for folder in (base_folder, plugin_folder):
            self.add_folder(folder)
35
36
37
38

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
39
40
41
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
42
43
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
44
        """
45
46
47
48
49
50
51
52
53
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
        app.register_blueprint(nif_blueprint)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
54
    def add_folder(self, folder):
55
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
56
57
58
59
60
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
            return True
        else:
61
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
62
63
            return False

64
    def analyse(self, **params):
65
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
66
        logger.debug("analysing with params: {}".format(params))
67
68
        if "algorithm" in params:
            algo = params["algorithm"]
69
        elif self.plugins:
70
            algo = self.default_plugin and self.default_plugin.name
71
72
73
74
        if not algo:
            return Error(status=404,
                         message=("No plugins found."
                                  " Please install one.").format(algo))
75
76
77
78
        if algo in self.plugins:
            if self.plugins[algo].is_activated:
                plug = self.plugins[algo]
                resp = plug.analyse(**params)
79
                resp.analysis.append(plug)
80
                logger.debug("Returning analysis result: {}".format(resp))
81
                return resp
82
83
            else:
                logger.debug("Plugin not activated: {}".format(algo))
84
85
86
                return Error(status=400,
                             message=("The algorithm '{}'"
                                      " is not activated yet").format(algo))
87
        else:
88
89
90
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
91
            return Error(status=404,
92
93
                         message="The algorithm '{}' is not valid"
                                 .format(algo))
94
95
96

    @property
    def default_plugin(self):
97
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
98
        if len(candidates) > 0:
99
            candidate = candidates.values()[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
100
            logger.debug("Default: {}".format(candidate))
101
102
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
103
            return None
104

105
    def parameters(self, algo):
106
107
108
        return getattr(self.plugins.get(algo) or self.default_plugin,
                       "params",
                       {})
109

110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
        th = gevent.spawn(plugin.activate)
        th.link_value(partial(self._set_active_plugin, plugin_name, True))
        if sync:
            th.join()
        else:
            return th

    def deactivate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
        th = gevent.spawn(plugin.deactivate)
        th.link_value(partial(self._set_active_plugin, plugin_name, False))
        if sync:
            th.join()
        else:
            return th
142
143

    def reload_plugin(self, plugin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
144
        logger.debug("Reloading {}".format(plugin))
145
146
147
148
        plug = self.plugins[plugin]
        nplug = self._load_plugin(plug.module, plug.path)
        del self.plugins[plugin]
        self.plugins[nplug.name] = nplug
149

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
150
    @staticmethod
151
152
153
    def _load_plugin(root, filename):
        logger.debug("Loading plugin: {}".format(filename))
        fpath = os.path.join(root, filename)
154
        with open(fpath, 'r') as f:
155
156
157
158
159
            info = json.load(f)
        logger.debug("Info: {}".format(info))
        sys.path.append(root)
        module = info["module"]
        name = info["name"]
160
        (fp, pathname, desc) = imp.find_module(module, [root, ])
161
        try:
162
163
164
165
166
            tmp = imp.load_module(module, fp, pathname, desc)
            sys.path.remove(root)
            candidate = None
            for _, obj in inspect.getmembers(tmp):
                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
167
168
169
                    logger.debug(("Found plugin class:"
                                  " {}@{}").format(obj, inspect.getmodule(obj))
                                 )
170
171
172
173
174
175
                    candidate = obj
                    break
            if not candidate:
                logger.debug("No valid plugin for: {}".format(filename))
                return
            module = candidate(info=info)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
176
            try:
177
                repo_path = root
178
                module._repo = Repo(repo_path)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
179
            except InvalidGitRepositoryError:
180
                module._repo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
181
        except Exception as ex:
182
183
184
            logger.debug("Exception importing {}: {}".format(filename, ex))
            return None, None
        return name, module
185
186
187

    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
188
        for search_folder in self._search_folders:
189
190
191
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
192
                    if plugin:
193
                        plugins[name] = plugin
194

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
195
        self._outdated = False
196
197
198
199
200
201
202
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
203
        """ Return the plugins registered for a given application.  """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
204
205
206
        if not hasattr(self, 'senpy_plugins') or self._outdated:
            self.senpy_plugins = self._load_plugins()
        return self.senpy_plugins
207

208
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
209
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
210

211
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
212
213
214
215
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
            logger.debug("matching {} with {}: {}".format(plug.name,
                                                          kwargs,
                                                          res))
216
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
217

218
219
220
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
221
            return {n: p for n, p in self.plugins.items() if matches(p)}
222
223

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
224
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
225
        return {p: plugin for p, plugin in self.plugins.items() if
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
226
                isinstance(plugin, SentimentPlugin)}