extensions.py 8.23 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
3
4
5
6
import gevent
from gevent import monkey
monkey.patch_all()

7
8
9
10
11
12
13
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
from .blueprints import nif_blueprint

from git import Repo, InvalidGitRepositoryError
from functools import partial

14
import os
15
16
import fnmatch
import inspect
17
import sys
18
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
19
import logging
20
import traceback
21
22
import gevent
import json
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23
24

logger = logging.getLogger(__name__)
25

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
26

27
class Senpy(object):
28

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29
    """ Default Senpy extension for Flask """
30

31
    def __init__(self, app=None, plugin_folder="plugins", default_plugins=False):
32
33
        self.app = app

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
34
35
36
        self._search_folders = set()
        self._outdated = True

37
        self.add_folder(plugin_folder)
38
        if default_plugins:
39
40
            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
            self.add_folder(base_folder)
41
42
43
44

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
45
46
47
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
48
49
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
50
        """
51
52
53
54
55
56
57
58
59
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
        app.register_blueprint(nif_blueprint)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
60
    def add_folder(self, folder):
61
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
62
63
64
65
66
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
            return True
        else:
67
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
68
69
            return False

70
    def analyse(self, **params):
71
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
72
        logger.debug("analysing with params: {}".format(params))
73
74
        if "algorithm" in params:
            algo = params["algorithm"]
75
        elif self.plugins:
76
            algo = self.default_plugin and self.default_plugin.name
77
78
79
80
        if not algo:
            return Error(status=404,
                         message=("No plugins found."
                                  " Please install one.").format(algo))
81
82
83
84
        if algo in self.plugins:
            if self.plugins[algo].is_activated:
                plug = self.plugins[algo]
                resp = plug.analyse(**params)
85
                resp.analysis.append(plug)
86
                logger.debug("Returning analysis result: {}".format(resp))
87
                return resp
88
89
            else:
                logger.debug("Plugin not activated: {}".format(algo))
90
91
92
                return Error(status=400,
                             message=("The algorithm '{}'"
                                      " is not activated yet").format(algo))
93
        else:
94
95
96
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
97
            return Error(status=404,
98
99
                         message="The algorithm '{}' is not valid"
                                 .format(algo))
100
101
102

    @property
    def default_plugin(self):
103
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
104
        if len(candidates) > 0:
105
            candidate = candidates.values()[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
106
            logger.debug("Default: {}".format(candidate))
107
108
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109
            return None
110

111
    def parameters(self, algo):
112
113
114
        return getattr(self.plugins.get(algo) or self.default_plugin,
                       "params",
                       {})
115

116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
133
134
135
136
137
138
        def act():
            try:
                plugin.activate()
            except Exception as ex:
                logger.error("Error activating plugin {}: {}".format(plugin.name,
                                                                     ex))
139
                logger.error("Trace: {}".format(traceback.format_exc()))
140
        th = gevent.spawn(act)
141
142
143
144
145
146
147
148
149
150
151
152
153
154
        th.link_value(partial(self._set_active_plugin, plugin_name, True))
        if sync:
            th.join()
        else:
            return th

    def deactivate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
        th = gevent.spawn(plugin.deactivate)
        th.link_value(partial(self._set_active_plugin, plugin_name, False))
        if sync:
            th.join()
        else:
            return th
155
156

    def reload_plugin(self, plugin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
157
        logger.debug("Reloading {}".format(plugin))
158
159
160
161
        plug = self.plugins[plugin]
        nplug = self._load_plugin(plug.module, plug.path)
        del self.plugins[plugin]
        self.plugins[nplug.name] = nplug
162

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
163
    @staticmethod
164
165
166
    def _load_plugin(root, filename):
        logger.debug("Loading plugin: {}".format(filename))
        fpath = os.path.join(root, filename)
167
        with open(fpath, 'r') as f:
168
169
170
171
172
            info = json.load(f)
        logger.debug("Info: {}".format(info))
        sys.path.append(root)
        module = info["module"]
        name = info["name"]
173
        (fp, pathname, desc) = imp.find_module(module, [root, ])
174
        try:
175
176
177
178
179
            tmp = imp.load_module(module, fp, pathname, desc)
            sys.path.remove(root)
            candidate = None
            for _, obj in inspect.getmembers(tmp):
                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
180
181
182
                    logger.debug(("Found plugin class:"
                                  " {}@{}").format(obj, inspect.getmodule(obj))
                                 )
183
184
185
186
187
188
                    candidate = obj
                    break
            if not candidate:
                logger.debug("No valid plugin for: {}".format(filename))
                return
            module = candidate(info=info)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
189
            try:
190
                repo_path = root
191
                module._repo = Repo(repo_path)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
192
            except InvalidGitRepositoryError:
193
                module._repo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
194
        except Exception as ex:
195
196
            logger.error("Exception importing {}: {}".format(filename, ex))
            logger.error("Trace: {}".format(traceback.format_exc()))
197
198
            return None, None
        return name, module
199
200
201

    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
202
        for search_folder in self._search_folders:
203
204
205
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
206
                    if plugin:
207
                        plugins[name] = plugin
208

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
209
        self._outdated = False
210
211
212
213
214
215
216
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
217
        """ Return the plugins registered for a given application.  """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
218
219
220
        if not hasattr(self, 'senpy_plugins') or self._outdated:
            self.senpy_plugins = self._load_plugins()
        return self.senpy_plugins
221

222
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
223
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
224

225
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
226
227
228
229
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
            logger.debug("matching {} with {}: {}".format(plug.name,
                                                          kwargs,
                                                          res))
230
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
231

232
233
234
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
235
            return {n: p for n, p in self.plugins.items() if matches(p)}
236
237

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
238
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
239
        return {p: plugin for p, plugin in self.plugins.items() if
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
240
                isinstance(plugin, SentimentPlugin)}