extensions.py 10.8 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
4
from future import standard_library
standard_library.install_aliases()
5
6
7
8
import gevent
from gevent import monkey
monkey.patch_all()

9
10
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
11
from .blueprints import api_blueprint, demo_blueprint
12
from .api import API_PARAMS, NIF_PARAMS, parse_params
13
14
15
16

from git import Repo, InvalidGitRepositoryError
from functools import partial

17
import os
18
19
import fnmatch
import inspect
20
import sys
21
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
22
import logging
23
import traceback
24
import gevent
25
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
26
import pip
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
27
28

logger = logging.getLogger(__name__)
29

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
30

31
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
32
    """ Default Senpy extension for Flask """
33

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
34
35
36
37
    def __init__(self,
                 app=None,
                 plugin_folder="plugins",
                 default_plugins=False):
38
39
        self.app = app

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
40
        self._search_folders = set()
41
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
42
43
        self._outdated = True

44
        self.add_folder(plugin_folder)
45
        if default_plugins:
46
47
            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
            self.add_folder(base_folder)
48
49
50
51

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
52
53
54
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
55
56
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
57
        """
58
59
60
61
62
63
64
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
65
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
66
        app.register_blueprint(demo_blueprint, url_prefix="/")
67

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
68
    def add_folder(self, folder):
69
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
70
71
72
73
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
74
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75

76
    def analyse(self, **params):
77
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
78
        logger.debug("analysing with params: {}".format(params))
79
80
81
        api_params = parse_params(params, spec=API_PARAMS)
        if "algorithm" in api_params and api_params["algorithm"]:
            algo = api_params["algorithm"]
82
        elif self.plugins:
83
            algo = self.default_plugin and self.default_plugin.name
84
        if not algo:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
85
86
87
88
            raise Error(
                status=404,
                message=("No plugins found."
                         " Please install one.").format(algo))
89
        if algo not in self.plugins:
90
91
92
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
93
94
95
            raise Error(
                status=404,
                message="The algorithm '{}' is not valid".format(algo))
96
97
98

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
99
100
101
102
            raise Error(
                status=400,
                message=("The algorithm '{}'"
                         " is not activated yet").format(algo))
103
        plug = self.plugins[algo]
104
105
106
107
        nif_params = parse_params(params, spec=NIF_PARAMS)
        extra_params = plug.get('extra_params', {})
        specific_params = parse_params(params, spec=extra_params)
        nif_params.update(specific_params)
108
        try:
109
            resp = plug.analyse(**nif_params)
110
111
112
113
114
            resp.analysis.append(plug)
            logger.debug("Returning analysis result: {}".format(resp))
        except Exception as ex:
            resp = Error(message=str(ex), status=500)
        return resp
115
116
117

    @property
    def default_plugin(self):
118
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
119
        if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
120
            candidate = list(candidates.values())[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
121
            logger.debug("Default: {}".format(candidate.name))
122
123
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
124
            return None
125

126
    def parameters(self, algo):
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
127
128
        return getattr(
            self.plugins.get(algo) or self.default_plugin, "extra_params", {})
129

130
131
132
133
134
135
136
137
138
139
140
141
142
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
143
144
145
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
146
147
148
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
149
150
151
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
152
153
154
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

155
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
156

157
158
159
        def act():
            try:
                plugin.activate()
160
                logger.info("Plugin activated: {}".format(plugin.name))
161
            except Exception as ex:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
162
163
                logger.error("Error activating plugin {}: {}".format(
                    plugin.name, ex))
164
                logger.error("Trace: {}".format(traceback.format_exc()))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
165

166
        th = gevent.spawn(act)
167
168
169
170
171
172
173
        th.link_value(partial(self._set_active_plugin, plugin_name, True))
        if sync:
            th.join()
        else:
            return th

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
174
175
176
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
177
178
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
179
180
181
182
183
184

        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
185
186
                logger.error("Error deactivating plugin {}: {}".format(
                    plugin.name, ex))
187
188
189
                logger.error("Trace: {}".format(traceback.format_exc()))

        th = gevent.spawn(deact)
190
191
192
193
194
        th.link_value(partial(self._set_active_plugin, plugin_name, False))
        if sync:
            th.join()
        else:
            return th
195

196
197
198
199
200
201
202
203
204
205
    def reload_plugin(self, name):
        logger.debug("Reloading {}".format(name))
        plugin = self.plugins[name]
        try:
            del self.plugins[name]
            nplug = self._load_plugin(plugin.module, plugin.path)
            self.plugins[nplug.name] = nplug
        except Exception as ex:
            logger.error('Error reloading {}: {}'.format(name, ex))
            self.plugins[name] = plugin
206

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
207
208
209
210
    @classmethod
    def validate_info(cls, info):
        return all(x in info for x in ('name', 'module', 'version'))

211
212
213
214
215
216
217
218
219
220
221
    def install_deps(self):
        for i in self.plugins.values():
            self._install_deps(i._info)

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
            pip_args = []
            pip_args.append('install')
            for req in requirements:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
222
                pip_args.append(req)
223
            logger.info('Installing requirements: ' + str(requirements))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
224
            pip.main(pip_args)
225

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
226
227
228
229
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
230
            return None, None
231
232
        module = info["module"]
        name = info["name"]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
233
234
        requirements = info.get("requirements", [])
        sys.path.append(root)
235
        (fp, pathname, desc) = imp.find_module(module, [root, ])
236
        try:
237
            cls._install_deps(info)
238
239
240
241
242
            tmp = imp.load_module(module, fp, pathname, desc)
            sys.path.remove(root)
            candidate = None
            for _, obj in inspect.getmembers(tmp):
                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
243
                    logger.debug(("Found plugin class:"
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
244
245
                                  " {}@{}").format(obj, inspect.getmodule(
                                      obj)))
246
247
248
                    candidate = obj
                    break
            if not candidate:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
249
                logger.debug("No valid plugin for: {}".format(module))
250
251
                return
            module = candidate(info=info)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
252
253
254
            repo_path = root
            module._repo = Repo(repo_path)
        except InvalidGitRepositoryError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
255
256
            logger.debug("The plugin {} is not in a Git repository".format(
                module))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
257
            module._repo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
258
        except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
259
            logger.error("Exception importing {}: {}".format(module, ex))
260
            logger.error("Trace: {}".format(traceback.format_exc()))
261
262
            return None, None
        return name, module
263

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
264
265
266
267
268
269
270
271
272
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)

273
274
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
275
        for search_folder in self._search_folders:
276
277
278
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
279
                    if plugin and name not in self._plugin_list:
280
                        plugins[name] = plugin
281

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
282
        self._outdated = False
283
284
285
286
287
288
289
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
290
        """ Return the plugins registered for a given application.  """
291
292
293
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
294

295
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
296
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
297

298
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
299
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
300
            logger.debug("matching {} with {}: {}".format(plug.name, kwargs,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
301
                                                          res))
302
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
303

304
305
306
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
307
            return {n: p for n, p in self.plugins.items() if matches(p)}
308
309

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
310
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
311
312
313
314
315
        return {
            p: plugin
            for p, plugin in self.plugins.items()
            if isinstance(plugin, SentimentPlugin)
        }