extensions.py 10.7 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
4
from future import standard_library
standard_library.install_aliases()
5
6
7
8
import gevent
from gevent import monkey
monkey.patch_all()

9
from .plugins import SentimentPlugin
10
from .models import Error
11
from .blueprints import api_blueprint, demo_blueprint
12
from .api import API_PARAMS, NIF_PARAMS, parse_params
13
14
15
16

from git import Repo, InvalidGitRepositoryError
from functools import partial

17
import os
18
19
import fnmatch
import inspect
20
import sys
21
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
22
import logging
23
import traceback
24
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
import pip
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
26
27

logger = logging.getLogger(__name__)
28

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29

30
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
31
    """ Default Senpy extension for Flask """
32

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
33
34
35
36
    def __init__(self,
                 app=None,
                 plugin_folder="plugins",
                 default_plugins=False):
37
38
        self.app = app

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
39
        self._search_folders = set()
40
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
41
42
        self._outdated = True

43
        self.add_folder(plugin_folder)
44
        if default_plugins:
45
46
            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
            self.add_folder(base_folder)
47
48
49
50

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
51
52
53
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
54
55
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
56
        """
57
58
59
60
61
62
63
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
64
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
65
        app.register_blueprint(demo_blueprint, url_prefix="/")
66

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
67
    def add_folder(self, folder):
68
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
69
70
71
72
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
73
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
74

75
    def analyse(self, **params):
76
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
77
        logger.debug("analysing with params: {}".format(params))
78
79
80
        api_params = parse_params(params, spec=API_PARAMS)
        if "algorithm" in api_params and api_params["algorithm"]:
            algo = api_params["algorithm"]
81
        elif self.plugins:
82
            algo = self.default_plugin and self.default_plugin.name
83
        if not algo:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
84
85
86
87
            raise Error(
                status=404,
                message=("No plugins found."
                         " Please install one.").format(algo))
88
        if algo not in self.plugins:
89
90
91
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
92
93
94
            raise Error(
                status=404,
                message="The algorithm '{}' is not valid".format(algo))
95
96
97

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
98
99
100
101
            raise Error(
                status=400,
                message=("The algorithm '{}'"
                         " is not activated yet").format(algo))
102
        plug = self.plugins[algo]
103
104
105
106
        nif_params = parse_params(params, spec=NIF_PARAMS)
        extra_params = plug.get('extra_params', {})
        specific_params = parse_params(params, spec=extra_params)
        nif_params.update(specific_params)
107
        try:
108
            resp = plug.analyse(**nif_params)
109
110
111
112
113
            resp.analysis.append(plug)
            logger.debug("Returning analysis result: {}".format(resp))
        except Exception as ex:
            resp = Error(message=str(ex), status=500)
        return resp
114
115
116

    @property
    def default_plugin(self):
117
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
118
        if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
119
            candidate = list(candidates.values())[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
120
            logger.debug("Default: {}".format(candidate.name))
121
122
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
123
            return None
124

125
    def parameters(self, algo):
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
126
127
        return getattr(
            self.plugins.get(algo) or self.default_plugin, "extra_params", {})
128

129
130
131
132
133
134
135
136
137
138
139
140
141
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
142
143
144
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
145
146
147
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
148
149
150
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
151
152
153
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

154
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
155

156
157
158
        def act():
            try:
                plugin.activate()
159
                logger.info("Plugin activated: {}".format(plugin.name))
160
            except Exception as ex:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
161
162
                logger.error("Error activating plugin {}: {}".format(
                    plugin.name, ex))
163
                logger.error("Trace: {}".format(traceback.format_exc()))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
164

165
        th = gevent.spawn(act)
166
167
168
169
170
171
172
        th.link_value(partial(self._set_active_plugin, plugin_name, True))
        if sync:
            th.join()
        else:
            return th

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
173
174
175
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
176
177
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
178
179
180
181
182
183

        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
184
185
                logger.error("Error deactivating plugin {}: {}".format(
                    plugin.name, ex))
186
187
188
                logger.error("Trace: {}".format(traceback.format_exc()))

        th = gevent.spawn(deact)
189
190
191
192
193
        th.link_value(partial(self._set_active_plugin, plugin_name, False))
        if sync:
            th.join()
        else:
            return th
194

195
196
197
198
199
200
201
202
203
204
    def reload_plugin(self, name):
        logger.debug("Reloading {}".format(name))
        plugin = self.plugins[name]
        try:
            del self.plugins[name]
            nplug = self._load_plugin(plugin.module, plugin.path)
            self.plugins[nplug.name] = nplug
        except Exception as ex:
            logger.error('Error reloading {}: {}'.format(name, ex))
            self.plugins[name] = plugin
205

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
206
207
208
209
    @classmethod
    def validate_info(cls, info):
        return all(x in info for x in ('name', 'module', 'version'))

210
211
212
213
214
215
216
217
218
219
220
    def install_deps(self):
        for i in self.plugins.values():
            self._install_deps(i._info)

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
            pip_args = []
            pip_args.append('install')
            for req in requirements:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
221
                pip_args.append(req)
222
            logger.info('Installing requirements: ' + str(requirements))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
223
            pip.main(pip_args)
224

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
225
226
227
228
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
229
            return None, None
230
231
        module = info["module"]
        name = info["name"]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
232
        sys.path.append(root)
233
        (fp, pathname, desc) = imp.find_module(module, [root, ])
234
        try:
235
            cls._install_deps(info)
236
237
238
239
240
            tmp = imp.load_module(module, fp, pathname, desc)
            sys.path.remove(root)
            candidate = None
            for _, obj in inspect.getmembers(tmp):
                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
241
                    logger.debug(("Found plugin class:"
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
242
243
                                  " {}@{}").format(obj, inspect.getmodule(
                                      obj)))
244
245
246
                    candidate = obj
                    break
            if not candidate:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
247
                logger.debug("No valid plugin for: {}".format(module))
248
249
                return
            module = candidate(info=info)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
250
251
252
            repo_path = root
            module._repo = Repo(repo_path)
        except InvalidGitRepositoryError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
253
254
            logger.debug("The plugin {} is not in a Git repository".format(
                module))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
255
            module._repo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
256
        except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
257
            logger.error("Exception importing {}: {}".format(module, ex))
258
            logger.error("Trace: {}".format(traceback.format_exc()))
259
260
            return None, None
        return name, module
261

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
262
263
264
265
266
267
268
269
270
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)

271
272
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
273
        for search_folder in self._search_folders:
274
275
276
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
277
                    if plugin and name not in self._plugin_list:
278
                        plugins[name] = plugin
279

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
280
        self._outdated = False
281
282
283
284
285
286
287
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
288
        """ Return the plugins registered for a given application.  """
289
290
291
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
292

293
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
294
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
295

296
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
297
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
298
            logger.debug("matching {} with {}: {}".format(plug.name, kwargs,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
299
                                                          res))
300
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
301

302
303
304
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
305
            return {n: p for n, p in self.plugins.items() if matches(p)}
306
307

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
308
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
309
310
311
312
313
        return {
            p: plugin
            for p, plugin in self.plugins.items()
            if isinstance(plugin, SentimentPlugin)
        }