extensions.py 10.9 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
4
from future import standard_library
standard_library.install_aliases()
5
6
7
8
import gevent
from gevent import monkey
monkey.patch_all()

9
10
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
11
from .blueprints import api_blueprint, demo_blueprint
12
from .api import API_PARAMS, NIF_PARAMS, parse_params
13
14
15
16

from git import Repo, InvalidGitRepositoryError
from functools import partial

17
import os
18
19
import fnmatch
import inspect
20
import sys
21
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
22
import logging
23
import traceback
24
import gevent
25
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
26
import pip
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
27
28

logger = logging.getLogger(__name__)
29

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
30

31
class Senpy(object):
32

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
33
    """ Default Senpy extension for Flask """
34

35
    def __init__(self, app=None, plugin_folder="plugins", default_plugins=False):
36
37
        self.app = app

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
        self._search_folders = set()
39
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
40
41
        self._outdated = True

42
        self.add_folder(plugin_folder)
43
        if default_plugins:
44
45
            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
            self.add_folder(base_folder)
46
47
48
49

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
50
51
52
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
53
54
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
55
        """
56
57
58
59
60
61
62
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
63
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
64
        app.register_blueprint(demo_blueprint, url_prefix="/")
65

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
66
    def add_folder(self, folder):
67
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
68
69
70
71
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
72
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
73

74
    def analyse(self, **params):
75
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
76
        logger.debug("analysing with params: {}".format(params))
77
78
79
        api_params = parse_params(params, spec=API_PARAMS)
        if "algorithm" in api_params and api_params["algorithm"]:
            algo = api_params["algorithm"]
80
        elif self.plugins:
81
            algo = self.default_plugin and self.default_plugin.name
82
        if not algo:
83
84
85
86
            raise Error(status=404,
                        message=("No plugins found."
                                 " Please install one.").format(algo))
        if algo not in self.plugins:
87
88
89
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
90
91
92
93
94
95
96
97
98
99
            raise Error(status=404,
                        message="The algorithm '{}' is not valid"
                        .format(algo))

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
            raise Error(status=400,
                        message=("The algorithm '{}'"
                                    " is not activated yet").format(algo))
        plug = self.plugins[algo]
100
101
102
103
        nif_params = parse_params(params, spec=NIF_PARAMS)
        extra_params = plug.get('extra_params', {})
        specific_params = parse_params(params, spec=extra_params)
        nif_params.update(specific_params)
104
        try:
105
            resp = plug.analyse(**nif_params)
106
107
108
109
110
            resp.analysis.append(plug)
            logger.debug("Returning analysis result: {}".format(resp))
        except Exception as ex:
            resp = Error(message=str(ex), status=500)
        return resp
111
112
113

    @property
    def default_plugin(self):
114
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
115
        if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
116
            candidate = list(candidates.values())[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
117
            logger.debug("Default: {}".format(candidate.name))
118
119
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
120
            return None
121

122
    def parameters(self, algo):
123
        return getattr(self.plugins.get(algo) or self.default_plugin,
124
                       "extra_params",
125
                       {})
126

127
128
129
130
131
132
133
134
135
136
137
138
139
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
140
141
142
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
143
144
145
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
146
147
148
149
150
151
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
            raise Error(message="Plugin not found: {}".format(plugin_name),
                        status=404)
            
152
        logger.info("Activating plugin: {}".format(plugin.name))
153
154
155
        def act():
            try:
                plugin.activate()
156
                logger.info("Plugin activated: {}".format(plugin.name))
157
158
159
            except Exception as ex:
                logger.error("Error activating plugin {}: {}".format(plugin.name,
                                                                     ex))
160
                logger.error("Trace: {}".format(traceback.format_exc()))
161
        th = gevent.spawn(act)
162
163
164
165
166
167
168
        th.link_value(partial(self._set_active_plugin, plugin_name, True))
        if sync:
            th.join()
        else:
            return th

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
169
170
171
172
173
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
            raise Error(message="Plugin not found: {}".format(plugin_name),
                        status=404)
174
175
176
177
178
179
180
181
182
183
184

        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
                logger.error("Error deactivating plugin {}: {}".format(plugin.name,
                                                                       ex))
                logger.error("Trace: {}".format(traceback.format_exc()))

        th = gevent.spawn(deact)
185
186
187
188
189
        th.link_value(partial(self._set_active_plugin, plugin_name, False))
        if sync:
            th.join()
        else:
            return th
190

191
192
193
194
195
196
197
198
199
200
    def reload_plugin(self, name):
        logger.debug("Reloading {}".format(name))
        plugin = self.plugins[name]
        try:
            del self.plugins[name]
            nplug = self._load_plugin(plugin.module, plugin.path)
            self.plugins[nplug.name] = nplug
        except Exception as ex:
            logger.error('Error reloading {}: {}'.format(name, ex))
            self.plugins[name] = plugin
201

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
202
203
204
205
206

    @classmethod
    def validate_info(cls, info):
        return all(x in info for x in ('name', 'module', 'version'))

207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
    def install_deps(self):
        for i in self.plugins.values():
            self._install_deps(i._info)

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
            pip_args = []
            pip_args.append('install')
            for req in requirements:
                pip_args.append( req )
            logger.info('Installing requirements: ' + str(requirements))
            pip.main(pip_args) 

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
222
223
224
225
226
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
            return None, None 
227
228
        module = info["module"]
        name = info["name"]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
229
230
        requirements = info.get("requirements", [])
        sys.path.append(root)
231
        (fp, pathname, desc) = imp.find_module(module, [root, ])
232
        try:
233
            cls._install_deps(info)
234
235
236
237
238
            tmp = imp.load_module(module, fp, pathname, desc)
            sys.path.remove(root)
            candidate = None
            for _, obj in inspect.getmembers(tmp):
                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
239
240
241
                    logger.debug(("Found plugin class:"
                                  " {}@{}").format(obj, inspect.getmodule(obj))
                                 )
242
243
244
                    candidate = obj
                    break
            if not candidate:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
245
                logger.debug("No valid plugin for: {}".format(module))
246
247
                return
            module = candidate(info=info)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
248
249
250
251
252
            repo_path = root
            module._repo = Repo(repo_path)
        except InvalidGitRepositoryError:
            logger.debug("The plugin {} is not in a Git repository".format(module))
            module._repo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
253
        except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
254
            logger.error("Exception importing {}: {}".format(module, ex))
255
            logger.error("Trace: {}".format(traceback.format_exc()))
256
257
            return None, None
        return name, module
258

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
259
260
261
262
263
264
265
266
267
268
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)


269
270
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
271
        for search_folder in self._search_folders:
272
273
274
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
275
                    if plugin and name not in self._plugin_list:
276
                        plugins[name] = plugin
277

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
278
        self._outdated = False
279
280
281
282
283
284
285
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
286
        """ Return the plugins registered for a given application.  """
287
288
289
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
290

291
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
292
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
293

294
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
295
296
297
298
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
            logger.debug("matching {} with {}: {}".format(plug.name,
                                                          kwargs,
                                                          res))
299
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
300

301
302
303
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
304
            return {n: p for n, p in self.plugins.items() if matches(p)}
305
306

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
307
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
308
        return {p: plugin for p, plugin in self.plugins.items() if
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
309
                isinstance(plugin, SentimentPlugin)}