extensions.py 10.7 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
4
from future import standard_library
standard_library.install_aliases()
5

6
from .plugins import SentimentPlugin
7
from .models import Error
8
from .blueprints import api_blueprint, demo_blueprint
9
from .api import API_PARAMS, NIF_PARAMS, parse_params
10
11

from git import Repo, InvalidGitRepositoryError
12
13

from threading import Thread
14

15
import os
16
17
import fnmatch
import inspect
18
import sys
19
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
20
import logging
21
import traceback
22
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23
import pip
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
24
25

logger = logging.getLogger(__name__)
26

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
27

28
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29
    """ Default Senpy extension for Flask """
30

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
31
32
33
34
    def __init__(self,
                 app=None,
                 plugin_folder="plugins",
                 default_plugins=False):
35
36
        self.app = app

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
37
        self._search_folders = set()
38
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
39
40
        self._outdated = True

41
        self.add_folder(plugin_folder)
42
        if default_plugins:
43
44
            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
            self.add_folder(base_folder)
45
46
47
48

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
49
50
51
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
52
53
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
54
        """
55
56
57
58
59
60
61
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
62
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
63
        app.register_blueprint(demo_blueprint, url_prefix="/")
64

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
65
    def add_folder(self, folder):
66
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
67
68
69
70
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
71
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
72

73
    def analyse(self, **params):
74
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
        logger.debug("analysing with params: {}".format(params))
76
77
78
        api_params = parse_params(params, spec=API_PARAMS)
        if "algorithm" in api_params and api_params["algorithm"]:
            algo = api_params["algorithm"]
79
        elif self.plugins:
80
            algo = self.default_plugin and self.default_plugin.name
81
        if not algo:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
82
83
84
85
            raise Error(
                status=404,
                message=("No plugins found."
                         " Please install one.").format(algo))
86
        if algo not in self.plugins:
87
88
89
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
90
91
92
            raise Error(
                status=404,
                message="The algorithm '{}' is not valid".format(algo))
93
94
95

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
96
97
98
99
            raise Error(
                status=400,
                message=("The algorithm '{}'"
                         " is not activated yet").format(algo))
100
        plug = self.plugins[algo]
101
102
103
104
        nif_params = parse_params(params, spec=NIF_PARAMS)
        extra_params = plug.get('extra_params', {})
        specific_params = parse_params(params, spec=extra_params)
        nif_params.update(specific_params)
105
        try:
106
            resp = plug.analyse(**nif_params)
107
108
109
110
111
            resp.analysis.append(plug)
            logger.debug("Returning analysis result: {}".format(resp))
        except Exception as ex:
            resp = Error(message=str(ex), status=500)
        return resp
112
113
114

    @property
    def default_plugin(self):
115
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
116
        if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
117
            candidate = list(candidates.values())[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
118
            logger.debug("Default: {}".format(candidate.name))
119
120
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
121
            return None
122

123
    def parameters(self, algo):
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
124
125
        return getattr(
            self.plugins.get(algo) or self.default_plugin, "extra_params", {})
126

127
128
129
130
131
132
133
134
135
136
137
138
139
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
140
141
142
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
143
144
145
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
146
147
148
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
149
150
151
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

152
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
153

154
        def act():
155
            success = False
156
157
            try:
                plugin.activate()
158
159
160
161
                msg = "Plugin activated: {}".format(plugin.name)
                logger.info(msg)
                success = True
                self._set_active_plugin(plugin_name, success)
162
            except Exception as ex:
163
164
165
166
                msg = "Error activating plugin {} - {} : \n\t{}".format(
                    plugin.name, ex, ex.format_exc())
                logger.error(msg)
                raise Error(msg)
167
        if sync:
168
            act()
169
        else:
170
171
            th = Thread(target=act)
            th.start()
172
173

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
174
175
176
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
177
178
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
179

180
181
        self._set_active_plugin(plugin_name, False)

182
183
184
185
186
        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
187
188
                logger.error("Error deactivating plugin {}: {}".format(
                    plugin.name, ex))
189
190
                logger.error("Trace: {}".format(traceback.format_exc()))

191
        if sync:
192
            deact()
193
        else:
194
195
            th = Thread(target=deact)
            th.start()
196

197
198
199
200
201
202
203
204
205
206
    def reload_plugin(self, name):
        logger.debug("Reloading {}".format(name))
        plugin = self.plugins[name]
        try:
            del self.plugins[name]
            nplug = self._load_plugin(plugin.module, plugin.path)
            self.plugins[nplug.name] = nplug
        except Exception as ex:
            logger.error('Error reloading {}: {}'.format(name, ex))
            self.plugins[name] = plugin
207

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
208
209
210
211
    @classmethod
    def validate_info(cls, info):
        return all(x in info for x in ('name', 'module', 'version'))

212
213
214
215
216
217
218
219
220
221
222
    def install_deps(self):
        for i in self.plugins.values():
            self._install_deps(i._info)

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
            pip_args = []
            pip_args.append('install')
            for req in requirements:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
223
                pip_args.append(req)
224
            logger.info('Installing requirements: ' + str(requirements))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
225
            pip.main(pip_args)
226

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
227
228
229
230
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
231
            return None, None
232
233
        module = info["module"]
        name = info["name"]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
234
        sys.path.append(root)
235
        (fp, pathname, desc) = imp.find_module(module, [root, ])
236
        try:
237
            cls._install_deps(info)
238
239
240
241
242
            tmp = imp.load_module(module, fp, pathname, desc)
            sys.path.remove(root)
            candidate = None
            for _, obj in inspect.getmembers(tmp):
                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
243
                    logger.debug(("Found plugin class:"
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
244
245
                                  " {}@{}").format(obj, inspect.getmodule(
                                      obj)))
246
247
248
                    candidate = obj
                    break
            if not candidate:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
249
                logger.debug("No valid plugin for: {}".format(module))
250
251
                return
            module = candidate(info=info)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
252
253
254
            repo_path = root
            module._repo = Repo(repo_path)
        except InvalidGitRepositoryError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
255
256
            logger.debug("The plugin {} is not in a Git repository".format(
                module))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
257
            module._repo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
258
        except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
259
            logger.error("Exception importing {}: {}".format(module, ex))
260
            logger.error("Trace: {}".format(traceback.format_exc()))
261
262
            return None, None
        return name, module
263

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
264
265
266
267
268
269
270
271
272
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)

273
274
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
275
        for search_folder in self._search_folders:
276
277
278
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
279
                    if plugin and name not in self._plugin_list:
280
                        plugins[name] = plugin
281

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
282
        self._outdated = False
283
284
285
286
287
288
289
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
290
        """ Return the plugins registered for a given application.  """
291
292
293
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
294

295
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
296
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
297

298
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
299
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
300
            logger.debug("matching {} with {}: {}".format(plug.name, kwargs,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
301
                                                          res))
302
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
303

304
305
306
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
307
            return {n: p for n, p in self.plugins.items() if matches(p)}
308
309

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
310
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
311
312
313
314
315
        return {
            p: plugin
            for p, plugin in self.plugins.items()
            if isinstance(plugin, SentimentPlugin)
        }