extensions.py 9.58 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
4
from future import standard_library
standard_library.install_aliases()
5
6
7
8
import gevent
from gevent import monkey
monkey.patch_all()

9
10
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
11
from .blueprints import api_blueprint, demo_blueprint
12
13
14
15

from git import Repo, InvalidGitRepositoryError
from functools import partial

16
import os
17
18
import fnmatch
import inspect
19
import sys
20
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21
import logging
22
import traceback
23
24
import gevent
import json
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
26

logger = logging.getLogger(__name__)
27

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
28

29
class Senpy(object):
30

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
31
    """ Default Senpy extension for Flask """
32

33
    def __init__(self, app=None, plugin_folder="plugins", default_plugins=False):
34
35
        self.app = app

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
36
        self._search_folders = set()
37
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
39
        self._outdated = True

40
        self.add_folder(plugin_folder)
41
        if default_plugins:
42
43
            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
            self.add_folder(base_folder)
44
45
46
47

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
48
49
50
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
51
52
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
53
        """
54
55
56
57
58
59
60
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
61
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
62
        app.register_blueprint(demo_blueprint, url_prefix="/")
63

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
64
    def add_folder(self, folder):
65
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
66
67
68
69
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
70
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
71

72
    def analyse(self, **params):
73
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
74
        logger.debug("analysing with params: {}".format(params))
75
76
        if "algorithm" in params:
            algo = params["algorithm"]
77
        elif self.plugins:
78
            algo = self.default_plugin and self.default_plugin.name
79
        if not algo:
80
81
82
83
            raise Error(status=404,
                        message=("No plugins found."
                                 " Please install one.").format(algo))
        if algo not in self.plugins:
84
85
86
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
            raise Error(status=404,
                        message="The algorithm '{}' is not valid"
                        .format(algo))

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
            raise Error(status=400,
                        message=("The algorithm '{}'"
                                    " is not activated yet").format(algo))
        plug = self.plugins[algo]
        try:
            resp = plug.analyse(**params)
            resp.analysis.append(plug)
            logger.debug("Returning analysis result: {}".format(resp))
        except Exception as ex:
            resp = Error(message=str(ex), status=500)
        return resp
104
105
106

    @property
    def default_plugin(self):
107
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
108
        if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109
            candidate = list(candidates.values())[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
110
            logger.debug("Default: {}".format(candidate))
111
112
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
113
            return None
114

115
    def parameters(self, algo):
116
        return getattr(self.plugins.get(algo) or self.default_plugin,
117
                       "extra_params",
118
                       {})
119

120
121
122
123
124
125
126
127
128
129
130
131
132
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
133
134
135
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
136
137
138
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
139
140
141
142
143
144
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
            raise Error(message="Plugin not found: {}".format(plugin_name),
                        status=404)
            
145
        logger.info("Activating plugin: {}".format(plugin.name))
146
147
148
        def act():
            try:
                plugin.activate()
149
                logger.info("Plugin activated: {}".format(plugin.name))
150
151
152
            except Exception as ex:
                logger.error("Error activating plugin {}: {}".format(plugin.name,
                                                                     ex))
153
                logger.error("Trace: {}".format(traceback.format_exc()))
154
        th = gevent.spawn(act)
155
156
157
158
159
160
161
        th.link_value(partial(self._set_active_plugin, plugin_name, True))
        if sync:
            th.join()
        else:
            return th

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
162
163
164
165
166
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
            raise Error(message="Plugin not found: {}".format(plugin_name),
                        status=404)
167
168
169
170
171
172
173
174
175
176
177

        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
                logger.error("Error deactivating plugin {}: {}".format(plugin.name,
                                                                       ex))
                logger.error("Trace: {}".format(traceback.format_exc()))

        th = gevent.spawn(deact)
178
179
180
181
182
        th.link_value(partial(self._set_active_plugin, plugin_name, False))
        if sync:
            th.join()
        else:
            return th
183

184
185
186
187
188
189
190
191
192
193
    def reload_plugin(self, name):
        logger.debug("Reloading {}".format(name))
        plugin = self.plugins[name]
        try:
            del self.plugins[name]
            nplug = self._load_plugin(plugin.module, plugin.path)
            self.plugins[nplug.name] = nplug
        except Exception as ex:
            logger.error('Error reloading {}: {}'.format(name, ex))
            self.plugins[name] = plugin
194

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
195
    @staticmethod
196
197
198
    def _load_plugin(root, filename):
        logger.debug("Loading plugin: {}".format(filename))
        fpath = os.path.join(root, filename)
199
        with open(fpath, 'r') as f:
200
201
202
203
204
            info = json.load(f)
        logger.debug("Info: {}".format(info))
        sys.path.append(root)
        module = info["module"]
        name = info["name"]
205
        (fp, pathname, desc) = imp.find_module(module, [root, ])
206
        try:
207
208
209
210
211
            tmp = imp.load_module(module, fp, pathname, desc)
            sys.path.remove(root)
            candidate = None
            for _, obj in inspect.getmembers(tmp):
                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
212
213
214
                    logger.debug(("Found plugin class:"
                                  " {}@{}").format(obj, inspect.getmodule(obj))
                                 )
215
216
217
218
219
220
                    candidate = obj
                    break
            if not candidate:
                logger.debug("No valid plugin for: {}".format(filename))
                return
            module = candidate(info=info)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
221
            try:
222
                repo_path = root
223
                module._repo = Repo(repo_path)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
224
            except InvalidGitRepositoryError:
225
                module._repo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
226
        except Exception as ex:
227
228
            logger.error("Exception importing {}: {}".format(filename, ex))
            logger.error("Trace: {}".format(traceback.format_exc()))
229
230
            return None, None
        return name, module
231
232
233

    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
234
        for search_folder in self._search_folders:
235
236
237
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
238
                    if plugin and name not in self._plugin_list:
239
                        plugins[name] = plugin
240

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
241
        self._outdated = False
242
243
244
245
246
247
248
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
249
        """ Return the plugins registered for a given application.  """
250
251
252
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
253

254
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
255
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
256

257
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
258
259
260
261
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
            logger.debug("matching {} with {}: {}".format(plug.name,
                                                          kwargs,
                                                          res))
262
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
263

264
265
266
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
267
            return {n: p for n, p in self.plugins.items() if matches(p)}
268
269

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
270
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
271
        return {p: plugin for p, plugin in self.plugins.items() if
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
272
                isinstance(plugin, SentimentPlugin)}