extensions.py 9.95 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
4
from future import standard_library
standard_library.install_aliases()
5
6
7
8
import gevent
from gevent import monkey
monkey.patch_all()

9
10
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
11
from .blueprints import api_blueprint, demo_blueprint
12
from .api import API_PARAMS, NIF_PARAMS, parse_params
13
14
15
16

from git import Repo, InvalidGitRepositoryError
from functools import partial

17
import os
18
19
import fnmatch
import inspect
20
import sys
21
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
22
import logging
23
import traceback
24
25
import gevent
import json
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
26
27

logger = logging.getLogger(__name__)
28

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29

30
class Senpy(object):
31

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
32
    """ Default Senpy extension for Flask """
33

34
    def __init__(self, app=None, plugin_folder="plugins", default_plugins=False):
35
36
        self.app = app

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
37
        self._search_folders = set()
38
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
39
40
        self._outdated = True

41
        self.add_folder(plugin_folder)
42
        if default_plugins:
43
44
            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
            self.add_folder(base_folder)
45
46
47
48

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
49
50
51
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
52
53
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
54
        """
55
56
57
58
59
60
61
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
62
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
63
        app.register_blueprint(demo_blueprint, url_prefix="/")
64

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
65
    def add_folder(self, folder):
66
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
67
68
69
70
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
71
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
72

73
    def analyse(self, **params):
74
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
        logger.debug("analysing with params: {}".format(params))
76
77
78
        api_params = parse_params(params, spec=API_PARAMS)
        if "algorithm" in api_params and api_params["algorithm"]:
            algo = api_params["algorithm"]
79
        elif self.plugins:
80
            algo = self.default_plugin and self.default_plugin.name
81
        if not algo:
82
83
84
85
            raise Error(status=404,
                        message=("No plugins found."
                                 " Please install one.").format(algo))
        if algo not in self.plugins:
86
87
88
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
89
90
91
92
93
94
95
96
97
98
            raise Error(status=404,
                        message="The algorithm '{}' is not valid"
                        .format(algo))

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
            raise Error(status=400,
                        message=("The algorithm '{}'"
                                    " is not activated yet").format(algo))
        plug = self.plugins[algo]
99
100
101
102
        nif_params = parse_params(params, spec=NIF_PARAMS)
        extra_params = plug.get('extra_params', {})
        specific_params = parse_params(params, spec=extra_params)
        nif_params.update(specific_params)
103
        try:
104
            resp = plug.analyse(**nif_params)
105
106
107
108
109
            resp.analysis.append(plug)
            logger.debug("Returning analysis result: {}".format(resp))
        except Exception as ex:
            resp = Error(message=str(ex), status=500)
        return resp
110
111
112

    @property
    def default_plugin(self):
113
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
114
        if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
115
            candidate = list(candidates.values())[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
116
            logger.debug("Default: {}".format(candidate.name))
117
118
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
119
            return None
120

121
    def parameters(self, algo):
122
        return getattr(self.plugins.get(algo) or self.default_plugin,
123
                       "extra_params",
124
                       {})
125

126
127
128
129
130
131
132
133
134
135
136
137
138
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
139
140
141
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
142
143
144
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
145
146
147
148
149
150
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
            raise Error(message="Plugin not found: {}".format(plugin_name),
                        status=404)
            
151
        logger.info("Activating plugin: {}".format(plugin.name))
152
153
154
        def act():
            try:
                plugin.activate()
155
                logger.info("Plugin activated: {}".format(plugin.name))
156
157
158
            except Exception as ex:
                logger.error("Error activating plugin {}: {}".format(plugin.name,
                                                                     ex))
159
                logger.error("Trace: {}".format(traceback.format_exc()))
160
        th = gevent.spawn(act)
161
162
163
164
165
166
167
        th.link_value(partial(self._set_active_plugin, plugin_name, True))
        if sync:
            th.join()
        else:
            return th

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
168
169
170
171
172
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
            raise Error(message="Plugin not found: {}".format(plugin_name),
                        status=404)
173
174
175
176
177
178
179
180
181
182
183

        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
                logger.error("Error deactivating plugin {}: {}".format(plugin.name,
                                                                       ex))
                logger.error("Trace: {}".format(traceback.format_exc()))

        th = gevent.spawn(deact)
184
185
186
187
188
        th.link_value(partial(self._set_active_plugin, plugin_name, False))
        if sync:
            th.join()
        else:
            return th
189

190
191
192
193
194
195
196
197
198
199
    def reload_plugin(self, name):
        logger.debug("Reloading {}".format(name))
        plugin = self.plugins[name]
        try:
            del self.plugins[name]
            nplug = self._load_plugin(plugin.module, plugin.path)
            self.plugins[nplug.name] = nplug
        except Exception as ex:
            logger.error('Error reloading {}: {}'.format(name, ex))
            self.plugins[name] = plugin
200

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
201
    @staticmethod
202
203
204
    def _load_plugin(root, filename):
        logger.debug("Loading plugin: {}".format(filename))
        fpath = os.path.join(root, filename)
205
        with open(fpath, 'r') as f:
206
207
208
209
210
            info = json.load(f)
        logger.debug("Info: {}".format(info))
        sys.path.append(root)
        module = info["module"]
        name = info["name"]
211
        (fp, pathname, desc) = imp.find_module(module, [root, ])
212
        try:
213
214
215
216
217
            tmp = imp.load_module(module, fp, pathname, desc)
            sys.path.remove(root)
            candidate = None
            for _, obj in inspect.getmembers(tmp):
                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
218
219
220
                    logger.debug(("Found plugin class:"
                                  " {}@{}").format(obj, inspect.getmodule(obj))
                                 )
221
222
223
224
225
226
                    candidate = obj
                    break
            if not candidate:
                logger.debug("No valid plugin for: {}".format(filename))
                return
            module = candidate(info=info)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
227
            try:
228
                repo_path = root
229
                module._repo = Repo(repo_path)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
230
            except InvalidGitRepositoryError:
231
                module._repo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
232
        except Exception as ex:
233
234
            logger.error("Exception importing {}: {}".format(filename, ex))
            logger.error("Trace: {}".format(traceback.format_exc()))
235
236
            return None, None
        return name, module
237
238
239

    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
240
        for search_folder in self._search_folders:
241
242
243
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
244
                    if plugin and name not in self._plugin_list:
245
                        plugins[name] = plugin
246

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
247
        self._outdated = False
248
249
250
251
252
253
254
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
255
        """ Return the plugins registered for a given application.  """
256
257
258
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
259

260
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
261
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
262

263
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
264
265
266
267
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
            logger.debug("matching {} with {}: {}".format(plug.name,
                                                          kwargs,
                                                          res))
268
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
269

270
271
272
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
273
            return {n: p for n, p in self.plugins.items() if matches(p)}
274
275

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
276
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
277
        return {p: plugin for p, plugin in self.plugins.items() if
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
278
                isinstance(plugin, SentimentPlugin)}