extensions.py 9.25 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
4
from future import standard_library
standard_library.install_aliases()
5
6
7
8
import gevent
from gevent import monkey
monkey.patch_all()

9
10
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
11
from .blueprints import nif_blueprint, demo_blueprint
12
13
14
15

from git import Repo, InvalidGitRepositoryError
from functools import partial

16
import os
17
18
import fnmatch
import inspect
19
import sys
20
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21
import logging
22
import traceback
23
24
import gevent
import json
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
26

logger = logging.getLogger(__name__)
27

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
28

29
class Senpy(object):
30

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
31
    """ Default Senpy extension for Flask """
32

33
    def __init__(self, app=None, plugin_folder="plugins", default_plugins=False):
34
35
        self.app = app

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
36
        self._search_folders = set()
37
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
39
        self._outdated = True

40
        self.add_folder(plugin_folder)
41
        if default_plugins:
42
43
            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
            self.add_folder(base_folder)
44
45
46
47

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
48
49
50
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
51
52
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
53
        """
54
55
56
57
58
59
60
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
61
62
        app.register_blueprint(nif_blueprint, url_prefix="/api")
        app.register_blueprint(demo_blueprint, url_prefix="/")
63

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
64
    def add_folder(self, folder):
65
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
66
67
68
69
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
70
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
71

72
    def analyse(self, **params):
73
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
74
        logger.debug("analysing with params: {}".format(params))
75
76
        if "algorithm" in params:
            algo = params["algorithm"]
77
        elif self.plugins:
78
            algo = self.default_plugin and self.default_plugin.name
79
80
81
82
        if not algo:
            return Error(status=404,
                         message=("No plugins found."
                                  " Please install one.").format(algo))
83
84
85
86
        if algo in self.plugins:
            if self.plugins[algo].is_activated:
                plug = self.plugins[algo]
                resp = plug.analyse(**params)
87
                resp.analysis.append(plug)
88
                logger.debug("Returning analysis result: {}".format(resp))
89
                return resp
90
91
            else:
                logger.debug("Plugin not activated: {}".format(algo))
92
93
94
                return Error(status=400,
                             message=("The algorithm '{}'"
                                      " is not activated yet").format(algo))
95
        else:
96
97
98
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
99
            return Error(status=404,
100
101
                         message="The algorithm '{}' is not valid"
                                 .format(algo))
102
103
104

    @property
    def default_plugin(self):
105
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
106
        if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
107
            candidate = list(candidates.values())[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
108
            logger.debug("Default: {}".format(candidate))
109
110
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
111
            return None
112

113
    def parameters(self, algo):
114
        return getattr(self.plugins.get(algo) or self.default_plugin,
115
                       "extra_params",
116
                       {})
117

118
119
120
121
122
123
124
125
126
127
128
129
130
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
131
132
133
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
134
135
136
137
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
138
        logger.info("Activating plugin: {}".format(plugin.name))
139
140
141
        def act():
            try:
                plugin.activate()
142
                logger.info("Plugin activated: {}".format(plugin.name))
143
144
145
            except Exception as ex:
                logger.error("Error activating plugin {}: {}".format(plugin.name,
                                                                     ex))
146
                logger.error("Trace: {}".format(traceback.format_exc()))
147
        th = gevent.spawn(act)
148
149
150
151
152
153
154
155
        th.link_value(partial(self._set_active_plugin, plugin_name, True))
        if sync:
            th.join()
        else:
            return th

    def deactivate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
156
157
158
159
160
161
162
163
164
165
166

        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
                logger.error("Error deactivating plugin {}: {}".format(plugin.name,
                                                                       ex))
                logger.error("Trace: {}".format(traceback.format_exc()))

        th = gevent.spawn(deact)
167
168
169
170
171
        th.link_value(partial(self._set_active_plugin, plugin_name, False))
        if sync:
            th.join()
        else:
            return th
172

173
174
175
176
177
178
179
180
181
182
    def reload_plugin(self, name):
        logger.debug("Reloading {}".format(name))
        plugin = self.plugins[name]
        try:
            del self.plugins[name]
            nplug = self._load_plugin(plugin.module, plugin.path)
            self.plugins[nplug.name] = nplug
        except Exception as ex:
            logger.error('Error reloading {}: {}'.format(name, ex))
            self.plugins[name] = plugin
183

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
184
    @staticmethod
185
186
187
    def _load_plugin(root, filename):
        logger.debug("Loading plugin: {}".format(filename))
        fpath = os.path.join(root, filename)
188
        with open(fpath, 'r') as f:
189
190
191
192
193
            info = json.load(f)
        logger.debug("Info: {}".format(info))
        sys.path.append(root)
        module = info["module"]
        name = info["name"]
194
        (fp, pathname, desc) = imp.find_module(module, [root, ])
195
        try:
196
197
198
199
200
            tmp = imp.load_module(module, fp, pathname, desc)
            sys.path.remove(root)
            candidate = None
            for _, obj in inspect.getmembers(tmp):
                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
201
202
203
                    logger.debug(("Found plugin class:"
                                  " {}@{}").format(obj, inspect.getmodule(obj))
                                 )
204
205
206
207
208
209
                    candidate = obj
                    break
            if not candidate:
                logger.debug("No valid plugin for: {}".format(filename))
                return
            module = candidate(info=info)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
210
            try:
211
                repo_path = root
212
                module._repo = Repo(repo_path)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
213
            except InvalidGitRepositoryError:
214
                module._repo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
215
        except Exception as ex:
216
217
            logger.error("Exception importing {}: {}".format(filename, ex))
            logger.error("Trace: {}".format(traceback.format_exc()))
218
219
            return None, None
        return name, module
220
221
222

    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
223
        for search_folder in self._search_folders:
224
225
226
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
227
                    if plugin and name not in self._plugin_list:
228
                        plugins[name] = plugin
229

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
230
        self._outdated = False
231
232
233
234
235
236
237
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
238
        """ Return the plugins registered for a given application.  """
239
240
241
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
242

243
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
244
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
245

246
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
247
248
249
250
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
            logger.debug("matching {} with {}: {}".format(plug.name,
                                                          kwargs,
                                                          res))
251
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
252

253
254
255
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
256
            return {n: p for n, p in self.plugins.items() if matches(p)}
257
258

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
259
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
260
        return {p: plugin for p, plugin in self.plugins.items() if
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
261
                isinstance(plugin, SentimentPlugin)}