extensions.py 8.31 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
4
from future import standard_library
standard_library.install_aliases()
5
6
7
8
import gevent
from gevent import monkey
monkey.patch_all()

9
10
11
12
13
14
15
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
from .blueprints import nif_blueprint

from git import Repo, InvalidGitRepositoryError
from functools import partial

16
import os
17
18
import fnmatch
import inspect
19
import sys
20
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21
import logging
22
import traceback
23
24
import gevent
import json
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
26

logger = logging.getLogger(__name__)
27

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
28

29
class Senpy(object):
30

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
31
    """ Default Senpy extension for Flask """
32

33
    def __init__(self, app=None, plugin_folder="plugins", default_plugins=False):
34
35
        self.app = app

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
36
37
38
        self._search_folders = set()
        self._outdated = True

39
        self.add_folder(plugin_folder)
40
        if default_plugins:
41
42
            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
            self.add_folder(base_folder)
43
44
45
46

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
47
48
49
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
50
51
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
52
        """
53
54
55
56
57
58
59
60
61
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
        app.register_blueprint(nif_blueprint)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
62
    def add_folder(self, folder):
63
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
64
65
66
67
68
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
            return True
        else:
69
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
70
71
            return False

72
    def analyse(self, **params):
73
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
74
        logger.debug("analysing with params: {}".format(params))
75
76
        if "algorithm" in params:
            algo = params["algorithm"]
77
        elif self.plugins:
78
            algo = self.default_plugin and self.default_plugin.name
79
80
81
82
        if not algo:
            return Error(status=404,
                         message=("No plugins found."
                                  " Please install one.").format(algo))
83
84
85
86
        if algo in self.plugins:
            if self.plugins[algo].is_activated:
                plug = self.plugins[algo]
                resp = plug.analyse(**params)
87
                resp.analysis.append(plug)
88
                logger.debug("Returning analysis result: {}".format(resp))
89
                return resp
90
91
            else:
                logger.debug("Plugin not activated: {}".format(algo))
92
93
94
                return Error(status=400,
                             message=("The algorithm '{}'"
                                      " is not activated yet").format(algo))
95
        else:
96
97
98
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
99
            return Error(status=404,
100
101
                         message="The algorithm '{}' is not valid"
                                 .format(algo))
102
103
104

    @property
    def default_plugin(self):
105
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
106
        if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
107
            candidate = list(candidates.values())[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
108
            logger.debug("Default: {}".format(candidate))
109
110
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
111
            return None
112

113
    def parameters(self, algo):
114
115
116
        return getattr(self.plugins.get(algo) or self.default_plugin,
                       "params",
                       {})
117

118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
135
136
137
138
139
140
        def act():
            try:
                plugin.activate()
            except Exception as ex:
                logger.error("Error activating plugin {}: {}".format(plugin.name,
                                                                     ex))
141
                logger.error("Trace: {}".format(traceback.format_exc()))
142
        th = gevent.spawn(act)
143
144
145
146
147
148
149
150
151
152
153
154
155
156
        th.link_value(partial(self._set_active_plugin, plugin_name, True))
        if sync:
            th.join()
        else:
            return th

    def deactivate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
        th = gevent.spawn(plugin.deactivate)
        th.link_value(partial(self._set_active_plugin, plugin_name, False))
        if sync:
            th.join()
        else:
            return th
157
158

    def reload_plugin(self, plugin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
159
        logger.debug("Reloading {}".format(plugin))
160
161
162
163
        plug = self.plugins[plugin]
        nplug = self._load_plugin(plug.module, plug.path)
        del self.plugins[plugin]
        self.plugins[nplug.name] = nplug
164

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
165
    @staticmethod
166
167
168
    def _load_plugin(root, filename):
        logger.debug("Loading plugin: {}".format(filename))
        fpath = os.path.join(root, filename)
169
        with open(fpath, 'r') as f:
170
171
172
173
174
            info = json.load(f)
        logger.debug("Info: {}".format(info))
        sys.path.append(root)
        module = info["module"]
        name = info["name"]
175
        (fp, pathname, desc) = imp.find_module(module, [root, ])
176
        try:
177
178
179
180
181
            tmp = imp.load_module(module, fp, pathname, desc)
            sys.path.remove(root)
            candidate = None
            for _, obj in inspect.getmembers(tmp):
                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
182
183
184
                    logger.debug(("Found plugin class:"
                                  " {}@{}").format(obj, inspect.getmodule(obj))
                                 )
185
186
187
188
189
190
                    candidate = obj
                    break
            if not candidate:
                logger.debug("No valid plugin for: {}".format(filename))
                return
            module = candidate(info=info)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
191
            try:
192
                repo_path = root
193
                module._repo = Repo(repo_path)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
194
            except InvalidGitRepositoryError:
195
                module._repo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
196
        except Exception as ex:
197
198
            logger.error("Exception importing {}: {}".format(filename, ex))
            logger.error("Trace: {}".format(traceback.format_exc()))
199
200
            return None, None
        return name, module
201
202
203

    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
204
        for search_folder in self._search_folders:
205
206
207
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
208
                    if plugin:
209
                        plugins[name] = plugin
210

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
211
        self._outdated = False
212
213
214
215
216
217
218
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
219
        """ Return the plugins registered for a given application.  """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
220
221
222
        if not hasattr(self, 'senpy_plugins') or self._outdated:
            self.senpy_plugins = self._load_plugins()
        return self.senpy_plugins
223

224
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
225
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
226

227
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
228
229
230
231
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
            logger.debug("matching {} with {}: {}".format(plug.name,
                                                          kwargs,
                                                          res))
232
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
233

234
235
236
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
237
            return {n: p for n, p in self.plugins.items() if matches(p)}
238
239

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
240
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
241
        return {p: plugin for p, plugin in self.plugins.items() if
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
242
                isinstance(plugin, SentimentPlugin)}