extensions.py 8.41 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
4
from future import standard_library
standard_library.install_aliases()
5
6
7
8
import gevent
from gevent import monkey
monkey.patch_all()

9
10
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
11
from .blueprints import nif_blueprint, demo_blueprint
12
13
14
15

from git import Repo, InvalidGitRepositoryError
from functools import partial

16
import os
17
18
import fnmatch
import inspect
19
import sys
20
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21
import logging
22
import traceback
23
24
import gevent
import json
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
26

logger = logging.getLogger(__name__)
27

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
28

29
class Senpy(object):
30

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
31
    """ Default Senpy extension for Flask """
32

33
    def __init__(self, app=None, plugin_folder="plugins", default_plugins=False):
34
35
        self.app = app

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
36
37
38
        self._search_folders = set()
        self._outdated = True

39
        self.add_folder(plugin_folder)
40
        if default_plugins:
41
42
            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
            self.add_folder(base_folder)
43
44
45
46

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
47
48
49
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
50
51
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
52
        """
53
54
55
56
57
58
59
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
60
61
        app.register_blueprint(nif_blueprint, url_prefix="/api")
        app.register_blueprint(demo_blueprint, url_prefix="/")
62

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
63
    def add_folder(self, folder):
64
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
65
66
67
68
69
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
            return True
        else:
70
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
71
72
            return False

73
    def analyse(self, **params):
74
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
        logger.debug("analysing with params: {}".format(params))
76
77
        if "algorithm" in params:
            algo = params["algorithm"]
78
        elif self.plugins:
79
            algo = self.default_plugin and self.default_plugin.name
80
81
82
83
        if not algo:
            return Error(status=404,
                         message=("No plugins found."
                                  " Please install one.").format(algo))
84
85
86
87
        if algo in self.plugins:
            if self.plugins[algo].is_activated:
                plug = self.plugins[algo]
                resp = plug.analyse(**params)
88
                resp.analysis.append(plug)
89
                logger.debug("Returning analysis result: {}".format(resp))
90
                return resp
91
92
            else:
                logger.debug("Plugin not activated: {}".format(algo))
93
94
95
                return Error(status=400,
                             message=("The algorithm '{}'"
                                      " is not activated yet").format(algo))
96
        else:
97
98
99
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
100
            return Error(status=404,
101
102
                         message="The algorithm '{}' is not valid"
                                 .format(algo))
103
104
105

    @property
    def default_plugin(self):
106
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
107
        if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
108
            candidate = list(candidates.values())[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109
            logger.debug("Default: {}".format(candidate))
110
111
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
112
            return None
113

114
    def parameters(self, algo):
115
116
117
        return getattr(self.plugins.get(algo) or self.default_plugin,
                       "params",
                       {})
118

119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
136
137
138
139
140
141
        def act():
            try:
                plugin.activate()
            except Exception as ex:
                logger.error("Error activating plugin {}: {}".format(plugin.name,
                                                                     ex))
142
                logger.error("Trace: {}".format(traceback.format_exc()))
143
        th = gevent.spawn(act)
144
145
146
147
148
149
150
151
152
153
154
155
156
157
        th.link_value(partial(self._set_active_plugin, plugin_name, True))
        if sync:
            th.join()
        else:
            return th

    def deactivate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
        th = gevent.spawn(plugin.deactivate)
        th.link_value(partial(self._set_active_plugin, plugin_name, False))
        if sync:
            th.join()
        else:
            return th
158
159

    def reload_plugin(self, plugin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
160
        logger.debug("Reloading {}".format(plugin))
161
162
163
164
        plug = self.plugins[plugin]
        nplug = self._load_plugin(plug.module, plug.path)
        del self.plugins[plugin]
        self.plugins[nplug.name] = nplug
165

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
166
    @staticmethod
167
168
169
    def _load_plugin(root, filename):
        logger.debug("Loading plugin: {}".format(filename))
        fpath = os.path.join(root, filename)
170
        with open(fpath, 'r') as f:
171
172
173
174
175
            info = json.load(f)
        logger.debug("Info: {}".format(info))
        sys.path.append(root)
        module = info["module"]
        name = info["name"]
176
        (fp, pathname, desc) = imp.find_module(module, [root, ])
177
        try:
178
179
180
181
182
            tmp = imp.load_module(module, fp, pathname, desc)
            sys.path.remove(root)
            candidate = None
            for _, obj in inspect.getmembers(tmp):
                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
183
184
185
                    logger.debug(("Found plugin class:"
                                  " {}@{}").format(obj, inspect.getmodule(obj))
                                 )
186
187
188
189
190
191
                    candidate = obj
                    break
            if not candidate:
                logger.debug("No valid plugin for: {}".format(filename))
                return
            module = candidate(info=info)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
192
            try:
193
                repo_path = root
194
                module._repo = Repo(repo_path)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
195
            except InvalidGitRepositoryError:
196
                module._repo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
197
        except Exception as ex:
198
199
            logger.error("Exception importing {}: {}".format(filename, ex))
            logger.error("Trace: {}".format(traceback.format_exc()))
200
201
            return None, None
        return name, module
202
203
204

    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
205
        for search_folder in self._search_folders:
206
207
208
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
209
                    if plugin:
210
                        plugins[name] = plugin
211

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
212
        self._outdated = False
213
214
215
216
217
218
219
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
220
        """ Return the plugins registered for a given application.  """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
221
222
223
        if not hasattr(self, 'senpy_plugins') or self._outdated:
            self.senpy_plugins = self._load_plugins()
        return self.senpy_plugins
224

225
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
226
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
227

228
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
229
230
231
232
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
            logger.debug("matching {} with {}: {}".format(plug.name,
                                                          kwargs,
                                                          res))
233
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
234

235
236
237
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
238
            return {n: p for n, p in self.plugins.items() if matches(p)}
239
240

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
241
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
242
        return {p: plugin for p, plugin in self.plugins.items() if
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
243
                isinstance(plugin, SentimentPlugin)}