extensions.py 10.1 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
"""
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
4
from future import standard_library
standard_library.install_aliases()
5

6
from .plugins import SentimentPlugin
7
from .models import Error
8
from .blueprints import api_blueprint, demo_blueprint
9
from .api import API_PARAMS, NIF_PARAMS, parse_params
10
11

from git import Repo, InvalidGitRepositoryError
12
13

from threading import Thread
14

15
import os
16
17
import fnmatch
import inspect
18
import sys
19
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
20
import logging
21
import traceback
22
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23
import pip
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
24
25

logger = logging.getLogger(__name__)
26

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
27

28
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29
    """ Default Senpy extension for Flask """
30

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
31
32
33
34
    def __init__(self,
                 app=None,
                 plugin_folder="plugins",
                 default_plugins=False):
35
36
        self.app = app

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
37
        self._search_folders = set()
38
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
39
40
        self._outdated = True

41
        self.add_folder(plugin_folder)
42
        if default_plugins:
43
44
            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
            self.add_folder(base_folder)
45
46
47
48

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
49
50
51
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
52
53
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
54
        """
55
56
57
58
59
60
61
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
62
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
63
        app.register_blueprint(demo_blueprint, url_prefix="/")
64

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
65
    def add_folder(self, folder):
66
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
67
68
69
70
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
71
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
72

73
    def analyse(self, **params):
74
        algo = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
        logger.debug("analysing with params: {}".format(params))
76
77
78
        api_params = parse_params(params, spec=API_PARAMS)
        if "algorithm" in api_params and api_params["algorithm"]:
            algo = api_params["algorithm"]
79
        elif self.plugins:
80
            algo = self.default_plugin and self.default_plugin.name
81
        if not algo:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
82
83
84
85
            raise Error(
                status=404,
                message=("No plugins found."
                         " Please install one.").format(algo))
86
        if algo not in self.plugins:
87
88
89
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
90
91
92
            raise Error(
                status=404,
                message="The algorithm '{}' is not valid".format(algo))
93
94
95

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
96
97
98
99
            raise Error(
                status=400,
                message=("The algorithm '{}'"
                         " is not activated yet").format(algo))
100
        plug = self.plugins[algo]
101
102
103
104
        nif_params = parse_params(params, spec=NIF_PARAMS)
        extra_params = plug.get('extra_params', {})
        specific_params = parse_params(params, spec=extra_params)
        nif_params.update(specific_params)
105
        try:
106
            resp = plug.analyse(**nif_params)
107
108
            resp.analysis.append(plug)
            logger.debug("Returning analysis result: {}".format(resp))
109
110
111
        except Error as ex:
            logger.exception('Error returning analysis result')
            resp = ex
112
113
        except Exception as ex:
            resp = Error(message=str(ex), status=500)
114
            logger.exception('Error returning analysis result')
115
        return resp
116
117
118

    @property
    def default_plugin(self):
119
        candidates = self.filter_plugins(is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
120
        if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
121
            candidate = list(candidates.values())[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
122
            logger.debug("Default: {}".format(candidate.name))
123
124
            return candidate
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
125
            return None
126

127
128
129
130
131
132
133
134
135
136
137
138
139
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
140
141
142
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
143
144
145
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
146
147
148
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
149
150
151
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

152
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
153

154
        def act():
155
            success = False
156
157
            try:
                plugin.activate()
158
159
160
161
                msg = "Plugin activated: {}".format(plugin.name)
                logger.info(msg)
                success = True
                self._set_active_plugin(plugin_name, success)
162
            except Exception as ex:
163
                msg = "Error activating plugin {} - {} : \n\t{}".format(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
164
                    plugin.name, ex, traceback.format_exc())
165
166
                logger.error(msg)
                raise Error(msg)
167
        if sync:
168
            act()
169
        else:
170
171
            th = Thread(target=act)
            th.start()
172
173

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
174
175
176
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
177
178
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
179

180
181
        self._set_active_plugin(plugin_name, False)

182
183
184
185
186
        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
187
188
                logger.error("Error deactivating plugin {}: {}".format(
                    plugin.name, ex))
189
190
                logger.error("Trace: {}".format(traceback.format_exc()))

191
        if sync:
192
            deact()
193
        else:
194
195
            th = Thread(target=deact)
            th.start()
196

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
197
198
199
200
    @classmethod
    def validate_info(cls, info):
        return all(x in info for x in ('name', 'module', 'version'))

201
202
203
204
205
206
207
208
209
210
    def install_deps(self):
        for i in self.plugins.values():
            self._install_deps(i._info)

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
            pip_args = []
            pip_args.append('install')
211
            pip_args.append('--use-wheel')
212
            for req in requirements:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
213
                pip_args.append(req)
214
            logger.info('Installing requirements: ' + str(requirements))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
215
            pip.main(pip_args)
216

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
217
218
219
220
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
221
            return None, None
222
223
        module = info["module"]
        name = info["name"]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
224
        sys.path.append(root)
225
        (fp, pathname, desc) = imp.find_module(module, [root, ])
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
        cls._install_deps(info)
        tmp = imp.load_module(module, fp, pathname, desc)
        sys.path.remove(root)
        candidate = None
        for _, obj in inspect.getmembers(tmp):
            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
                logger.debug(("Found plugin class:"
                              " {}@{}").format(obj, inspect.getmodule(obj)))
                candidate = obj
                break
        if not candidate:
            logger.debug("No valid plugin for: {}".format(module))
            return
        module = candidate(info=info)
        repo_path = root
241
        try:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
242
243
            module._repo = Repo(repo_path)
        except InvalidGitRepositoryError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
244
245
            logger.debug("The plugin {} is not in a Git repository".format(
                module))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
246
            module._repo = None
247
        return name, module
248

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
249
250
251
252
253
254
255
256
257
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)

258
259
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
260
        for search_folder in self._search_folders:
261
262
263
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
264
                    if plugin and name not in self._plugin_list:
265
                        plugins[name] = plugin
266

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
267
        self._outdated = False
268
269
270
271
272
273
274
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
275
        """ Return the plugins registered for a given application.  """
276
277
278
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
279

280
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
281
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
282

283
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
284
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
285
            logger.debug("matching {} with {}: {}".format(plug.name, kwargs,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
286
                                                          res))
287
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
288

289
290
291
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
292
            return {n: p for n, p in self.plugins.items() if matches(p)}
293
294

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
295
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
296
297
298
299
300
        return {
            p: plugin
            for p, plugin in self.plugins.items()
            if isinstance(plugin, SentimentPlugin)
        }