extensions.py 13.4 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2
3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
5
6
from future import standard_library
standard_library.install_aliases()
7

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8
9
10
from .plugins import SentimentPlugin, SenpyPlugin
from .models import Error, Entry, Results
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
11
from .api import API_PARAMS, NIF_PARAMS, parse_params
12

13
from threading import Thread
14

15
import os
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
16
import copy
17
18
import fnmatch
import inspect
19
import sys
20
import importlib
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21
import logging
22
import traceback
23
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
24
import pip
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
26

logger = logging.getLogger(__name__)
27

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
28

29
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
30
    """ Default Senpy extension for Flask """
31

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
32
33
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
34
                 plugin_folder=".",
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
35
                 default_plugins=False):
36
        self.app = app
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
37
        self._search_folders = set()
38
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
39
        self._outdated = True
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
40
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
41

42
        self.add_folder(plugin_folder)
43
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
44
45
46
47
48
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
49
50
51
52

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
53
54
55
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
56
57
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
58
        """
59
60
61
62
63
64
65
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
66
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
67
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
68
        app.register_blueprint(demo_blueprint, url_prefix="/")
69

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
70
71
72
    def add_folder(self, folder, from_root=False):
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
73
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
74
75
76
77
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
78
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
79

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
80
    def _find_plugin(self, params):
81
        api_params = parse_params(params, spec=API_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
82
        algo = None
83
84
        if "algorithm" in api_params and api_params["algorithm"]:
            algo = api_params["algorithm"]
85
        elif self.plugins:
86
            algo = self.default_plugin and self.default_plugin.name
87
        if not algo:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
88
89
90
91
            raise Error(
                status=404,
                message=("No plugins found."
                         " Please install one.").format(algo))
92
        if algo not in self.plugins:
93
94
95
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
96
97
98
            raise Error(
                status=404,
                message="The algorithm '{}' is not valid".format(algo))
99
100
101

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
102
103
104
105
            raise Error(
                status=400,
                message=("The algorithm '{}'"
                         " is not activated yet").format(algo))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
106
107
108
        return self.plugins[algo]

    def _get_params(self, params, plugin):
109
        nif_params = parse_params(params, spec=NIF_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
110
        extra_params = plugin.get('extra_params', {})
111
112
        specific_params = parse_params(params, spec=extra_params)
        nif_params.update(specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
        return nif_params

    def _get_entries(self, params):
        entry = None
        if params['informat'] == 'text':
            entry = Entry(text=params['input'])
        else:
            raise NotImplemented('Only text input format implemented')
        yield entry

    def analyse(self, **api_params):
        logger.debug("analysing with params: {}".format(api_params))
        plugin = self._find_plugin(api_params)
        nif_params = self._get_params(api_params, plugin)
        resp = Results()
        if 'with_parameters' in api_params:
            resp.parameters = nif_params
130
        try:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
131
132
133
134
135
136
            entries = []
            for i in self._get_entries(nif_params):
                entries += list(plugin.analyse_entry(i, nif_params))
            resp.entries = entries
            self.convert_emotions(resp, plugin, nif_params)
            resp.analysis.append(plugin.id)
137
            logger.debug("Returning analysis result: {}".format(resp))
138
139
140
        except Error as ex:
            logger.exception('Error returning analysis result')
            resp = ex
141
        except Exception as ex:
142
            logger.exception('Error returning analysis result')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
143
            resp = Error(message=str(ex), status=500)
144
        return resp
145

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
    def _conversion_candidates(self, fromModel, toModel):
        candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
        for name, candidate in candidates.items():
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    # logging.debug('Found candidate: {}'.format(candidate))
                    yield candidate

    def convert_emotions(self, resp, plugin, params):
        """
        Conversion of all emotions in a response.
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
        fromModel = plugin.get('onyx:usesEmotionModel', None)
        toModel = params.get('emotionModel', None)
        output = params.get('conversion', None)
        logger.debug('Asked for model: {}'.format(toModel))
        logger.debug('Analysis plugin uses model: {}'.format(fromModel))

        if not toModel:
            return
        try:
            candidate = next(self._conversion_candidates(fromModel, toModel))
        except StopIteration:
            e = Error(('No conversion plugin found for: '
                       '{} -> {}'.format(fromModel, toModel)))
            e.original_response = resp
            e.parameters = params
            raise e
        newentries = []
        for i in resp.entries:
            if output == "full":
184
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
185
186
187
188
189
190
191
192
193
194
195
196
197
            else:
                newemotions = []
            for j in i.emotions:
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries
        resp.analysis.append(candidate.id)

198
199
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
200
201
202
203
204
205
206
207
208
209
210
211
        candidate = self._default
        if not candidate:
            candidates = self.filter_plugins(is_activated=True)
            if len(candidates) > 0:
                candidate = list(candidates.values())[0]
        logger.debug("Default: {}".format(candidate))
        return candidate

    @default_plugin.setter
    def default_plugin(self, value):
        if isinstance(value, SenpyPlugin):
            self._default = value
212
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
213
            self._default = self.plugins[value]
214

215
216
217
218
219
220
221
222
223
224
225
226
227
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
228
229
230
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
231
232
233
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
234
235
236
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
237
238
239
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

240
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
241

242
        def act():
243
            success = False
244
245
            try:
                plugin.activate()
246
247
248
249
                msg = "Plugin activated: {}".format(plugin.name)
                logger.info(msg)
                success = True
                self._set_active_plugin(plugin_name, success)
250
            except Exception as ex:
251
                msg = "Error activating plugin {} - {} : \n\t{}".format(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
252
                    plugin.name, ex, traceback.format_exc())
253
254
                logger.error(msg)
                raise Error(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
255

256
        if sync:
257
            act()
258
        else:
259
260
            th = Thread(target=act)
            th.start()
261
262

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
263
264
265
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
266
267
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
268

269
270
        self._set_active_plugin(plugin_name, False)

271
272
273
274
275
        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
276
277
                logger.error(
                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
278
279
                logger.error("Trace: {}".format(traceback.format_exc()))

280
        if sync:
281
            deact()
282
        else:
283
284
            th = Thread(target=deact)
            th.start()
285

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
286
287
    @classmethod
    def validate_info(cls, info):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
288
        return all(x in info for x in ('name', 'module', 'description', 'version'))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
289

290
291
    def install_deps(self):
        for i in self.plugins.values():
292
            self._install_deps(i)
293
294
295
296
297
298
299

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
            pip_args = []
            pip_args.append('install')
300
            pip_args.append('--use-wheel')
301
            for req in requirements:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
302
                pip_args.append(req)
303
            logger.info('Installing requirements: ' + str(requirements))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
304
            pip.main(pip_args)
305

306
307
308
309
310
311
312
    @classmethod
    def _load_module(cls, name, root):
        sys.path.append(root)
        tmp = importlib.import_module(name)
        sys.path.remove(root)
        return tmp

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
313
314
315
316
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
317
            return None, None
318
319
        module = info["module"]
        name = info["name"]
320

321
        cls._install_deps(info)
322
323
        tmp = cls._load_module(module, root)

324
325
326
327
328
329
330
331
332
333
334
        candidate = None
        for _, obj in inspect.getmembers(tmp):
            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
                logger.debug(("Found plugin class:"
                              " {}@{}").format(obj, inspect.getmodule(obj)))
                candidate = obj
                break
        if not candidate:
            logger.debug("No valid plugin for: {}".format(module))
            return
        module = candidate(info=info)
335
        return name, module
336

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
337
338
339
340
341
342
343
344
345
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)

346
347
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
348
        for search_folder in self._search_folders:
349
350
351
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
352
                    if plugin and name:
353
                        plugins[name] = plugin
354

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
355
        self._outdated = False
356
357
358
359
360
361
362
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
363
        """ Return the plugins registered for a given application.  """
364
365
366
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
367

368
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
369
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
370

371
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
372
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
373
374
            logger.debug(
                "matching {} with {}: {}".format(plug.name, kwargs, res))
375
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
376

377
378
379
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
380
            return {n: p for n, p in self.plugins.items() if matches(p)}
381
382

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
383
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
384
385
386
387
388
        return {
            p: plugin
            for p, plugin in self.plugins.items()
            if isinstance(plugin, SentimentPlugin)
        }