extensions.py 15.5 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2
3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
5
6
from future import standard_library
standard_library.install_aliases()
7

8
9
from . import plugins
from .plugins import SenpyPlugin
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
10
from .models import Error, Entry, Results, from_dict
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
11
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
12
from .api import API_PARAMS, NIF_PARAMS, parse_params
13

14
from threading import Thread
15

16
import os
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
17
import copy
18
19
import fnmatch
import inspect
20
import sys
21
import importlib
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
22
import logging
23
import traceback
24
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
import pip
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
26
27

logger = logging.getLogger(__name__)
28

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29

30
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
31
    """ Default Senpy extension for Flask """
32

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
33
34
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
35
                 plugin_folder=".",
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
36
                 default_plugins=False):
37
        self.app = app
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
        self._search_folders = set()
39
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
40
        self._outdated = True
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
41
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
42

43
        self.add_folder(plugin_folder)
44
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
45
46
47
48
49
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
50
51
52
53

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
54
55
56
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
57
58
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
59
        """
60
61
62
63
64
65
66
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
67
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
68
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
69
        app.register_blueprint(demo_blueprint, url_prefix="/")
70

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
71
72
73
    def add_folder(self, folder, from_root=False):
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
74
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
76
77
78
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
79
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
80

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
81
82
    def _find_plugins(self, params):
        if not self.analysis_plugins:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
83
84
85
            raise Error(
                status=404,
                message=("No plugins found."
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
86
87
88
89
90
91
92
93
                         " Please install one."))
        api_params = parse_params(params, spec=API_PARAMS)
        algos = None
        if "algorithm" in api_params and api_params["algorithm"]:
            algos = api_params["algorithm"].split(',')
        elif self.default_plugin:
            algos = [self.default_plugin.name, ]
        else:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
94
95
            raise Error(
                status=404,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
96
                message="No default plugin found, and None provided")
97

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
98
99
100
101
102
103
104
105
106
        plugins = list()
        for algo in algos:
            if algo not in self.plugins:
                logger.debug(("The algorithm '{}' is not valid\n"
                              "Valid algorithms: {}").format(algo,
                                                             self.plugins.keys()))
                raise Error(
                    status=404,
                    message="The algorithm '{}' is not valid".format(algo))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
107

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
108
109
110
111
112
113
114
115
116
117
            if not self.plugins[algo].is_activated:
                logger.debug("Plugin not activated: {}".format(algo))
                raise Error(
                    status=400,
                    message=("The algorithm '{}'"
                             " is not activated yet").format(algo))
            plugins.append(self.plugins[algo])
        return plugins

    def _get_params(self, params, plugin=None):
118
        nif_params = parse_params(params, spec=NIF_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
119
120
121
122
        if plugin:
            extra_params = plugin.get('extra_params', {})
            specific_params = parse_params(params, spec=extra_params)
            nif_params.update(specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
123
124
125
126
        return nif_params

    def _get_entries(self, params):
        if params['informat'] == 'text':
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
127
            results = Results()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
128
            entry = Entry(text=params['input'])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
129
130
131
            results.entries.append(entry)
        elif params['informat'] == 'json-ld':
            results = from_dict(params['input'])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
132
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
            raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
        return results

    def _process_entries(self, entries, plugins, nif_params):
        if not plugins:
            for i in entries:
                yield i
            return
        plugin = plugins[0]
        specific_params = self._get_params(nif_params, plugin)
        results = plugin.analyse_entries(entries, specific_params)
        for i in self._process_entries(results, plugins[1:], nif_params):
            yield i

    def _process_response(self, resp, plugins, nif_params):
        entries = resp.entries
        resp.entries = []
        for plug in plugins:
            resp.analysis.append(plug.id)
        for i in self._process_entries(entries, plugins, nif_params):
            resp.entries.append(i)
        return resp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
155
156

    def analyse(self, **api_params):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
157
158
159
160
        """
        Main method that analyses a request, either from CLI or HTTP.
        It uses a dictionary of parameters, provided by the user.
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
161
        logger.debug("analysing with params: {}".format(api_params))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
162
163
164
        plugins = self._find_plugins(api_params)
        nif_params = self._get_params(api_params)
        resp = self._get_entries(nif_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
165
166
        if 'with_parameters' in api_params:
            resp.parameters = nif_params
167
        try:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
168
169
            resp = self._process_response(resp, plugins, nif_params)
            self.convert_emotions(resp, plugins, nif_params)
170
            logger.debug("Returning analysis result: {}".format(resp))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
171
172
173
        except (Error, Exception) as ex:
            if not isinstance(ex, Error):
                ex = Error(message=str(ex), status=500)
174
            logger.exception('Error returning analysis result')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
175
            raise ex
176
        return resp
177

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
178
179
180
181
182
183
184
185
186
187
188
    def _conversion_candidates(self, fromModel, toModel):
        candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
        for name, candidate in candidates.items():
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    # logging.debug('Found candidate: {}'.format(candidate))
                    yield candidate

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
189
    def convert_emotions(self, resp, plugins, params):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
190
191
192
193
194
195
196
197
198
199
        """
        Conversion of all emotions in a response.
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
        toModel = params.get('emotionModel', None)
        if not toModel:
            return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214

        logger.debug('Asked for model: {}'.format(toModel))
        output = params.get('conversion', None)
        candidates = {}
        for plugin in plugins:
            try:
                fromModel = plugin.get('onyx:usesEmotionModel', None)
                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
            except StopIteration:
                e = Error(('No conversion plugin found for: '
                           '{} -> {}'.format(fromModel, toModel)))
                e.original_response = resp
                e.parameters = params
                raise e
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
215
        newentries = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
216
        resp.analysis = set(resp.analysis)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
217
218
        for i in resp.entries:
            if output == "full":
219
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
220
221
222
            else:
                newemotions = []
            for j in i.emotions:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
223
224
225
                plugname = j['prov:wasGeneratedBy']
                candidate = candidates[plugname]
                resp.analysis.add(candidate.id)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
226
227
228
229
230
231
232
233
234
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries

235
236
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
237
238
239
240
241
242
243
244
245
246
247
248
        candidate = self._default
        if not candidate:
            candidates = self.filter_plugins(is_activated=True)
            if len(candidates) > 0:
                candidate = list(candidates.values())[0]
        logger.debug("Default: {}".format(candidate))
        return candidate

    @default_plugin.setter
    def default_plugin(self, value):
        if isinstance(value, SenpyPlugin):
            self._default = value
249
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
250
            self._default = self.plugins[value]
251

252
253
254
255
256
257
258
259
260
261
262
263
264
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
265
266
267
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
268
269
270
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
271
272
273
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
274
275
276
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

277
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
278

279
        def act():
280
            success = False
281
282
            try:
                plugin.activate()
283
284
285
286
                msg = "Plugin activated: {}".format(plugin.name)
                logger.info(msg)
                success = True
                self._set_active_plugin(plugin_name, success)
287
            except Exception as ex:
288
                msg = "Error activating plugin {} - {} : \n\t{}".format(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
289
                    plugin.name, ex, traceback.format_exc())
290
291
                logger.error(msg)
                raise Error(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
292

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
293
        if sync or 'async' in plugin and not plugin.async:
294
            act()
295
        else:
296
297
            th = Thread(target=act)
            th.start()
298
299

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
300
301
302
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
303
304
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
305

306
307
        self._set_active_plugin(plugin_name, False)

308
309
310
311
312
        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
313
314
                logger.error(
                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
315
316
                logger.error("Trace: {}".format(traceback.format_exc()))

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
317
        if sync or 'async' in plugin and not plugin.async:
318
            deact()
319
        else:
320
321
            th = Thread(target=deact)
            th.start()
322

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
323
324
    @classmethod
    def validate_info(cls, info):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
325
        return all(x in info for x in ('name', 'module', 'description', 'version'))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
326

327
328
    def install_deps(self):
        for i in self.plugins.values():
329
            self._install_deps(i)
330
331
332
333
334
335
336

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
            pip_args = []
            pip_args.append('install')
337
            pip_args.append('--use-wheel')
338
            for req in requirements:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
339
                pip_args.append(req)
340
            logger.info('Installing requirements: ' + str(requirements))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
341
            pip.main(pip_args)
342

343
344
345
346
347
348
349
    @classmethod
    def _load_module(cls, name, root):
        sys.path.append(root)
        tmp = importlib.import_module(name)
        sys.path.remove(root)
        return tmp

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
350
351
352
353
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
354
            return None, None
355
356
        module = info["module"]
        name = info["name"]
357

358
        cls._install_deps(info)
359
360
        tmp = cls._load_module(module, root)

361
362
363
364
365
366
367
368
369
370
371
        candidate = None
        for _, obj in inspect.getmembers(tmp):
            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
                logger.debug(("Found plugin class:"
                              " {}@{}").format(obj, inspect.getmodule(obj)))
                candidate = obj
                break
        if not candidate:
            logger.debug("No valid plugin for: {}".format(module))
            return
        module = candidate(info=info)
372
        return name, module
373

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
374
375
376
377
378
379
380
381
382
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)

383
384
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
385
        for search_folder in self._search_folders:
386
387
388
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
389
                    if plugin and name:
390
                        plugins[name] = plugin
391

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
392
        self._outdated = False
393
394
395
396
397
398
399
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
400
        """ Return the plugins registered for a given application.  """
401
402
403
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
404

405
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
406
        """ Filter plugins by different criteria """
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
        ptype = kwargs.pop('plugin_type', None)
        logger.debug('#' * 100)
        logger.debug('ptype {}'.format(ptype))
        if ptype:
            try:
                ptype = ptype[0].upper() + ptype[1:]
                pclass = getattr(plugins, ptype)
                logger.debug('Class: {}'.format(pclass))
                candidates = filter(lambda x: isinstance(x, pclass),
                                    self.plugins.values())
            except AttributeError:
                raise Error('{} is not a valid type'.format(ptype))
        else:
            candidates = self.plugins.values()

        logger.debug(candidates)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
423

424
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
425
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
426
427
            logger.debug(
                "matching {} with {}: {}".format(plug.name, kwargs, res))
428
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
429

430
431
432
433
434
435
436
437
        if kwargs:
            candidates = filter(matches, candidates)
        return {p.name: p for p in candidates}

    @property
    def analysis_plugins(self):
        """ Return only the analysis plugins """
        return self.filter_plugins(plugin_type='analysisPlugin')