extensions.py 14 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2
3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
5
6
from future import standard_library
standard_library.install_aliases()
7

8
9
from . import plugins
from .plugins import SenpyPlugin
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
10
11
from .models import Error, Entry, Results
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
12
from .api import API_PARAMS, NIF_PARAMS, parse_params
13

14
from threading import Thread
15

16
import os
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
17
import copy
18
19
import fnmatch
import inspect
20
import sys
21
import importlib
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
22
import logging
23
import traceback
24
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
import pip
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
26
27

logger = logging.getLogger(__name__)
28

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29

30
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
31
    """ Default Senpy extension for Flask """
32

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
33
34
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
35
                 plugin_folder=".",
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
36
                 default_plugins=False):
37
        self.app = app
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
        self._search_folders = set()
39
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
40
        self._outdated = True
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
41
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
42

43
        self.add_folder(plugin_folder)
44
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
45
46
47
48
49
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
50
51
52
53

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
54
55
56
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
57
58
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
59
        """
60
61
62
63
64
65
66
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
67
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
68
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
69
        app.register_blueprint(demo_blueprint, url_prefix="/")
70

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
71
72
73
    def add_folder(self, folder, from_root=False):
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
74
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
76
77
78
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
79
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
80

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
81
    def _find_plugin(self, params):
82
        api_params = parse_params(params, spec=API_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
83
        algo = None
84
85
        if "algorithm" in api_params and api_params["algorithm"]:
            algo = api_params["algorithm"]
86
        elif self.plugins:
87
            algo = self.default_plugin and self.default_plugin.name
88
        if not algo:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
89
90
91
92
            raise Error(
                status=404,
                message=("No plugins found."
                         " Please install one.").format(algo))
93
        if algo not in self.plugins:
94
95
96
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
97
98
99
            raise Error(
                status=404,
                message="The algorithm '{}' is not valid".format(algo))
100
101
102

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
103
104
105
106
            raise Error(
                status=400,
                message=("The algorithm '{}'"
                         " is not activated yet").format(algo))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
107
108
109
        return self.plugins[algo]

    def _get_params(self, params, plugin):
110
        nif_params = parse_params(params, spec=NIF_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
111
        extra_params = plugin.get('extra_params', {})
112
113
        specific_params = parse_params(params, spec=extra_params)
        nif_params.update(specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
        return nif_params

    def _get_entries(self, params):
        entry = None
        if params['informat'] == 'text':
            entry = Entry(text=params['input'])
        else:
            raise NotImplemented('Only text input format implemented')
        yield entry

    def analyse(self, **api_params):
        logger.debug("analysing with params: {}".format(api_params))
        plugin = self._find_plugin(api_params)
        nif_params = self._get_params(api_params, plugin)
        resp = Results()
        if 'with_parameters' in api_params:
            resp.parameters = nif_params
131
        try:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
132
133
134
135
136
137
            entries = []
            for i in self._get_entries(nif_params):
                entries += list(plugin.analyse_entry(i, nif_params))
            resp.entries = entries
            self.convert_emotions(resp, plugin, nif_params)
            resp.analysis.append(plugin.id)
138
            logger.debug("Returning analysis result: {}".format(resp))
139
140
141
        except Error as ex:
            logger.exception('Error returning analysis result')
            resp = ex
142
        except Exception as ex:
143
            logger.exception('Error returning analysis result')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
144
            resp = Error(message=str(ex), status=500)
145
        return resp
146

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
    def _conversion_candidates(self, fromModel, toModel):
        candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
        for name, candidate in candidates.items():
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    # logging.debug('Found candidate: {}'.format(candidate))
                    yield candidate

    def convert_emotions(self, resp, plugin, params):
        """
        Conversion of all emotions in a response.
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
        fromModel = plugin.get('onyx:usesEmotionModel', None)
        toModel = params.get('emotionModel', None)
        output = params.get('conversion', None)
        logger.debug('Asked for model: {}'.format(toModel))
        logger.debug('Analysis plugin uses model: {}'.format(fromModel))

        if not toModel:
            return
        try:
            candidate = next(self._conversion_candidates(fromModel, toModel))
        except StopIteration:
            e = Error(('No conversion plugin found for: '
                       '{} -> {}'.format(fromModel, toModel)))
            e.original_response = resp
            e.parameters = params
            raise e
        newentries = []
        for i in resp.entries:
            if output == "full":
185
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
186
187
188
189
190
191
192
193
194
195
196
197
198
            else:
                newemotions = []
            for j in i.emotions:
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries
        resp.analysis.append(candidate.id)

199
200
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
201
202
203
204
205
206
207
208
209
210
211
212
        candidate = self._default
        if not candidate:
            candidates = self.filter_plugins(is_activated=True)
            if len(candidates) > 0:
                candidate = list(candidates.values())[0]
        logger.debug("Default: {}".format(candidate))
        return candidate

    @default_plugin.setter
    def default_plugin(self, value):
        if isinstance(value, SenpyPlugin):
            self._default = value
213
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
214
            self._default = self.plugins[value]
215

216
217
218
219
220
221
222
223
224
225
226
227
228
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
229
230
231
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
232
233
234
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
235
236
237
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
238
239
240
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

241
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
242

243
        def act():
244
            success = False
245
246
            try:
                plugin.activate()
247
248
249
250
                msg = "Plugin activated: {}".format(plugin.name)
                logger.info(msg)
                success = True
                self._set_active_plugin(plugin_name, success)
251
            except Exception as ex:
252
                msg = "Error activating plugin {} - {} : \n\t{}".format(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
253
                    plugin.name, ex, traceback.format_exc())
254
255
                logger.error(msg)
                raise Error(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
256

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
257
        if sync or 'async' in plugin and not plugin.async:
258
            act()
259
        else:
260
261
            th = Thread(target=act)
            th.start()
262
263

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
264
265
266
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
267
268
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
269

270
271
        self._set_active_plugin(plugin_name, False)

272
273
274
275
276
        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
277
278
                logger.error(
                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
279
280
                logger.error("Trace: {}".format(traceback.format_exc()))

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
281
        if sync or 'async' in plugin and not plugin.async:
282
            deact()
283
        else:
284
285
            th = Thread(target=deact)
            th.start()
286

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
287
288
    @classmethod
    def validate_info(cls, info):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
289
        return all(x in info for x in ('name', 'module', 'description', 'version'))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
290

291
292
    def install_deps(self):
        for i in self.plugins.values():
293
            self._install_deps(i)
294
295
296
297
298
299
300

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
            pip_args = []
            pip_args.append('install')
301
            pip_args.append('--use-wheel')
302
            for req in requirements:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
303
                pip_args.append(req)
304
            logger.info('Installing requirements: ' + str(requirements))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
305
            pip.main(pip_args)
306

307
308
309
310
311
312
313
    @classmethod
    def _load_module(cls, name, root):
        sys.path.append(root)
        tmp = importlib.import_module(name)
        sys.path.remove(root)
        return tmp

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
314
315
316
317
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
318
            return None, None
319
320
        module = info["module"]
        name = info["name"]
321

322
        cls._install_deps(info)
323
324
        tmp = cls._load_module(module, root)

325
326
327
328
329
330
331
332
333
334
335
        candidate = None
        for _, obj in inspect.getmembers(tmp):
            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
                logger.debug(("Found plugin class:"
                              " {}@{}").format(obj, inspect.getmodule(obj)))
                candidate = obj
                break
        if not candidate:
            logger.debug("No valid plugin for: {}".format(module))
            return
        module = candidate(info=info)
336
        return name, module
337

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
338
339
340
341
342
343
344
345
346
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)

347
348
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
349
        for search_folder in self._search_folders:
350
351
352
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
353
                    if plugin and name:
354
                        plugins[name] = plugin
355

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
356
        self._outdated = False
357
358
359
360
361
362
363
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
364
        """ Return the plugins registered for a given application.  """
365
366
367
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
368

369
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
370
        """ Filter plugins by different criteria """
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
        ptype = kwargs.pop('plugin_type', None)
        logger.debug('#' * 100)
        logger.debug('ptype {}'.format(ptype))
        if ptype:
            try:
                ptype = ptype[0].upper() + ptype[1:]
                pclass = getattr(plugins, ptype)
                logger.debug('Class: {}'.format(pclass))
                candidates = filter(lambda x: isinstance(x, pclass),
                                    self.plugins.values())
            except AttributeError:
                raise Error('{} is not a valid type'.format(ptype))
        else:
            candidates = self.plugins.values()

        logger.debug(candidates)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
387

388
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
389
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
390
391
            logger.debug(
                "matching {} with {}: {}".format(plug.name, kwargs, res))
392
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
393

394
395
396
397
398
399
400
401
        if kwargs:
            candidates = filter(matches, candidates)
        return {p.name: p for p in candidates}

    @property
    def analysis_plugins(self):
        """ Return only the analysis plugins """
        return self.filter_plugins(plugin_type='analysisPlugin')