extensions.py 13.4 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2
3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
5

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
6
7
8
from .plugins import SentimentPlugin, SenpyPlugin
from .models import Error, Entry, Results
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
9
from .api import API_PARAMS, NIF_PARAMS, parse_params
10

11
from threading import Thread
12

13
import os
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
14
import copy
15
16
import fnmatch
import inspect
17
import sys
18
import importlib
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
19
import logging
20
import traceback
21
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
22
import pip
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23
24

logger = logging.getLogger(__name__)
25

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
26

27
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
28
    """ Default Senpy extension for Flask """
29

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
30
31
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
32
                 plugin_folder=".",
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
33
                 default_plugins=False):
34
        self.app = app
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
35
        self._search_folders = set()
36
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
37
        self._outdated = True
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
39

40
        self.add_folder(plugin_folder)
41
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
42
43
44
45
46
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
47
48
49
50

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
51
52
53
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
54
55
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
56
        """
57
58
59
60
61
62
63
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
64
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
65
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
66
        app.register_blueprint(demo_blueprint, url_prefix="/")
67

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
68
69
70
    def add_folder(self, folder, from_root=False):
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
71
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
72
73
74
75
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
76
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
77

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
78
    def _find_plugin(self, params):
79
        api_params = parse_params(params, spec=API_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
80
        algo = None
81
82
        if "algorithm" in api_params and api_params["algorithm"]:
            algo = api_params["algorithm"]
83
        elif self.plugins:
84
            algo = self.default_plugin and self.default_plugin.name
85
        if not algo:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
86
87
88
89
            raise Error(
                status=404,
                message=("No plugins found."
                         " Please install one.").format(algo))
90
        if algo not in self.plugins:
91
92
93
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
94
95
96
            raise Error(
                status=404,
                message="The algorithm '{}' is not valid".format(algo))
97
98
99

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
100
101
102
103
            raise Error(
                status=400,
                message=("The algorithm '{}'"
                         " is not activated yet").format(algo))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
104
105
106
        return self.plugins[algo]

    def _get_params(self, params, plugin):
107
        nif_params = parse_params(params, spec=NIF_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
108
        extra_params = plugin.get('extra_params', {})
109
110
        specific_params = parse_params(params, spec=extra_params)
        nif_params.update(specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
        return nif_params

    def _get_entries(self, params):
        entry = None
        if params['informat'] == 'text':
            entry = Entry(text=params['input'])
        else:
            raise NotImplemented('Only text input format implemented')
        yield entry

    def analyse(self, **api_params):
        logger.debug("analysing with params: {}".format(api_params))
        plugin = self._find_plugin(api_params)
        nif_params = self._get_params(api_params, plugin)
        resp = Results()
        if 'with_parameters' in api_params:
            resp.parameters = nif_params
128
        try:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
129
130
131
132
133
134
            entries = []
            for i in self._get_entries(nif_params):
                entries += list(plugin.analyse_entry(i, nif_params))
            resp.entries = entries
            self.convert_emotions(resp, plugin, nif_params)
            resp.analysis.append(plugin.id)
135
            logger.debug("Returning analysis result: {}".format(resp))
136
137
138
        except Error as ex:
            logger.exception('Error returning analysis result')
            resp = ex
139
        except Exception as ex:
140
            logger.exception('Error returning analysis result')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
141
            resp = Error(message=str(ex), status=500)
142
        return resp
143

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
    def _conversion_candidates(self, fromModel, toModel):
        candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
        for name, candidate in candidates.items():
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    # logging.debug('Found candidate: {}'.format(candidate))
                    yield candidate

    def convert_emotions(self, resp, plugin, params):
        """
        Conversion of all emotions in a response.
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
        fromModel = plugin.get('onyx:usesEmotionModel', None)
        toModel = params.get('emotionModel', None)
        output = params.get('conversion', None)
        logger.debug('Asked for model: {}'.format(toModel))
        logger.debug('Analysis plugin uses model: {}'.format(fromModel))

        if not toModel:
            return
        try:
            candidate = next(self._conversion_candidates(fromModel, toModel))
        except StopIteration:
            e = Error(('No conversion plugin found for: '
                       '{} -> {}'.format(fromModel, toModel)))
            e.original_response = resp
            e.parameters = params
            raise e
        newentries = []
        for i in resp.entries:
            if output == "full":
182
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
183
184
185
186
187
188
189
190
191
192
193
194
195
            else:
                newemotions = []
            for j in i.emotions:
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries
        resp.analysis.append(candidate.id)

196
197
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
198
199
200
201
202
203
204
205
206
207
208
209
        candidate = self._default
        if not candidate:
            candidates = self.filter_plugins(is_activated=True)
            if len(candidates) > 0:
                candidate = list(candidates.values())[0]
        logger.debug("Default: {}".format(candidate))
        return candidate

    @default_plugin.setter
    def default_plugin(self, value):
        if isinstance(value, SenpyPlugin):
            self._default = value
210
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
211
            self._default = self.plugins[value]
212

213
214
215
216
217
218
219
220
221
222
223
224
225
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
226
227
228
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
229
230
231
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
232
233
234
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
235
236
237
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

238
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
239

240
        def act():
241
            success = False
242
243
            try:
                plugin.activate()
244
245
246
247
                msg = "Plugin activated: {}".format(plugin.name)
                logger.info(msg)
                success = True
                self._set_active_plugin(plugin_name, success)
248
            except Exception as ex:
249
                msg = "Error activating plugin {} - {} : \n\t{}".format(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
250
                    plugin.name, ex, traceback.format_exc())
251
252
                logger.error(msg)
                raise Error(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
253

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
254
        if sync or 'async' in plugin and not plugin.async:
255
            act()
256
        else:
257
258
            th = Thread(target=act)
            th.start()
259
260

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
261
262
263
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
264
265
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
266

267
268
        self._set_active_plugin(plugin_name, False)

269
270
271
272
273
        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
274
275
                logger.error(
                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
276
277
                logger.error("Trace: {}".format(traceback.format_exc()))

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
278
        if sync or 'async' in plugin and not plugin.async:
279
            deact()
280
        else:
281
282
            th = Thread(target=deact)
            th.start()
283

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
284
285
    @classmethod
    def validate_info(cls, info):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
286
        return all(x in info for x in ('name', 'module', 'description', 'version'))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
287

288
289
    def install_deps(self):
        for i in self.plugins.values():
290
            self._install_deps(i)
291
292
293
294
295
296
297

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
            pip_args = []
            pip_args.append('install')
298
            pip_args.append('--use-wheel')
299
            for req in requirements:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
300
                pip_args.append(req)
301
            logger.info('Installing requirements: ' + str(requirements))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
302
            pip.main(pip_args)
303

304
305
306
307
308
309
310
    @classmethod
    def _load_module(cls, name, root):
        sys.path.append(root)
        tmp = importlib.import_module(name)
        sys.path.remove(root)
        return tmp

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
311
312
313
314
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
315
            return None, None
316
317
        module = info["module"]
        name = info["name"]
318

319
        cls._install_deps(info)
320
321
        tmp = cls._load_module(module, root)

322
323
324
325
326
327
328
329
330
331
332
        candidate = None
        for _, obj in inspect.getmembers(tmp):
            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
                logger.debug(("Found plugin class:"
                              " {}@{}").format(obj, inspect.getmodule(obj)))
                candidate = obj
                break
        if not candidate:
            logger.debug("No valid plugin for: {}".format(module))
            return
        module = candidate(info=info)
333
        return name, module
334

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
335
336
337
338
339
340
341
342
343
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)

344
345
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
346
        for search_folder in self._search_folders:
347
348
349
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
350
                    if plugin and name:
351
                        plugins[name] = plugin
352

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
353
        self._outdated = False
354
355
356
357
358
359
360
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
361
        """ Return the plugins registered for a given application.  """
362
363
364
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
365

366
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
367
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
368

369
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
370
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
371
372
            logger.debug(
                "matching {} with {}: {}".format(plug.name, kwargs, res))
373
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
374

375
376
377
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
378
            return {n: p for n, p in self.plugins.items() if matches(p)}
379
380

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
381
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
382
383
384
385
386
        return {
            p: plugin
            for p, plugin in self.plugins.items()
            if isinstance(plugin, SentimentPlugin)
        }