extensions.py 13.3 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2
3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
5
6
from future import standard_library
standard_library.install_aliases()
7

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8
9
10
from .plugins import SentimentPlugin, SenpyPlugin
from .models import Error, Entry, Results
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
11
from .api import API_PARAMS, NIF_PARAMS, parse_params
12

13
from threading import Thread
14

15
import os
16
17
import fnmatch
import inspect
18
import sys
19
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
20
import logging
21
import traceback
22
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23
import pip
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
24
25

logger = logging.getLogger(__name__)
26

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
27

28
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29
    """ Default Senpy extension for Flask """
30

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
31
32
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
33
                 plugin_folder=".",
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
34
                 default_plugins=False):
35
        self.app = app
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
36
        self._search_folders = set()
37
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
        self._outdated = True
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
39
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
40

41
        self.add_folder(plugin_folder)
42
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
43
44
45
46
47
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
48
49
50
51

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
52
53
54
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
55
56
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
57
        """
58
59
60
61
62
63
64
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
65
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
66
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
67
        app.register_blueprint(demo_blueprint, url_prefix="/")
68

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
69
70
71
    def add_folder(self, folder, from_root=False):
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
72
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
73
74
75
76
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
77
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
78

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
79
    def _find_plugin(self, params):
80
        api_params = parse_params(params, spec=API_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
81
        algo = None
82
83
        if "algorithm" in api_params and api_params["algorithm"]:
            algo = api_params["algorithm"]
84
        elif self.plugins:
85
            algo = self.default_plugin and self.default_plugin.name
86
        if not algo:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
87
88
89
90
            raise Error(
                status=404,
                message=("No plugins found."
                         " Please install one.").format(algo))
91
        if algo not in self.plugins:
92
93
94
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
95
96
97
            raise Error(
                status=404,
                message="The algorithm '{}' is not valid".format(algo))
98
99
100

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
101
102
103
104
            raise Error(
                status=400,
                message=("The algorithm '{}'"
                         " is not activated yet").format(algo))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
105
106
107
        return self.plugins[algo]

    def _get_params(self, params, plugin):
108
        nif_params = parse_params(params, spec=NIF_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109
        extra_params = plugin.get('extra_params', {})
110
111
        specific_params = parse_params(params, spec=extra_params)
        nif_params.update(specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
        return nif_params

    def _get_entries(self, params):
        entry = None
        if params['informat'] == 'text':
            entry = Entry(text=params['input'])
        else:
            raise NotImplemented('Only text input format implemented')
        yield entry

    def analyse(self, **api_params):
        logger.debug("analysing with params: {}".format(api_params))
        plugin = self._find_plugin(api_params)
        nif_params = self._get_params(api_params, plugin)
        resp = Results()
        if 'with_parameters' in api_params:
            resp.parameters = nif_params
129
        try:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
130
131
132
133
134
135
            entries = []
            for i in self._get_entries(nif_params):
                entries += list(plugin.analyse_entry(i, nif_params))
            resp.entries = entries
            self.convert_emotions(resp, plugin, nif_params)
            resp.analysis.append(plugin.id)
136
            logger.debug("Returning analysis result: {}".format(resp))
137
138
139
        except Error as ex:
            logger.exception('Error returning analysis result')
            resp = ex
140
        except Exception as ex:
141
            logger.exception('Error returning analysis result')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
142
            resp = Error(message=str(ex), status=500)
143
        return resp
144

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
    def _conversion_candidates(self, fromModel, toModel):
        candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
        for name, candidate in candidates.items():
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    # logging.debug('Found candidate: {}'.format(candidate))
                    yield candidate

    def convert_emotions(self, resp, plugin, params):
        """
        Conversion of all emotions in a response.
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
        fromModel = plugin.get('onyx:usesEmotionModel', None)
        toModel = params.get('emotionModel', None)
        output = params.get('conversion', None)
        logger.debug('Asked for model: {}'.format(toModel))
        logger.debug('Analysis plugin uses model: {}'.format(fromModel))

        if not toModel:
            return
        try:
            candidate = next(self._conversion_candidates(fromModel, toModel))
        except StopIteration:
            e = Error(('No conversion plugin found for: '
                       '{} -> {}'.format(fromModel, toModel)))
            e.original_response = resp
            e.parameters = params
            raise e
        newentries = []
        for i in resp.entries:
            if output == "full":
                newemotions = i.emotions.copy()
            else:
                newemotions = []
            for j in i.emotions:
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries
        resp.analysis.append(candidate.id)

197
198
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
199
200
201
202
203
204
205
206
207
208
209
210
        candidate = self._default
        if not candidate:
            candidates = self.filter_plugins(is_activated=True)
            if len(candidates) > 0:
                candidate = list(candidates.values())[0]
        logger.debug("Default: {}".format(candidate))
        return candidate

    @default_plugin.setter
    def default_plugin(self, value):
        if isinstance(value, SenpyPlugin):
            self._default = value
211
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
212
            self._default = self.plugins[value]
213

214
215
216
217
218
219
220
221
222
223
224
225
226
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
227
228
229
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
230
231
232
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
233
234
235
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
236
237
238
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

239
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
240

241
        def act():
242
            success = False
243
244
            try:
                plugin.activate()
245
246
247
248
                msg = "Plugin activated: {}".format(plugin.name)
                logger.info(msg)
                success = True
                self._set_active_plugin(plugin_name, success)
249
            except Exception as ex:
250
                msg = "Error activating plugin {} - {} : \n\t{}".format(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
251
                    plugin.name, ex, traceback.format_exc())
252
253
                logger.error(msg)
                raise Error(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
254

255
        if sync:
256
            act()
257
        else:
258
259
            th = Thread(target=act)
            th.start()
260
261

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
262
263
264
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
265
266
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
267

268
269
        self._set_active_plugin(plugin_name, False)

270
271
272
273
274
        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
275
276
                logger.error(
                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
277
278
                logger.error("Trace: {}".format(traceback.format_exc()))

279
        if sync:
280
            deact()
281
        else:
282
283
            th = Thread(target=deact)
            th.start()
284

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
285
286
    @classmethod
    def validate_info(cls, info):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
287
        return all(x in info for x in ('name', 'module', 'description', 'version'))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
288

289
290
    def install_deps(self):
        for i in self.plugins.values():
291
            self._install_deps(i)
292
293
294
295
296
297
298

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
            pip_args = []
            pip_args.append('install')
299
            pip_args.append('--use-wheel')
300
            for req in requirements:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
301
                pip_args.append(req)
302
            logger.info('Installing requirements: ' + str(requirements))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
303
            pip.main(pip_args)
304

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
305
306
307
308
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
309
            return None, None
310
311
        module = info["module"]
        name = info["name"]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
312
        sys.path.append(root)
313
        (fp, pathname, desc) = imp.find_module(module, [root, ])
314
315
316
317
318
319
320
321
322
323
324
325
326
327
        cls._install_deps(info)
        tmp = imp.load_module(module, fp, pathname, desc)
        sys.path.remove(root)
        candidate = None
        for _, obj in inspect.getmembers(tmp):
            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
                logger.debug(("Found plugin class:"
                              " {}@{}").format(obj, inspect.getmodule(obj)))
                candidate = obj
                break
        if not candidate:
            logger.debug("No valid plugin for: {}".format(module))
            return
        module = candidate(info=info)
328
        return name, module
329

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
330
331
332
333
334
335
336
337
338
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)

339
340
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
341
        for search_folder in self._search_folders:
342
343
344
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
345
                    if plugin and name:
346
                        plugins[name] = plugin
347

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
348
        self._outdated = False
349
350
351
352
353
354
355
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
356
        """ Return the plugins registered for a given application.  """
357
358
359
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
360

361
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
362
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
363

364
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
365
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
366
367
            logger.debug(
                "matching {} with {}: {}".format(plug.name, kwargs, res))
368
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
369

370
371
372
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
373
            return {n: p for n, p in self.plugins.items() if matches(p)}
374
375

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
376
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
377
378
379
380
381
        return {
            p: plugin
            for p, plugin in self.plugins.items()
            if isinstance(plugin, SentimentPlugin)
        }