extensions.py 13.9 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2
3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
5
6
from future import standard_library
standard_library.install_aliases()
7

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8
from . import plugins, api
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
9
from .plugins import Plugin, evaluate
10
from .models import Error, AggregatedEvaluation
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
11
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
12

13
from threading import Thread
14
from functools import partial
15
import os
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
16
import copy
17
import errno
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
18
19
import logging

20

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21
from . import gsitk_compat
22

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23
logger = logging.getLogger(__name__)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
24

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25

26
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
27
    """ Default Senpy extension for Flask """
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
28
29
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
30
                 plugin_folder=".",
31
                 data_folder=None,
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
32
                 default_plugins=False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
33
34
35
36
37
38
39
40
41
42
43

        default_data = os.path.join(os.getcwd(), 'senpy_data')
        self.data_folder = data_folder or os.environ.get('SENPY_DATA', default_data)
        try:
            os.makedirs(self.data_folder)
        except OSError as e:
            if e.errno == errno.EEXIST:
                logger.debug('Data folder exists: {}'.format(self.data_folder))
            else:  # pragma: no cover
                raise

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
44
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
45
46
47
        self._plugins = {}
        if plugin_folder:
            self.add_folder(plugin_folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
48

49
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
50
51
52
53
54
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
55
        self.app = app
56
57
58
        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
59
60
61
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
62
63
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
64
        """
65
66
67
68
69
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
70
        else:  # pragma: no cover
71
            app.teardown_request(self.teardown)
72
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
73
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
74
        app.register_blueprint(demo_blueprint, url_prefix="/")
75

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
    def add_plugin(self, plugin):
        self._plugins[plugin.name.lower()] = plugin

    def delete_plugin(self, plugin):
        del self._plugins[plugin.name.lower()]

    def plugins(self, **kwargs):
        """ Return the plugins registered for a given application. Filtered by criteria  """
        return list(plugins.pfilter(self._plugins, **kwargs))

    def get_plugin(self, name, default=None):
        if name == 'default':
            return self.default_plugin
        plugin = name.lower()
        if plugin in self._plugins:
            return self._plugins[plugin]

J. Fernando Sánchez's avatar
WIP    
J. Fernando Sánchez committed
93
        results = self.plugins(id='endpoint:plugins/{}'.format(name))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
94
95
96
97
98
99
100
101
102
103

        if not results:
            return Error(message="Plugin not found", status=404)
        return results[0]

    @property
    def analysis_plugins(self):
        """ Return only the analysis plugins """
        return self.plugins(plugin_type='analysisPlugin')

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
104
    def add_folder(self, folder, from_root=False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
105
        """ Find plugins in this folder and add them to this instance """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
106
107
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
108
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109
        if os.path.isdir(folder):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
110
111
112
113
            new_plugins = plugins.from_folder([folder],
                                              data_folder=self.data_folder)
            for plugin in new_plugins:
                self.add_plugin(plugin)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
114
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
115
            raise AttributeError("Not a folder or does not exist: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
116

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
117
    def _get_plugins(self, request):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
118
        if not self.analysis_plugins:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
119
120
121
            raise Error(
                status=404,
                message=("No plugins found."
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
122
                         " Please install one."))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
123
124
125
126
127
128
129
130
        algos = request.parameters.get('algorithm', None)
        if not algos:
            if self.default_plugin:
                algos = [self.default_plugin.name, ]
            else:
                raise Error(
                    status=404,
                    message="No default plugin found, and None provided")
131

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
132
133
        plugins = list()
        for algo in algos:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
134
135
136
137
138
139
            algo = algo.lower()
            if algo not in self._plugins:
                msg = ("The algorithm '{}' is not valid\n"
                       "Valid algorithms: {}").format(algo,
                                                      self._plugins.keys())
                logger.debug(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
140
141
                raise Error(
                    status=404,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
142
143
                    message=msg)
            plugins.append(self._plugins[algo])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
144
145
        return plugins

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
146
    def _process_entries(self, entries, req, plugins):
147
148
149
150
        """
        Recursively process the entries with the first plugin in the list, and pass the results
        to the rest of the plugins.
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
151
152
153
154
155
        if not plugins:
            for i in entries:
                yield i
            return
        plugin = plugins[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
156
        specific_params = api.parse_extra_params(req, plugin)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
157
158
        req.analysis.append({'plugin': plugin,
                             'parameters': specific_params})
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
159
        results = plugin.analyse_entries(entries, specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
160
        for i in self._process_entries(results, req, plugins[1:]):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
161
162
            yield i

163
    def install_deps(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
164
        plugins.install_deps(*self.plugins())
165

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
166
    def analyse(self, request):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
167
168
        """
        Main method that analyses a request, either from CLI or HTTP.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
169
170
        It takes a processed request, provided by the user, as returned
        by api.parse_call().
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
171
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
172
        logger.debug("analysing request: {}".format(request))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
173
174
175
176
177
178
179
180
        entries = request.entries
        request.entries = []
        plugins = self._get_plugins(request)
        results = request
        for i in self._process_entries(entries, results, plugins):
            results.entries.append(i)
        self.convert_emotions(results)
        logger.debug("Returning analysis result: {}".format(results))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
181
182
        results.analysis = [i['plugin'].id for i in results.analysis]
        return results
183

184
185
186
187
188
189
190
191
192
193
194
    def _get_datasets(self, request):
        if not self.datasets:
            raise Error(
                status=404,
                message=("No datasets found."
                         " Please verify DatasetManager"))
        datasets_name = request.parameters.get('dataset', None).split(',')
        for dataset in datasets_name:
            if dataset not in self.datasets:
                logger.debug(("The dataset '{}' is not valid\n"
                              "Valid datasets: {}").format(dataset,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
195
                                                           self.datasets.keys()))
196
197
198
                raise Error(
                    status=404,
                    message="The dataset '{}' is not valid".format(dataset))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
199
        dm = gsitk_compat.DatasetManager()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
200
        datasets = dm.prepare_datasets(datasets_name)
201
        return datasets
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
202

203
204
205
    @property
    def datasets(self):
        self._dataset_list = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
206
        dm = gsitk_compat.DatasetManager()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
207
        for item in dm.get_datasets():
208
209
210
211
212
213
214
215
216
217
            for key in item:
                if key in self._dataset_list:
                    continue
                properties = item[key]
                properties['@id'] = key
                self._dataset_list[key] = properties
        return self._dataset_list

    def evaluate(self, params):
        logger.debug("evaluating request: {}".format(params))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
218
219
220
221
222
223
224
225
226
        results = AggregatedEvaluation()
        results.parameters = params
        datasets = self._get_datasets(results)
        plugins = self._get_plugins(results)
        for eval in evaluate(plugins, datasets):
            results.evaluations.append(eval)
        if 'with_parameters' not in results.parameters:
            del results.parameters
        logger.debug("Returning evaluation result: {}".format(results))
227
228
        return results

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
229
    def _conversion_candidates(self, fromModel, toModel):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
230
231
        candidates = self.plugins(plugin_type='emotionConversionPlugin')
        for candidate in candidates:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
232
233
234
235
236
237
238
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    yield candidate

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
239
    def convert_emotions(self, resp):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
240
        """
241
        Conversion of all emotions in a response **in place**.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
242
243
244
245
246
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
247
248
        plugins = [i['plugin'] for i in resp.analysis]
        params = resp.parameters
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
249
250
251
        toModel = params.get('emotionModel', None)
        if not toModel:
            return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
252
253
254
255
256
257
258
259
260
261
262

        logger.debug('Asked for model: {}'.format(toModel))
        output = params.get('conversion', None)
        candidates = {}
        for plugin in plugins:
            try:
                fromModel = plugin.get('onyx:usesEmotionModel', None)
                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
            except StopIteration:
                e = Error(('No conversion plugin found for: '
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
263
264
                           '{} -> {}'.format(fromModel, toModel)),
                          status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
265
266
267
                e.original_response = resp
                e.parameters = params
                raise e
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
268
269
270
        newentries = []
        for i in resp.entries:
            if output == "full":
271
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
272
273
274
            else:
                newemotions = []
            for j in i.emotions:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
275
276
                plugname = j['prov:wasGeneratedBy']
                candidate = candidates[plugname]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
277
278
                resp.analysis.append({'plugin': candidate,
                                      'parameters': params})
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
279
280
281
282
283
284
285
286
287
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries

288
289
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
290
291
292
        if not self._default or not self._default.is_activated:
            candidates = self.plugins(plugin_type='analysisPlugin',
                                      is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
293
            if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
294
295
296
297
298
                self._default = candidates[0]
            else:
                self._default = None
            logger.debug("Default: {}".format(self._default))
        return self._default
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
299
300
301

    @default_plugin.setter
    def default_plugin(self, value):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
302
303
304
        if isinstance(value, Plugin):
            if not value.is_activated:
                raise AttributeError('The default plugin has to be activated.')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
305
            self._default = value
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
306

307
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
308
            self._default = self._plugins[value.lower()]
309

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
310
    def activate_all(self, sync=True, allow_fail=False):
311
        ps = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
312
        for plug in self._plugins.keys():
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
313
314
315
316
317
318
            try:
                self.activate_plugin(plug, sync=sync)
            except Exception as ex:
                if not allow_fail:
                    raise
                logger.error('Could not activate {}: {}'.format(plug, ex))
319
320
        return ps

321
    def deactivate_all(self, sync=True):
322
        ps = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
323
        for plug in self._plugins.keys():
324
325
326
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

327
    def _set_active(self, plugin, active=True, *args, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
328
        ''' We're using a variable in the plugin itself to activate/deactivate plugins.\
329
330
        Note that plugins may activate themselves by setting this variable.
        '''
331
        plugin.is_activated = active
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
332

333
334
335
336
337
    def _activate(self, plugin):
        success = False
        with plugin._lock:
            if plugin.is_activated:
                return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
338
339
340
341
342
            plugin.activate()
            msg = "Plugin activated: {}".format(plugin.name)
            logger.info(msg)
            success = True
            self._set_active(plugin, success)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
343
        return success
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
344

345
    def activate_plugin(self, plugin_name, sync=True):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
346
347
        plugin_name = plugin_name.lower()
        if plugin_name not in self._plugins:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
348
349
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
350
        plugin = self._plugins[plugin_name]
351

352
353
        logger.info("Activating plugin: {}".format(plugin.name))

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
354
        if sync or not getattr(plugin, 'async', True) or getattr(plugin, 'sync', False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
355
            return self._activate(plugin)
356
357
358
359
        else:
            th = Thread(target=partial(self._activate, plugin))
            th.start()
            return th
360

361
362
363
364
    def _deactivate(self, plugin):
        with plugin._lock:
            if not plugin.is_activated:
                return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
365
366
            plugin.deactivate()
            logger.info("Plugin deactivated: {}".format(plugin.name))
367

368
    def deactivate_plugin(self, plugin_name, sync=True):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
369
370
        plugin_name = plugin_name.lower()
        if plugin_name not in self._plugins:
371
372
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
373
        plugin = self._plugins[plugin_name]
374
375
376

        self._set_active(plugin, False)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
377
        if sync or not getattr(plugin, 'async', True) or not getattr(plugin, 'sync', False):
378
            self._deactivate(plugin)
379
        else:
380
            th = Thread(target=partial(self._deactivate, plugin))
381
            th.start()
382
            return th
383

384
385
    def teardown(self, exception):
        pass