extensions.py 14.3 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2
3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
5
6
from future import standard_library
standard_library.install_aliases()
7

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8
from . import plugins, api
9
from .models import Error, AggregatedEvaluation
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
10
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
11

12
from threading import Thread
13
from functools import partial
14
import os
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
15
import copy
16
import errno
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
17
18
import logging

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
19
from . import gsitk_compat
20

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21
logger = logging.getLogger(__name__)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
22

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23

24
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
    """ Default Senpy extension for Flask """
26

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
27
28
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29
                 plugin_folder=".",
30
                 data_folder=None,
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
31
                 default_plugins=False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
32
33
34
35
36
37
38
39
40
41
42

        default_data = os.path.join(os.getcwd(), 'senpy_data')
        self.data_folder = data_folder or os.environ.get('SENPY_DATA', default_data)
        try:
            os.makedirs(self.data_folder)
        except OSError as e:
            if e.errno == errno.EEXIST:
                logger.debug('Data folder exists: {}'.format(self.data_folder))
            else:  # pragma: no cover
                raise

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
43
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
44
45
46
        self._plugins = {}
        if plugin_folder:
            self.add_folder(plugin_folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
47

48
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
49
50
51
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
52
            self.add_folder(os.path.join('plugins', 'postprocessing'),
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
53
                            from_root=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
54
        self.app = app
55
56
57
        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
58
59
60
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
61
62
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
63
        """
64
65
66
67
68
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
69
        else:  # pragma: no cover
70
            app.teardown_request(self.teardown)
71
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
72
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
73
        app.register_blueprint(demo_blueprint, url_prefix="/")
74

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
    def add_plugin(self, plugin):
        self._plugins[plugin.name.lower()] = plugin

    def delete_plugin(self, plugin):
        del self._plugins[plugin.name.lower()]

    def plugins(self, **kwargs):
        """ Return the plugins registered for a given application. Filtered by criteria  """
        return list(plugins.pfilter(self._plugins, **kwargs))

    def get_plugin(self, name, default=None):
        if name == 'default':
            return self.default_plugin
        plugin = name.lower()
        if plugin in self._plugins:
            return self._plugins[plugin]

J. Fernando Sánchez's avatar
WIP    
J. Fernando Sánchez committed
92
        results = self.plugins(id='endpoint:plugins/{}'.format(name))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
93
94
95
96
97
98
99
100
101
102

        if not results:
            return Error(message="Plugin not found", status=404)
        return results[0]

    @property
    def analysis_plugins(self):
        """ Return only the analysis plugins """
        return self.plugins(plugin_type='analysisPlugin')

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
103
    def add_folder(self, folder, from_root=False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
104
        """ Find plugins in this folder and add them to this instance """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
105
106
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
107
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
108
        if os.path.isdir(folder):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109
110
111
112
            new_plugins = plugins.from_folder([folder],
                                              data_folder=self.data_folder)
            for plugin in new_plugins:
                self.add_plugin(plugin)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
113
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
114
            raise AttributeError("Not a folder or does not exist: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
115

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
116
    def _get_plugins(self, request):
117
        '''Get a list of plugins that should be run for a specific request'''
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
118
        if not self.analysis_plugins:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
119
120
121
            raise Error(
                status=404,
                message=("No plugins found."
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
122
                         " Please install one."))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
123
124
125
126
127
128
129
130
        algos = request.parameters.get('algorithm', None)
        if not algos:
            if self.default_plugin:
                algos = [self.default_plugin.name, ]
            else:
                raise Error(
                    status=404,
                    message="No default plugin found, and None provided")
131

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
132
133
        plugins = list()
        for algo in algos:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
134
            algo = algo.lower()
135
136
            if algo == 'conversion':
                continue  # Allow 'conversion' as a virtual plugin, which does nothing
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
137
138
139
140
141
            if algo not in self._plugins:
                msg = ("The algorithm '{}' is not valid\n"
                       "Valid algorithms: {}").format(algo,
                                                      self._plugins.keys())
                logger.debug(msg)
142
                raise Error(status=404, message=msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
143
            plugins.append(self._plugins[algo])
144

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
145
146
        return plugins

147
    def _process(self, req, pending, done=None):
148
149
150
151
        """
        Recursively process the entries with the first plugin in the list, and pass the results
        to the rest of the plugins.
        """
152
153
154
155
156
157
158
159
160
        done = done or []
        if not pending:
            return req

        plugin = pending[0]
        results = plugin.process(req, conversions_applied=done)
        if plugin not in results.analysis:
            results.analysis.append(plugin)
        return self._process(results, pending[1:], done)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
161

162
    def install_deps(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
163
        plugins.install_deps(*self.plugins())
164

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
165
    def analyse(self, request):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
166
167
        """
        Main method that analyses a request, either from CLI or HTTP.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
168
169
        It takes a processed request, provided by the user, as returned
        by api.parse_call().
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
170
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
171
        logger.debug("analysing request: {}".format(request))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
172
        plugins = self._get_plugins(request)
173
174
175
176
177
        request.parameters = api.parse_extra_params(request, plugins)
        results = self._process(request, plugins)
        logger.debug("Got analysis result: {}".format(results))
        results = self.postprocess(results)
        logger.debug("Returning post-processed result: {}".format(results))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
178
        return results
179

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
180
    def convert_emotions(self, resp):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
181
        """
182
        Conversion of all emotions in a response **in place**.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
183
184
185
186
187
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
188
189
        plugins = resp.analysis

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
190
        params = resp.parameters
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
191
192
        toModel = params.get('emotionModel', None)
        if not toModel:
193
            return resp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
194
195
196
197
198
199
200
201

        logger.debug('Asked for model: {}'.format(toModel))
        output = params.get('conversion', None)
        candidates = {}
        for plugin in plugins:
            try:
                fromModel = plugin.get('onyx:usesEmotionModel', None)
                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
202
203
                logger.debug('Analysis plugin {} uses model: {}'.format(
                    plugin.id, fromModel))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
204
205
            except StopIteration:
                e = Error(('No conversion plugin found for: '
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
206
207
                           '{} -> {}'.format(fromModel, toModel)),
                          status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
208
209
210
                e.original_response = resp
                e.parameters = params
                raise e
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
211
        newentries = []
212
        done = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
213
214
        for i in resp.entries:
            if output == "full":
215
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
216
217
218
            else:
                newemotions = []
            for j in i.emotions:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
219
220
                plugname = j['prov:wasGeneratedBy']
                candidate = candidates[plugname]
221
                done.append({'plugin': candidate, 'parameters': params})
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
222
223
224
225
226
227
228
229
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
        return resp

    def _conversion_candidates(self, fromModel, toModel):
        candidates = self.plugins(plugin_type=plugins.EmotionConversion)
        for candidate in candidates:
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)
                if candidate.can_convert(fromModel, toModel):
                    yield candidate

    def postprocess(self, response):
        '''
        Transform the results from the analysis plugins.
        It has some pre-defined post-processing like emotion conversion,
        and it also allows plugins to auto-select themselves.
        '''

        response = self.convert_emotions(response)

        for plug in self.plugins(plugin_type=plugins.PostProcessing):
            if plug.check(response, response.analysis):
                response = plug.process(response)
        return response

    def _get_datasets(self, request):
        if not self.datasets:
            raise Error(
                status=404,
                message=("No datasets found."
                         " Please verify DatasetManager"))
        datasets_name = request.parameters.get('dataset', None).split(',')
        for dataset in datasets_name:
            if dataset not in self.datasets:
                logger.debug(("The dataset '{}' is not valid\n"
                              "Valid datasets: {}").format(
                                  dataset, self.datasets.keys()))
                raise Error(
                    status=404,
                    message="The dataset '{}' is not valid".format(dataset))
        dm = gsitk_compat.DatasetManager()
        datasets = dm.prepare_datasets(datasets_name)
        return datasets

    @property
    def datasets(self):
        self._dataset_list = {}
        dm = gsitk_compat.DatasetManager()
        for item in dm.get_datasets():
            for key in item:
                if key in self._dataset_list:
                    continue
                properties = item[key]
                properties['@id'] = key
                self._dataset_list[key] = properties
        return self._dataset_list

    def evaluate(self, params):
        logger.debug("evaluating request: {}".format(params))
        results = AggregatedEvaluation()
        results.parameters = params
        datasets = self._get_datasets(results)
        plugins = self._get_plugins(results)
        for eval in plugins.evaluate(plugins, datasets):
            results.evaluations.append(eval)
        if 'with_parameters' not in results.parameters:
            del results.parameters
        logger.debug("Returning evaluation result: {}".format(results))
        return results
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
298

299
300
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
301
        if not self._default or not self._default.is_activated:
302
303
            candidates = self.plugins(
                plugin_type='analysisPlugin', is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
304
            if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
305
306
307
308
309
                self._default = candidates[0]
            else:
                self._default = None
            logger.debug("Default: {}".format(self._default))
        return self._default
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
310
311
312

    @default_plugin.setter
    def default_plugin(self, value):
313
        if isinstance(value, plugins.Plugin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
314
315
            if not value.is_activated:
                raise AttributeError('The default plugin has to be activated.')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
316
            self._default = value
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
317

318
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
319
            self._default = self._plugins[value.lower()]
320

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
321
    def activate_all(self, sync=True, allow_fail=False):
322
        ps = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
323
        for plug in self._plugins.keys():
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
324
325
326
327
328
329
            try:
                self.activate_plugin(plug, sync=sync)
            except Exception as ex:
                if not allow_fail:
                    raise
                logger.error('Could not activate {}: {}'.format(plug, ex))
330
331
        return ps

332
    def deactivate_all(self, sync=True):
333
        ps = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
334
        for plug in self._plugins.keys():
335
336
337
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

338
    def _set_active(self, plugin, active=True, *args, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
339
        ''' We're using a variable in the plugin itself to activate/deactivate plugins.\
340
341
        Note that plugins may activate themselves by setting this variable.
        '''
342
        plugin.is_activated = active
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
343

344
345
346
347
348
    def _activate(self, plugin):
        success = False
        with plugin._lock:
            if plugin.is_activated:
                return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
349
350
351
352
353
            plugin.activate()
            msg = "Plugin activated: {}".format(plugin.name)
            logger.info(msg)
            success = True
            self._set_active(plugin, success)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
354
        return success
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
355

356
    def activate_plugin(self, plugin_name, sync=True):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
357
358
        plugin_name = plugin_name.lower()
        if plugin_name not in self._plugins:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
359
360
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
361
        plugin = self._plugins[plugin_name]
362

363
364
        logger.info("Activating plugin: {}".format(plugin.name))

365
366
        if sync or not getattr(plugin, 'async', True) or getattr(
                plugin, 'sync', False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
367
            return self._activate(plugin)
368
369
370
371
        else:
            th = Thread(target=partial(self._activate, plugin))
            th.start()
            return th
372

373
374
375
376
    def _deactivate(self, plugin):
        with plugin._lock:
            if not plugin.is_activated:
                return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
377
378
            plugin.deactivate()
            logger.info("Plugin deactivated: {}".format(plugin.name))
379

380
    def deactivate_plugin(self, plugin_name, sync=True):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
381
382
        plugin_name = plugin_name.lower()
        if plugin_name not in self._plugins:
383
384
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
385
        plugin = self._plugins[plugin_name]
386
387
388

        self._set_active(plugin, False)

389
390
        if sync or not getattr(plugin, 'async', True) or not getattr(
                plugin, 'sync', False):
391
            self._deactivate(plugin)
392
        else:
393
            th = Thread(target=partial(self._deactivate, plugin))
394
            th.start()
395
            return th
396

397
398
    def teardown(self, exception):
        pass