extensions.py 14.4 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2
3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
5
6
from future import standard_library
standard_library.install_aliases()
7

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8
from . import plugins, api
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
9
from .plugins import Plugin, evaluate
10
from .models import Error, AggregatedEvaluation
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
11
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
12

13
from threading import Thread
14
from functools import partial
15
import os
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
16
import copy
17
import errno
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
18
19
import logging

20

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21
logger = logging.getLogger(__name__)
22

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23
24
25
26
27
28
29
try:
    from gsitk.datasets.datasets import DatasetManager
    GSITK_AVAILABLE = True
except ImportError:
    logger.warn('GSITK is not installed. Some functions will be unavailable.')
    GSITK_AVAILABLE = False

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
30

31
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
32
    """ Default Senpy extension for Flask """
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
33
34
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
35
                 plugin_folder=".",
36
                 data_folder=None,
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
37
                 default_plugins=False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
39
40
41
42
43
44
45
46
47
48

        default_data = os.path.join(os.getcwd(), 'senpy_data')
        self.data_folder = data_folder or os.environ.get('SENPY_DATA', default_data)
        try:
            os.makedirs(self.data_folder)
        except OSError as e:
            if e.errno == errno.EEXIST:
                logger.debug('Data folder exists: {}'.format(self.data_folder))
            else:  # pragma: no cover
                raise

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
49
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
50
51
52
        self._plugins = {}
        if plugin_folder:
            self.add_folder(plugin_folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
53

54
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
55
56
57
58
59
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
60
        self.app = app
61
62
63
        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
64
65
66
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
67
68
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
69
        """
70
71
72
73
74
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
        else:  # pragma: no cover
76
            app.teardown_request(self.teardown)
77
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
78
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
79
        app.register_blueprint(demo_blueprint, url_prefix="/")
80

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    def add_plugin(self, plugin):
        self._plugins[plugin.name.lower()] = plugin

    def delete_plugin(self, plugin):
        del self._plugins[plugin.name.lower()]

    def plugins(self, **kwargs):
        """ Return the plugins registered for a given application. Filtered by criteria  """
        return list(plugins.pfilter(self._plugins, **kwargs))

    def get_plugin(self, name, default=None):
        if name == 'default':
            return self.default_plugin
        plugin = name.lower()
        if plugin in self._plugins:
            return self._plugins[plugin]

        results = self.plugins(id='plugins/{}'.format(name))

        if not results:
            return Error(message="Plugin not found", status=404)
        return results[0]

    @property
    def analysis_plugins(self):
        """ Return only the analysis plugins """
        return self.plugins(plugin_type='analysisPlugin')

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109
    def add_folder(self, folder, from_root=False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
110
        """ Find plugins in this folder and add them to this instance """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
111
112
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
113
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
114
        if os.path.isdir(folder):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
115
116
117
118
            new_plugins = plugins.from_folder([folder],
                                              data_folder=self.data_folder)
            for plugin in new_plugins:
                self.add_plugin(plugin)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
119
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
120
            raise AttributeError("Not a folder or does not exist: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
121

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
122
    def _get_plugins(self, request):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
123
        if not self.analysis_plugins:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
124
125
126
            raise Error(
                status=404,
                message=("No plugins found."
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
127
                         " Please install one."))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
128
129
130
131
132
133
134
135
        algos = request.parameters.get('algorithm', None)
        if not algos:
            if self.default_plugin:
                algos = [self.default_plugin.name, ]
            else:
                raise Error(
                    status=404,
                    message="No default plugin found, and None provided")
136

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
137
138
        plugins = list()
        for algo in algos:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
139
140
141
142
143
144
            algo = algo.lower()
            if algo not in self._plugins:
                msg = ("The algorithm '{}' is not valid\n"
                       "Valid algorithms: {}").format(algo,
                                                      self._plugins.keys())
                logger.debug(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
145
146
                raise Error(
                    status=404,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
147
148
                    message=msg)
            plugins.append(self._plugins[algo])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
149
150
        return plugins

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
151
    def _process_entries(self, entries, req, plugins):
152
153
154
155
        """
        Recursively process the entries with the first plugin in the list, and pass the results
        to the rest of the plugins.
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
156
157
158
159
160
        if not plugins:
            for i in entries:
                yield i
            return
        plugin = plugins[0]
161
        self._activate(plugin)  # Make sure the plugin is activated
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
162
        specific_params = api.parse_extra_params(req, plugin)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
163
164
        req.analysis.append({'plugin': plugin,
                             'parameters': specific_params})
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
165
        results = plugin.analyse_entries(entries, specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
166
        for i in self._process_entries(results, req, plugins[1:]):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
167
168
            yield i

169
    def install_deps(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
170
        for plugin in self.plugins(is_activated=True):
171
172
            plugins.install_deps(plugin)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
173
    def analyse(self, request):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
174
175
        """
        Main method that analyses a request, either from CLI or HTTP.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
176
177
        It takes a processed request, provided by the user, as returned
        by api.parse_call().
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
178
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
179
        logger.debug("analysing request: {}".format(request))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
180
181
182
183
184
185
186
187
        entries = request.entries
        request.entries = []
        plugins = self._get_plugins(request)
        results = request
        for i in self._process_entries(entries, results, plugins):
            results.entries.append(i)
        self.convert_emotions(results)
        logger.debug("Returning analysis result: {}".format(results))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
188
189
        results.analysis = [i['plugin'].id for i in results.analysis]
        return results
190

191
192
193
194
195
196
197
198
199
200
201
    def _get_datasets(self, request):
        if not self.datasets:
            raise Error(
                status=404,
                message=("No datasets found."
                         " Please verify DatasetManager"))
        datasets_name = request.parameters.get('dataset', None).split(',')
        for dataset in datasets_name:
            if dataset not in self.datasets:
                logger.debug(("The dataset '{}' is not valid\n"
                              "Valid datasets: {}").format(dataset,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
202
                                                           self.datasets.keys()))
203
204
205
                raise Error(
                    status=404,
                    message="The dataset '{}' is not valid".format(dataset))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
206
207
        dm = DatasetManager()
        datasets = dm.prepare_datasets(datasets_name)
208
        return datasets
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
209

210
211
    @property
    def datasets(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
212
213
        if not GSITK_AVAILABLE:
            raise Exception('GSITK is not available. Install it to use this function.')
214
        self._dataset_list = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
215
216
        dm = DatasetManager()
        for item in dm.get_datasets():
217
218
219
220
221
222
223
224
225
            for key in item:
                if key in self._dataset_list:
                    continue
                properties = item[key]
                properties['@id'] = key
                self._dataset_list[key] = properties
        return self._dataset_list

    def evaluate(self, params):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
226
227
        if not GSITK_AVAILABLE:
            raise Exception('GSITK is not available. Install it to use this function.')
228
        logger.debug("evaluating request: {}".format(params))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
229
230
231
232
233
234
235
236
237
        results = AggregatedEvaluation()
        results.parameters = params
        datasets = self._get_datasets(results)
        plugins = self._get_plugins(results)
        for eval in evaluate(plugins, datasets):
            results.evaluations.append(eval)
        if 'with_parameters' not in results.parameters:
            del results.parameters
        logger.debug("Returning evaluation result: {}".format(results))
238
239
        return results

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
240
    def _conversion_candidates(self, fromModel, toModel):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
241
242
        candidates = self.plugins(plugin_type='emotionConversionPlugin')
        for candidate in candidates:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
243
244
245
246
247
248
249
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    yield candidate

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
250
    def convert_emotions(self, resp):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
251
        """
252
        Conversion of all emotions in a response **in place**.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
253
254
255
256
257
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
258
259
        plugins = [i['plugin'] for i in resp.analysis]
        params = resp.parameters
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
260
261
262
        toModel = params.get('emotionModel', None)
        if not toModel:
            return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
263
264
265
266
267
268
269
270
271
272
273

        logger.debug('Asked for model: {}'.format(toModel))
        output = params.get('conversion', None)
        candidates = {}
        for plugin in plugins:
            try:
                fromModel = plugin.get('onyx:usesEmotionModel', None)
                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
            except StopIteration:
                e = Error(('No conversion plugin found for: '
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
274
275
                           '{} -> {}'.format(fromModel, toModel)),
                          status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
276
277
278
                e.original_response = resp
                e.parameters = params
                raise e
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
279
280
281
        newentries = []
        for i in resp.entries:
            if output == "full":
282
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
283
284
285
            else:
                newemotions = []
            for j in i.emotions:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
286
287
                plugname = j['prov:wasGeneratedBy']
                candidate = candidates[plugname]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
288
289
                resp.analysis.append({'plugin': candidate,
                                      'parameters': params})
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
290
291
292
293
294
295
296
297
298
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries

299
300
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
301
302
303
        if not self._default or not self._default.is_activated:
            candidates = self.plugins(plugin_type='analysisPlugin',
                                      is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
304
            if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
305
306
307
308
309
                self._default = candidates[0]
            else:
                self._default = None
            logger.debug("Default: {}".format(self._default))
        return self._default
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
310
311
312

    @default_plugin.setter
    def default_plugin(self, value):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
313
314
315
        if isinstance(value, Plugin):
            if not value.is_activated:
                raise AttributeError('The default plugin has to be activated.')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
316
            self._default = value
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
317

318
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
319
            self._default = self._plugins[value.lower()]
320

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
321
    def activate_all(self, sync=True, allow_fail=False):
322
        ps = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
323
        for plug in self._plugins.keys():
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
324
325
326
327
328
329
            try:
                self.activate_plugin(plug, sync=sync)
            except Exception as ex:
                if not allow_fail:
                    raise
                logger.error('Could not activate {}: {}'.format(plug, ex))
330
331
        return ps

332
    def deactivate_all(self, sync=True):
333
        ps = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
334
        for plug in self._plugins.keys():
335
336
337
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

338
    def _set_active(self, plugin, active=True, *args, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
339
        ''' We're using a variable in the plugin itself to activate/deactivate plugins.\
340
341
        Note that plugins may activate themselves by setting this variable.
        '''
342
        plugin.is_activated = active
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
343

344
345
346
347
348
    def _activate(self, plugin):
        success = False
        with plugin._lock:
            if plugin.is_activated:
                return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
349
350
351
352
353
            plugin.activate()
            msg = "Plugin activated: {}".format(plugin.name)
            logger.info(msg)
            success = True
            self._set_active(plugin, success)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
354

355
    def activate_plugin(self, plugin_name, sync=True):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
356
357
        plugin_name = plugin_name.lower()
        if plugin_name not in self._plugins:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
358
359
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
360
        plugin = self._plugins[plugin_name]
361

362
363
364
365
366
367
368
369
        logger.info("Activating plugin: {}".format(plugin.name))

        if sync or 'async' in plugin and not plugin.async:
            self._activate(plugin)
        else:
            th = Thread(target=partial(self._activate, plugin))
            th.start()
            return th
370

371
372
373
374
    def _deactivate(self, plugin):
        with plugin._lock:
            if not plugin.is_activated:
                return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
375
376
            plugin.deactivate()
            logger.info("Plugin deactivated: {}".format(plugin.name))
377

378
    def deactivate_plugin(self, plugin_name, sync=True):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
379
380
        plugin_name = plugin_name.lower()
        if plugin_name not in self._plugins:
381
382
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
383
        plugin = self._plugins[plugin_name]
384
385
386

        self._set_active(plugin, False)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
387
        if sync or 'async' in plugin and not plugin.async:
388
            self._deactivate(plugin)
389
        else:
390
            th = Thread(target=partial(self._deactivate, plugin))
391
            th.start()
392
            return th
393

394
395
    def teardown(self, exception):
        pass