extensions.py 14.4 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2 3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
5 6
from future import standard_library
standard_library.install_aliases()
7

8
from . import plugins, api
9
from .plugins import Plugin, evaluate
10
from .models import Error, AggregatedEvaluation
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
11
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
12

13
from threading import Thread
14
from functools import partial
15
import os
16
import copy
17
import errno
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
18 19
import logging

20

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21
logger = logging.getLogger(__name__)
22

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23 24 25 26 27 28 29
try:
    from gsitk.datasets.datasets import DatasetManager
    GSITK_AVAILABLE = True
except ImportError:
    logger.warn('GSITK is not installed. Some functions will be unavailable.')
    GSITK_AVAILABLE = False

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
30

31
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
32
    """ Default Senpy extension for Flask """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
33 34
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
35
                 plugin_folder=".",
36
                 data_folder=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
37
                 default_plugins=False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38 39 40 41 42 43 44 45 46 47 48

        default_data = os.path.join(os.getcwd(), 'senpy_data')
        self.data_folder = data_folder or os.environ.get('SENPY_DATA', default_data)
        try:
            os.makedirs(self.data_folder)
        except OSError as e:
            if e.errno == errno.EEXIST:
                logger.debug('Data folder exists: {}'.format(self.data_folder))
            else:  # pragma: no cover
                raise

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
49
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
50 51 52
        self._plugins = {}
        if plugin_folder:
            self.add_folder(plugin_folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
53

54
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
55 56 57 58 59
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
60
        self.app = app
61 62 63
        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
64 65 66
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
67 68
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
69
        """
70 71 72 73 74
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
        else:  # pragma: no cover
76
            app.teardown_request(self.teardown)
77
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
78
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
79
        app.register_blueprint(demo_blueprint, url_prefix="/")
80

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97
    def add_plugin(self, plugin):
        self._plugins[plugin.name.lower()] = plugin

    def delete_plugin(self, plugin):
        del self._plugins[plugin.name.lower()]

    def plugins(self, **kwargs):
        """ Return the plugins registered for a given application. Filtered by criteria  """
        return list(plugins.pfilter(self._plugins, **kwargs))

    def get_plugin(self, name, default=None):
        if name == 'default':
            return self.default_plugin
        plugin = name.lower()
        if plugin in self._plugins:
            return self._plugins[plugin]

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
98
        results = self.plugins(id='endpoint:plugins/{}'.format(name))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
99 100 101 102 103 104 105 106 107 108

        if not results:
            return Error(message="Plugin not found", status=404)
        return results[0]

    @property
    def analysis_plugins(self):
        """ Return only the analysis plugins """
        return self.plugins(plugin_type='analysisPlugin')

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109
    def add_folder(self, folder, from_root=False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
110
        """ Find plugins in this folder and add them to this instance """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
111 112
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
113
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
114
        if os.path.isdir(folder):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
115 116 117 118
            new_plugins = plugins.from_folder([folder],
                                              data_folder=self.data_folder)
            for plugin in new_plugins:
                self.add_plugin(plugin)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
119
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
120
            raise AttributeError("Not a folder or does not exist: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
121

122
    def _get_plugins(self, request):
123
        if not self.analysis_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
124 125 126
            raise Error(
                status=404,
                message=("No plugins found."
127
                         " Please install one."))
128 129 130 131 132 133 134 135
        algos = request.parameters.get('algorithm', None)
        if not algos:
            if self.default_plugin:
                algos = [self.default_plugin.name, ]
            else:
                raise Error(
                    status=404,
                    message="No default plugin found, and None provided")
136

137 138
        plugins = list()
        for algo in algos:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
139 140 141 142 143 144
            algo = algo.lower()
            if algo not in self._plugins:
                msg = ("The algorithm '{}' is not valid\n"
                       "Valid algorithms: {}").format(algo,
                                                      self._plugins.keys())
                logger.debug(msg)
145 146
                raise Error(
                    status=404,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
147 148
                    message=msg)
            plugins.append(self._plugins[algo])
149 150
        return plugins

151
    def _process_entries(self, entries, req, plugins):
152 153 154 155
        """
        Recursively process the entries with the first plugin in the list, and pass the results
        to the rest of the plugins.
        """
156 157 158 159 160
        if not plugins:
            for i in entries:
                yield i
            return
        plugin = plugins[0]
161
        self._activate(plugin)  # Make sure the plugin is activated
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
162
        specific_params = api.parse_extra_params(req, plugin)
163 164
        req.analysis.append({'plugin': plugin,
                             'parameters': specific_params})
165
        results = plugin.analyse_entries(entries, specific_params)
166
        for i in self._process_entries(results, req, plugins[1:]):
167 168
            yield i

169
    def install_deps(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
170
        for plugin in self.plugins(is_activated=True):
171 172
            plugins.install_deps(plugin)

173
    def analyse(self, request):
174 175
        """
        Main method that analyses a request, either from CLI or HTTP.
176 177
        It takes a processed request, provided by the user, as returned
        by api.parse_call().
178
        """
179
        logger.debug("analysing request: {}".format(request))
180 181 182 183 184 185 186 187
        entries = request.entries
        request.entries = []
        plugins = self._get_plugins(request)
        results = request
        for i in self._process_entries(entries, results, plugins):
            results.entries.append(i)
        self.convert_emotions(results)
        logger.debug("Returning analysis result: {}".format(results))
188 189
        results.analysis = [i['plugin'].id for i in results.analysis]
        return results
190

191 192 193 194 195 196 197 198 199 200 201
    def _get_datasets(self, request):
        if not self.datasets:
            raise Error(
                status=404,
                message=("No datasets found."
                         " Please verify DatasetManager"))
        datasets_name = request.parameters.get('dataset', None).split(',')
        for dataset in datasets_name:
            if dataset not in self.datasets:
                logger.debug(("The dataset '{}' is not valid\n"
                              "Valid datasets: {}").format(dataset,
202
                                                           self.datasets.keys()))
203 204 205
                raise Error(
                    status=404,
                    message="The dataset '{}' is not valid".format(dataset))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
206 207
        dm = DatasetManager()
        datasets = dm.prepare_datasets(datasets_name)
208
        return datasets
209

210 211
    @property
    def datasets(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
212 213
        if not GSITK_AVAILABLE:
            raise Exception('GSITK is not available. Install it to use this function.')
214
        self._dataset_list = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
215 216
        dm = DatasetManager()
        for item in dm.get_datasets():
217 218 219 220 221 222 223 224 225
            for key in item:
                if key in self._dataset_list:
                    continue
                properties = item[key]
                properties['@id'] = key
                self._dataset_list[key] = properties
        return self._dataset_list

    def evaluate(self, params):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
226 227
        if not GSITK_AVAILABLE:
            raise Exception('GSITK is not available. Install it to use this function.')
228
        logger.debug("evaluating request: {}".format(params))
229 230 231 232 233 234 235 236 237
        results = AggregatedEvaluation()
        results.parameters = params
        datasets = self._get_datasets(results)
        plugins = self._get_plugins(results)
        for eval in evaluate(plugins, datasets):
            results.evaluations.append(eval)
        if 'with_parameters' not in results.parameters:
            del results.parameters
        logger.debug("Returning evaluation result: {}".format(results))
238 239
        return results

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
240
    def _conversion_candidates(self, fromModel, toModel):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
241 242
        candidates = self.plugins(plugin_type='emotionConversionPlugin')
        for candidate in candidates:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
243 244 245 246 247 248 249
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    yield candidate

250
    def convert_emotions(self, resp):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
251
        """
252
        Conversion of all emotions in a response **in place**.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
253 254 255 256 257
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
258 259
        plugins = [i['plugin'] for i in resp.analysis]
        params = resp.parameters
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
260 261 262
        toModel = params.get('emotionModel', None)
        if not toModel:
            return
263 264 265 266 267 268 269 270 271 272 273

        logger.debug('Asked for model: {}'.format(toModel))
        output = params.get('conversion', None)
        candidates = {}
        for plugin in plugins:
            try:
                fromModel = plugin.get('onyx:usesEmotionModel', None)
                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
            except StopIteration:
                e = Error(('No conversion plugin found for: '
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
274 275
                           '{} -> {}'.format(fromModel, toModel)),
                          status=404)
276 277 278
                e.original_response = resp
                e.parameters = params
                raise e
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
279 280 281
        newentries = []
        for i in resp.entries:
            if output == "full":
282
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
283 284 285
            else:
                newemotions = []
            for j in i.emotions:
286 287
                plugname = j['prov:wasGeneratedBy']
                candidate = candidates[plugname]
288 289
                resp.analysis.append({'plugin': candidate,
                                      'parameters': params})
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
290 291 292 293 294 295 296 297 298
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries

299 300
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
301 302 303
        if not self._default or not self._default.is_activated:
            candidates = self.plugins(plugin_type='analysisPlugin',
                                      is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
304
            if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
305 306 307 308 309
                self._default = candidates[0]
            else:
                self._default = None
            logger.debug("Default: {}".format(self._default))
        return self._default
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
310 311 312

    @default_plugin.setter
    def default_plugin(self, value):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
313 314 315
        if isinstance(value, Plugin):
            if not value.is_activated:
                raise AttributeError('The default plugin has to be activated.')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
316
            self._default = value
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
317

318
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
319
            self._default = self._plugins[value.lower()]
320

321
    def activate_all(self, sync=True, allow_fail=False):
322
        ps = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
323
        for plug in self._plugins.keys():
324 325 326 327 328 329
            try:
                self.activate_plugin(plug, sync=sync)
            except Exception as ex:
                if not allow_fail:
                    raise
                logger.error('Could not activate {}: {}'.format(plug, ex))
330 331
        return ps

332
    def deactivate_all(self, sync=True):
333
        ps = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
334
        for plug in self._plugins.keys():
335 336 337
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

338
    def _set_active(self, plugin, active=True, *args, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
339
        ''' We're using a variable in the plugin itself to activate/deactivate plugins.\
340 341
        Note that plugins may activate themselves by setting this variable.
        '''
342
        plugin.is_activated = active
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
343

344 345 346 347 348
    def _activate(self, plugin):
        success = False
        with plugin._lock:
            if plugin.is_activated:
                return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
349 350 351 352 353
            plugin.activate()
            msg = "Plugin activated: {}".format(plugin.name)
            logger.info(msg)
            success = True
            self._set_active(plugin, success)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
354

355
    def activate_plugin(self, plugin_name, sync=True):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
356 357
        plugin_name = plugin_name.lower()
        if plugin_name not in self._plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
358 359
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
360
        plugin = self._plugins[plugin_name]
361

362 363 364 365 366 367 368 369
        logger.info("Activating plugin: {}".format(plugin.name))

        if sync or 'async' in plugin and not plugin.async:
            self._activate(plugin)
        else:
            th = Thread(target=partial(self._activate, plugin))
            th.start()
            return th
370

371 372 373 374
    def _deactivate(self, plugin):
        with plugin._lock:
            if not plugin.is_activated:
                return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
375 376
            plugin.deactivate()
            logger.info("Plugin deactivated: {}".format(plugin.name))
377

378
    def deactivate_plugin(self, plugin_name, sync=True):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
379 380
        plugin_name = plugin_name.lower()
        if plugin_name not in self._plugins:
381 382
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
383
        plugin = self._plugins[plugin_name]
384 385 386

        self._set_active(plugin, False)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
387
        if sync or 'async' in plugin and not plugin.async:
388
            self._deactivate(plugin)
389
        else:
390
            th = Thread(target=partial(self._deactivate, plugin))
391
            th.start()
392
            return th
393

394 395
    def teardown(self, exception):
        pass