extensions.py 14.2 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2 3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
5 6
from future import standard_library
standard_library.install_aliases()
7

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8
from . import plugins, api
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
9
from .plugins import Plugin, evaluate
10
from .models import Error, AggregatedEvaluation
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
11
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
12

13
from threading import Thread
14
from functools import partial
15
import os
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
16
import copy
17
import errno
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
18 19
import logging

20

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21
logger = logging.getLogger(__name__)
22

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23 24 25 26 27 28 29
try:
    from gsitk.datasets.datasets import DatasetManager
    GSITK_AVAILABLE = True
except ImportError:
    logger.warn('GSITK is not installed. Some functions will be unavailable.')
    GSITK_AVAILABLE = False

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
30

31
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
32
    """ Default Senpy extension for Flask """
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
33 34
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
35
                 plugin_folder=".",
36
                 data_folder=None,
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
37
                 default_plugins=False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38 39 40 41 42 43 44 45 46 47 48

        default_data = os.path.join(os.getcwd(), 'senpy_data')
        self.data_folder = data_folder or os.environ.get('SENPY_DATA', default_data)
        try:
            os.makedirs(self.data_folder)
        except OSError as e:
            if e.errno == errno.EEXIST:
                logger.debug('Data folder exists: {}'.format(self.data_folder))
            else:  # pragma: no cover
                raise

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
49
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
50 51 52
        self._plugins = {}
        if plugin_folder:
            self.add_folder(plugin_folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
53

54
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
55 56 57 58 59
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
60
        self.app = app
61 62 63
        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
64 65 66
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
67 68
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
69
        """
70 71 72 73 74
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
        else:  # pragma: no cover
76
            app.teardown_request(self.teardown)
77
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
78
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
79
        app.register_blueprint(demo_blueprint, url_prefix="/")
80

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
    def add_plugin(self, plugin):
        self._plugins[plugin.name.lower()] = plugin

    def delete_plugin(self, plugin):
        del self._plugins[plugin.name.lower()]

    def plugins(self, **kwargs):
        """ Return the plugins registered for a given application. Filtered by criteria  """
        return list(plugins.pfilter(self._plugins, **kwargs))

    def get_plugin(self, name, default=None):
        if name == 'default':
            return self.default_plugin
        plugin = name.lower()
        if plugin in self._plugins:
            return self._plugins[plugin]

        results = self.plugins(id='plugins/{}'.format(name))

        if not results:
            return Error(message="Plugin not found", status=404)
        return results[0]

    @property
    def analysis_plugins(self):
        """ Return only the analysis plugins """
        return self.plugins(plugin_type='analysisPlugin')

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109
    def add_folder(self, folder, from_root=False):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
110
        """ Find plugins in this folder and add them to this instance """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
111 112
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
113
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
114
        if os.path.isdir(folder):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
115 116 117 118
            new_plugins = plugins.from_folder([folder],
                                              data_folder=self.data_folder)
            for plugin in new_plugins:
                self.add_plugin(plugin)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
119
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
120
            raise AttributeError("Not a folder or does not exist: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
121

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
122
    def _get_plugins(self, request):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
123
        if not self.analysis_plugins:
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
124 125 126
            raise Error(
                status=404,
                message=("No plugins found."
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
127
                         " Please install one."))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
128 129 130 131 132 133 134 135
        algos = request.parameters.get('algorithm', None)
        if not algos:
            if self.default_plugin:
                algos = [self.default_plugin.name, ]
            else:
                raise Error(
                    status=404,
                    message="No default plugin found, and None provided")
136

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
137 138
        plugins = list()
        for algo in algos:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
139 140 141 142 143 144
            algo = algo.lower()
            if algo not in self._plugins:
                msg = ("The algorithm '{}' is not valid\n"
                       "Valid algorithms: {}").format(algo,
                                                      self._plugins.keys())
                logger.debug(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
145 146
                raise Error(
                    status=404,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
147 148
                    message=msg)
            plugins.append(self._plugins[algo])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
149 150
        return plugins

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
151
    def _process_entries(self, entries, req, plugins):
152 153 154 155
        """
        Recursively process the entries with the first plugin in the list, and pass the results
        to the rest of the plugins.
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
156 157 158 159 160
        if not plugins:
            for i in entries:
                yield i
            return
        plugin = plugins[0]
161
        self._activate(plugin)  # Make sure the plugin is activated
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
162
        specific_params = api.parse_extra_params(req, plugin)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
163 164
        req.analysis.append({'plugin': plugin,
                             'parameters': specific_params})
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
165
        results = plugin.analyse_entries(entries, specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
166
        for i in self._process_entries(results, req, plugins[1:]):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
167 168
            yield i

169
    def install_deps(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
170
        for plugin in self.plugins(is_activated=True):
171 172
            plugins.install_deps(plugin)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
173
    def analyse(self, request):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
174 175
        """
        Main method that analyses a request, either from CLI or HTTP.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
176 177
        It takes a processed request, provided by the user, as returned
        by api.parse_call().
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
178
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
179
        logger.debug("analysing request: {}".format(request))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
180 181 182 183 184 185 186 187
        entries = request.entries
        request.entries = []
        plugins = self._get_plugins(request)
        results = request
        for i in self._process_entries(entries, results, plugins):
            results.entries.append(i)
        self.convert_emotions(results)
        logger.debug("Returning analysis result: {}".format(results))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
188 189
        results.analysis = [i['plugin'].id for i in results.analysis]
        return results
190

191 192 193 194 195 196 197 198 199 200 201
    def _get_datasets(self, request):
        if not self.datasets:
            raise Error(
                status=404,
                message=("No datasets found."
                         " Please verify DatasetManager"))
        datasets_name = request.parameters.get('dataset', None).split(',')
        for dataset in datasets_name:
            if dataset not in self.datasets:
                logger.debug(("The dataset '{}' is not valid\n"
                              "Valid datasets: {}").format(dataset,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
202
                                                           self.datasets.keys()))
203 204 205
                raise Error(
                    status=404,
                    message="The dataset '{}' is not valid".format(dataset))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
206 207
        dm = DatasetManager()
        datasets = dm.prepare_datasets(datasets_name)
208
        return datasets
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
209

210 211
    @property
    def datasets(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
212 213
        if not GSITK_AVAILABLE:
            raise Exception('GSITK is not available. Install it to use this function.')
214
        self._dataset_list = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
215 216
        dm = DatasetManager()
        for item in dm.get_datasets():
217 218 219 220 221 222 223 224 225
            for key in item:
                if key in self._dataset_list:
                    continue
                properties = item[key]
                properties['@id'] = key
                self._dataset_list[key] = properties
        return self._dataset_list

    def evaluate(self, params):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
226 227
        if not GSITK_AVAILABLE:
            raise Exception('GSITK is not available. Install it to use this function.')
228
        logger.debug("evaluating request: {}".format(params))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
229 230 231 232 233 234 235 236 237
        results = AggregatedEvaluation()
        results.parameters = params
        datasets = self._get_datasets(results)
        plugins = self._get_plugins(results)
        for eval in evaluate(plugins, datasets):
            results.evaluations.append(eval)
        if 'with_parameters' not in results.parameters:
            del results.parameters
        logger.debug("Returning evaluation result: {}".format(results))
238 239
        return results

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
240
    def _conversion_candidates(self, fromModel, toModel):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
241 242
        candidates = self.plugins(plugin_type='emotionConversionPlugin')
        for candidate in candidates:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
243 244 245 246 247 248 249
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    yield candidate

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
250
    def convert_emotions(self, resp):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
251
        """
252
        Conversion of all emotions in a response **in place**.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
253 254 255 256 257
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
258 259
        plugins = [i['plugin'] for i in resp.analysis]
        params = resp.parameters
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
260 261 262
        toModel = params.get('emotionModel', None)
        if not toModel:
            return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
263 264 265 266 267 268 269 270 271 272 273

        logger.debug('Asked for model: {}'.format(toModel))
        output = params.get('conversion', None)
        candidates = {}
        for plugin in plugins:
            try:
                fromModel = plugin.get('onyx:usesEmotionModel', None)
                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
            except StopIteration:
                e = Error(('No conversion plugin found for: '
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
274 275
                           '{} -> {}'.format(fromModel, toModel)),
                          status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
276 277 278
                e.original_response = resp
                e.parameters = params
                raise e
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
279 280 281
        newentries = []
        for i in resp.entries:
            if output == "full":
282
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
283 284 285
            else:
                newemotions = []
            for j in i.emotions:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
286 287
                plugname = j['prov:wasGeneratedBy']
                candidate = candidates[plugname]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
288 289
                resp.analysis.append({'plugin': candidate,
                                      'parameters': params})
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
290 291 292 293 294 295 296 297 298
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries

299 300
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
301 302 303
        if not self._default or not self._default.is_activated:
            candidates = self.plugins(plugin_type='analysisPlugin',
                                      is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
304
            if len(candidates) > 0:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
305 306 307 308 309
                self._default = candidates[0]
            else:
                self._default = None
            logger.debug("Default: {}".format(self._default))
        return self._default
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
310 311 312

    @default_plugin.setter
    def default_plugin(self, value):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
313 314 315
        if isinstance(value, Plugin):
            if not value.is_activated:
                raise AttributeError('The default plugin has to be activated.')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
316
            self._default = value
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
317

318
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
319
            self._default = self._plugins[value.lower()]
320

321
    def activate_all(self, sync=True):
322
        ps = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
323
        for plug in self._plugins.keys():
324 325 326
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

327
    def deactivate_all(self, sync=True):
328
        ps = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
329
        for plug in self._plugins.keys():
330 331 332
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

333
    def _set_active(self, plugin, active=True, *args, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
334
        ''' We're using a variable in the plugin itself to activate/deactivate plugins.\
335 336
        Note that plugins may activate themselves by setting this variable.
        '''
337
        plugin.is_activated = active
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
338

339 340 341 342 343
    def _activate(self, plugin):
        success = False
        with plugin._lock:
            if plugin.is_activated:
                return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
344 345 346 347 348
            plugin.activate()
            msg = "Plugin activated: {}".format(plugin.name)
            logger.info(msg)
            success = True
            self._set_active(plugin, success)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
349

350
    def activate_plugin(self, plugin_name, sync=True):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
351 352
        plugin_name = plugin_name.lower()
        if plugin_name not in self._plugins:
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
353 354
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
355
        plugin = self._plugins[plugin_name]
356

357 358 359 360 361 362 363 364
        logger.info("Activating plugin: {}".format(plugin.name))

        if sync or 'async' in plugin and not plugin.async:
            self._activate(plugin)
        else:
            th = Thread(target=partial(self._activate, plugin))
            th.start()
            return th
365

366 367 368 369
    def _deactivate(self, plugin):
        with plugin._lock:
            if not plugin.is_activated:
                return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
370 371
            plugin.deactivate()
            logger.info("Plugin deactivated: {}".format(plugin.name))
372

373
    def deactivate_plugin(self, plugin_name, sync=True):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
374 375
        plugin_name = plugin_name.lower()
        if plugin_name not in self._plugins:
376 377
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
378
        plugin = self._plugins[plugin_name]
379 380 381

        self._set_active(plugin, False)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
382
        if sync or 'async' in plugin and not plugin.async:
383
            self._deactivate(plugin)
384
        else:
385
            th = Thread(target=partial(self._deactivate, plugin))
386
            th.start()
387
            return th
388

389 390
    def teardown(self, exception):
        pass