extensions.py 13.3 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2 3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
5 6
from future import standard_library
standard_library.install_aliases()
7

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8 9 10
from .plugins import SentimentPlugin, SenpyPlugin
from .models import Error, Entry, Results
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
11
from .api import API_PARAMS, NIF_PARAMS, parse_params
12

13
from threading import Thread
14

15
import os
16 17
import fnmatch
import inspect
18
import sys
19
import imp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
20
import logging
21
import traceback
22
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23
import pip
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
24 25

logger = logging.getLogger(__name__)
26

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
27

28
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29
    """ Default Senpy extension for Flask """
30

J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
31 32
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
33
                 plugin_folder=".",
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
34
                 default_plugins=False):
35
        self.app = app
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
36
        self._search_folders = set()
37
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
        self._outdated = True
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
39
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
40

41
        self.add_folder(plugin_folder)
42
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
43 44 45 46 47
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
48 49 50 51

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
52 53 54
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
55 56
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
57
        """
58 59 60 61 62 63 64
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
65
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
66
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
67
        app.register_blueprint(demo_blueprint, url_prefix="/")
68

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
69 70 71
    def add_folder(self, folder, from_root=False):
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
72
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
73 74 75 76
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
77
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
78

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
79
    def _find_plugin(self, params):
80
        api_params = parse_params(params, spec=API_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
81
        algo = None
82 83
        if "algorithm" in api_params and api_params["algorithm"]:
            algo = api_params["algorithm"]
84
        elif self.plugins:
85
            algo = self.default_plugin and self.default_plugin.name
86
        if not algo:
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
87 88 89 90
            raise Error(
                status=404,
                message=("No plugins found."
                         " Please install one.").format(algo))
91
        if algo not in self.plugins:
92 93 94
            logger.debug(("The algorithm '{}' is not valid\n"
                          "Valid algorithms: {}").format(algo,
                                                         self.plugins.keys()))
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
95 96 97
            raise Error(
                status=404,
                message="The algorithm '{}' is not valid".format(algo))
98 99 100

        if not self.plugins[algo].is_activated:
            logger.debug("Plugin not activated: {}".format(algo))
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
101 102 103 104
            raise Error(
                status=400,
                message=("The algorithm '{}'"
                         " is not activated yet").format(algo))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
105 106 107
        return self.plugins[algo]

    def _get_params(self, params, plugin):
108
        nif_params = parse_params(params, spec=NIF_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109
        extra_params = plugin.get('extra_params', {})
110 111
        specific_params = parse_params(params, spec=extra_params)
        nif_params.update(specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128
        return nif_params

    def _get_entries(self, params):
        entry = None
        if params['informat'] == 'text':
            entry = Entry(text=params['input'])
        else:
            raise NotImplemented('Only text input format implemented')
        yield entry

    def analyse(self, **api_params):
        logger.debug("analysing with params: {}".format(api_params))
        plugin = self._find_plugin(api_params)
        nif_params = self._get_params(api_params, plugin)
        resp = Results()
        if 'with_parameters' in api_params:
            resp.parameters = nif_params
129
        try:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
130 131 132 133 134 135
            entries = []
            for i in self._get_entries(nif_params):
                entries += list(plugin.analyse_entry(i, nif_params))
            resp.entries = entries
            self.convert_emotions(resp, plugin, nif_params)
            resp.analysis.append(plugin.id)
136
            logger.debug("Returning analysis result: {}".format(resp))
137 138 139
        except Error as ex:
            logger.exception('Error returning analysis result')
            resp = ex
140
        except Exception as ex:
141
            logger.exception('Error returning analysis result')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
142
            resp = Error(message=str(ex), status=500)
143
        return resp
144

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
    def _conversion_candidates(self, fromModel, toModel):
        candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
        for name, candidate in candidates.items():
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    # logging.debug('Found candidate: {}'.format(candidate))
                    yield candidate

    def convert_emotions(self, resp, plugin, params):
        """
        Conversion of all emotions in a response.
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
        fromModel = plugin.get('onyx:usesEmotionModel', None)
        toModel = params.get('emotionModel', None)
        output = params.get('conversion', None)
        logger.debug('Asked for model: {}'.format(toModel))
        logger.debug('Analysis plugin uses model: {}'.format(fromModel))

        if not toModel:
            return
        try:
            candidate = next(self._conversion_candidates(fromModel, toModel))
        except StopIteration:
            e = Error(('No conversion plugin found for: '
                       '{} -> {}'.format(fromModel, toModel)))
            e.original_response = resp
            e.parameters = params
            raise e
        newentries = []
        for i in resp.entries:
            if output == "full":
                newemotions = i.emotions.copy()
            else:
                newemotions = []
            for j in i.emotions:
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries
        resp.analysis.append(candidate.id)

197 198
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
199 200 201 202 203 204 205 206 207 208 209 210
        candidate = self._default
        if not candidate:
            candidates = self.filter_plugins(is_activated=True)
            if len(candidates) > 0:
                candidate = list(candidates.values())[0]
        logger.debug("Default: {}".format(candidate))
        return candidate

    @default_plugin.setter
    def default_plugin(self, value):
        if isinstance(value, SenpyPlugin):
            self._default = value
211
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
212
            self._default = self.plugins[value]
213

214 215 216 217 218 219 220 221 222 223 224 225 226
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
227 228 229
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
230 231 232
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend  
J. Fernando Sánchez committed
233 234 235
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
236 237 238
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

239
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
240

241
        def act():
242
            success = False
243 244
            try:
                plugin.activate()
245 246 247 248
                msg = "Plugin activated: {}".format(plugin.name)
                logger.info(msg)
                success = True
                self._set_active_plugin(plugin_name, success)
249
            except Exception as ex:
250
                msg = "Error activating plugin {} - {} : \n\t{}".format(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
251
                    plugin.name, ex, traceback.format_exc())
252 253
                logger.error(msg)
                raise Error(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
254

255
        if sync:
256
            act()
257
        else:
258 259
            th = Thread(target=act)
            th.start()
260 261

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend  
J. Fernando Sánchez committed
262 263 264
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
265 266
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
267

268 269
        self._set_active_plugin(plugin_name, False)

270 271 272 273 274
        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
275 276
                logger.error(
                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
277 278
                logger.error("Trace: {}".format(traceback.format_exc()))

279
        if sync:
280
            deact()
281
        else:
282 283
            th = Thread(target=deact)
            th.start()
284

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
285 286 287 288
    @classmethod
    def validate_info(cls, info):
        return all(x in info for x in ('name', 'module', 'version'))

289 290 291 292 293 294 295 296 297 298
    def install_deps(self):
        for i in self.plugins.values():
            self._install_deps(i._info)

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
            pip_args = []
            pip_args.append('install')
299
            pip_args.append('--use-wheel')
300
            for req in requirements:
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
301
                pip_args.append(req)
302
            logger.info('Installing requirements: ' + str(requirements))
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
303
            pip.main(pip_args)
304

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
305 306 307 308
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
309
            return None, None
310 311
        module = info["module"]
        name = info["name"]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
312
        sys.path.append(root)
313
        (fp, pathname, desc) = imp.find_module(module, [root, ])
314 315 316 317 318 319 320 321 322 323 324 325 326 327
        cls._install_deps(info)
        tmp = imp.load_module(module, fp, pathname, desc)
        sys.path.remove(root)
        candidate = None
        for _, obj in inspect.getmembers(tmp):
            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
                logger.debug(("Found plugin class:"
                              " {}@{}").format(obj, inspect.getmodule(obj)))
                candidate = obj
                break
        if not candidate:
            logger.debug("No valid plugin for: {}".format(module))
            return
        module = candidate(info=info)
328
        return name, module
329

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
330 331 332 333 334 335 336 337 338
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)

339 340
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
341
        for search_folder in self._search_folders:
342 343 344
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
345
                    if plugin and name:
346
                        plugins[name] = plugin
347

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
348
        self._outdated = False
349 350 351 352 353 354 355
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
356
        """ Return the plugins registered for a given application.  """
357 358 359
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
360

361
    def filter_plugins(self, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
362
        """ Filter plugins by different criteria """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
363

364
        def matches(plug):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
365
            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
366 367
            logger.debug(
                "matching {} with {}: {}".format(plug.name, kwargs, res))
368
            return res
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
369

370 371 372
        if not kwargs:
            return self.plugins
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
373
            return {n: p for n, p in self.plugins.items() if matches(p)}
374 375

    def sentiment_plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
376
        """ Return only the sentiment plugins """
J. Fernando Sánchez's avatar
YAPFed  
J. Fernando Sánchez committed
377 378 379 380 381
        return {
            p: plugin
            for p, plugin in self.plugins.items()
            if isinstance(plugin, SentimentPlugin)
        }