extensions.py 15.1 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2
3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
5
6
from future import standard_library
standard_library.install_aliases()
7

8
9
from . import plugins
from .plugins import SenpyPlugin
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
10
from .models import Error, Entry, Results, from_string
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
11
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
12
from .api import API_PARAMS, NIF_PARAMS, parse_params
13

14
from threading import Thread
15

16
import os
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
17
import copy
18
19
import fnmatch
import inspect
20
import sys
21
import importlib
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
22
import logging
23
import traceback
24
import yaml
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
import subprocess
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
26
27

logger = logging.getLogger(__name__)
28

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
30
31
32
33
34
35
36
def log_subprocess_output(process):
    for line in iter(process.stdout.readline, b''):
        logger.info('%r', line)
    for line in iter(process.stderr.readline, b''):
        logger.error('%r', line)


37
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
    """ Default Senpy extension for Flask """
39

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
40
41
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
42
                 plugin_folder=".",
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
43
                 default_plugins=False):
44
        self.app = app
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
45
        self._search_folders = set()
46
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
47
        self._outdated = True
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
48
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
49

50
        self.add_folder(plugin_folder)
51
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
52
53
54
55
56
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
57
58
59
60

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
61
62
63
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
64
65
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
66
        """
67
68
69
70
71
72
73
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
74
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
76
        app.register_blueprint(demo_blueprint, url_prefix="/")
77

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
78
79
80
    def add_folder(self, folder, from_root=False):
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
81
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
82
83
84
85
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
86
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
87

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
88
89
    def _find_plugins(self, params):
        if not self.analysis_plugins:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
90
91
92
            raise Error(
                status=404,
                message=("No plugins found."
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
93
94
95
96
97
98
99
100
                         " Please install one."))
        api_params = parse_params(params, spec=API_PARAMS)
        algos = None
        if "algorithm" in api_params and api_params["algorithm"]:
            algos = api_params["algorithm"].split(',')
        elif self.default_plugin:
            algos = [self.default_plugin.name, ]
        else:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
101
102
            raise Error(
                status=404,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
103
                message="No default plugin found, and None provided")
104

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
105
106
107
108
109
110
111
112
113
        plugins = list()
        for algo in algos:
            if algo not in self.plugins:
                logger.debug(("The algorithm '{}' is not valid\n"
                              "Valid algorithms: {}").format(algo,
                                                             self.plugins.keys()))
                raise Error(
                    status=404,
                    message="The algorithm '{}' is not valid".format(algo))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
114

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
115
116
117
118
119
120
121
122
123
124
            if not self.plugins[algo].is_activated:
                logger.debug("Plugin not activated: {}".format(algo))
                raise Error(
                    status=400,
                    message=("The algorithm '{}'"
                             " is not activated yet").format(algo))
            plugins.append(self.plugins[algo])
        return plugins

    def _get_params(self, params, plugin=None):
125
        nif_params = parse_params(params, spec=NIF_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
126
127
128
129
        if plugin:
            extra_params = plugin.get('extra_params', {})
            specific_params = parse_params(params, spec=extra_params)
            nif_params.update(specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
130
131
132
133
        return nif_params

    def _get_entries(self, params):
        if params['informat'] == 'text':
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
134
            results = Results()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
135
            entry = Entry(text=params['input'])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
136
137
            results.entries.append(entry)
        elif params['informat'] == 'json-ld':
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
138
            results = from_string(params['input'], cls=Results)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
139
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
            raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
        return results

    def _process_entries(self, entries, plugins, nif_params):
        if not plugins:
            for i in entries:
                yield i
            return
        plugin = plugins[0]
        specific_params = self._get_params(nif_params, plugin)
        results = plugin.analyse_entries(entries, specific_params)
        for i in self._process_entries(results, plugins[1:], nif_params):
            yield i

    def _process_response(self, resp, plugins, nif_params):
        entries = resp.entries
        resp.entries = []
        for plug in plugins:
            resp.analysis.append(plug.id)
        for i in self._process_entries(entries, plugins, nif_params):
            resp.entries.append(i)
        return resp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
162
163

    def analyse(self, **api_params):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
164
165
166
167
        """
        Main method that analyses a request, either from CLI or HTTP.
        It uses a dictionary of parameters, provided by the user.
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
168
        logger.debug("analysing with params: {}".format(api_params))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
169
170
171
        plugins = self._find_plugins(api_params)
        nif_params = self._get_params(api_params)
        resp = self._get_entries(nif_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
172
173
        if 'with_parameters' in api_params:
            resp.parameters = nif_params
174
        try:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
175
176
            resp = self._process_response(resp, plugins, nif_params)
            self.convert_emotions(resp, plugins, nif_params)
177
            logger.debug("Returning analysis result: {}".format(resp))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
178
179
180
        except (Error, Exception) as ex:
            if not isinstance(ex, Error):
                ex = Error(message=str(ex), status=500)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
181
            logger.error('Error returning analysis result')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
182
            raise ex
183
        return resp
184

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
185
    def _conversion_candidates(self, fromModel, toModel):
186
        candidates = self.filter_plugins(plugin_type='emotionConversionPlugin')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
187
188
189
190
191
192
193
194
195
        for name, candidate in candidates.items():
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    # logging.debug('Found candidate: {}'.format(candidate))
                    yield candidate

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
196
    def convert_emotions(self, resp, plugins, params):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
197
198
199
200
201
202
203
204
205
206
        """
        Conversion of all emotions in a response.
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
        toModel = params.get('emotionModel', None)
        if not toModel:
            return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221

        logger.debug('Asked for model: {}'.format(toModel))
        output = params.get('conversion', None)
        candidates = {}
        for plugin in plugins:
            try:
                fromModel = plugin.get('onyx:usesEmotionModel', None)
                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
            except StopIteration:
                e = Error(('No conversion plugin found for: '
                           '{} -> {}'.format(fromModel, toModel)))
                e.original_response = resp
                e.parameters = params
                raise e
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
222
        newentries = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
223
        resp.analysis = set(resp.analysis)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
224
225
        for i in resp.entries:
            if output == "full":
226
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
227
228
229
            else:
                newemotions = []
            for j in i.emotions:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
230
231
232
                plugname = j['prov:wasGeneratedBy']
                candidate = candidates[plugname]
                resp.analysis.add(candidate.id)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
233
234
235
236
237
238
239
240
241
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries

242
243
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
244
245
        candidate = self._default
        if not candidate:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
246
247
            candidates = self.filter_plugins(plugin_type='analysisPlugin',
                                             is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
248
249
250
251
252
253
254
255
256
            if len(candidates) > 0:
                candidate = list(candidates.values())[0]
        logger.debug("Default: {}".format(candidate))
        return candidate

    @default_plugin.setter
    def default_plugin(self, value):
        if isinstance(value, SenpyPlugin):
            self._default = value
257
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
258
            self._default = self.plugins[value]
259

260
261
262
263
264
265
266
267
268
269
270
271
272
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
273
274
275
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
276
277
278
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
279
280
281
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
282
283
284
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

285
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
286

287
        def act():
288
            success = False
289
290
            try:
                plugin.activate()
291
292
293
294
                msg = "Plugin activated: {}".format(plugin.name)
                logger.info(msg)
                success = True
                self._set_active_plugin(plugin_name, success)
295
            except Exception as ex:
296
                msg = "Error activating plugin {} - {} : \n\t{}".format(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
297
                    plugin.name, ex, traceback.format_exc())
298
299
                logger.error(msg)
                raise Error(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
300

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
301
        if sync or 'async' in plugin and not plugin.async:
302
            act()
303
        else:
304
305
            th = Thread(target=act)
            th.start()
306
            return th
307
308

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
309
310
311
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
312
313
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
314

315
316
        self._set_active_plugin(plugin_name, False)

317
318
319
320
321
        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
322
323
                logger.error(
                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
324
325
                logger.error("Trace: {}".format(traceback.format_exc()))

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
326
        if sync or 'async' in plugin and not plugin.async:
327
            deact()
328
        else:
329
330
            th = Thread(target=deact)
            th.start()
331
            return th
332

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
333
334
    @classmethod
    def validate_info(cls, info):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
335
        return all(x in info for x in ('name', 'module', 'description', 'version'))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
336

337
338
    def install_deps(self):
        for i in self.plugins.values():
339
            self._install_deps(i)
340
341
342
343
344

    @classmethod
    def _install_deps(cls, info=None):
        requirements = info.get('requirements', [])
        if requirements:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
345
            pip_args = ['pip']
346
            pip_args.append('install')
347
            pip_args.append('--use-wheel')
348
            for req in requirements:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
349
                pip_args.append(req)
350
            logger.info('Installing requirements: ' + str(requirements))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
351
352
353
354
355
356
357
            process = subprocess.Popen(pip_args,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
            log_subprocess_output(process)
            exitcode = process.wait()
            if exitcode != 0:
                raise Error("Dependencies not properly installed")
358

359
360
361
362
363
364
365
    @classmethod
    def _load_module(cls, name, root):
        sys.path.append(root)
        tmp = importlib.import_module(name)
        sys.path.remove(root)
        return tmp

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
366
367
368
369
    @classmethod
    def _load_plugin_from_info(cls, info, root):
        if not cls.validate_info(info):
            logger.warn('The module info is not valid.\n\t{}'.format(info))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
370
            return None, None
371
372
        module = info["module"]
        name = info["name"]
373

374
        cls._install_deps(info)
375
376
        tmp = cls._load_module(module, root)

377
378
379
380
381
382
383
384
385
386
387
        candidate = None
        for _, obj in inspect.getmembers(tmp):
            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
                logger.debug(("Found plugin class:"
                              " {}@{}").format(obj, inspect.getmodule(obj)))
                candidate = obj
                break
        if not candidate:
            logger.debug("No valid plugin for: {}".format(module))
            return
        module = candidate(info=info)
388
        return name, module
389

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
390
391
392
393
394
395
396
397
398
    @classmethod
    def _load_plugin(cls, root, filename):
        fpath = os.path.join(root, filename)
        logger.debug("Loading plugin: {}".format(fpath))
        with open(fpath, 'r') as f:
            info = yaml.load(f)
        logger.debug("Info: {}".format(info))
        return cls._load_plugin_from_info(info, root)

399
400
    def _load_plugins(self):
        plugins = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
401
        for search_folder in self._search_folders:
402
403
404
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
405
                    if plugin and name:
406
                        plugins[name] = plugin
407

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
408
        self._outdated = False
409
410
411
412
413
414
415
        return plugins

    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
416
        """ Return the plugins registered for a given application.  """
417
418
419
        if self._outdated:
            self._plugin_list = self._load_plugins()
        return self._plugin_list
420

421
    def filter_plugins(self, **kwargs):
422
        return plugins.pfilter(self.plugins, **kwargs)
423
424
425
426
427

    @property
    def analysis_plugins(self):
        """ Return only the analysis plugins """
        return self.filter_plugins(plugin_type='analysisPlugin')