extensions.py 12.2 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2
3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
5
6
from future import standard_library
standard_library.install_aliases()
7

8
9
from . import plugins
from .plugins import SenpyPlugin
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
10
from .models import Error, Entry, Results, from_string
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
11
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
12
from .api import API_PARAMS, NIF_PARAMS, parse_params
13

14
from threading import Thread
15

16
import os
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
17
import copy
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
18
import logging
19
import traceback
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
20
21

logger = logging.getLogger(__name__)
22

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
23

24
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
    """ Default Senpy extension for Flask """
26

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
27
28
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
29
                 plugin_folder=".",
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
30
                 default_plugins=False):
31
        self.app = app
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
32
        self._search_folders = set()
33
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
34
        self._outdated = True
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
35
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
36

37
        self.add_folder(plugin_folder)
38
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
39
40
41
42
43
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
44
45
46
47

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
48
49
50
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
51
52
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
53
        """
54
55
56
57
58
59
60
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
61
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
62
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
63
        app.register_blueprint(demo_blueprint, url_prefix="/")
64

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
65
66
67
    def add_folder(self, folder, from_root=False):
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
68
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
69
70
71
72
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
73
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
74

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
76
    def _find_plugins(self, params):
        if not self.analysis_plugins:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
77
78
79
            raise Error(
                status=404,
                message=("No plugins found."
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
80
81
82
83
84
85
86
87
                         " Please install one."))
        api_params = parse_params(params, spec=API_PARAMS)
        algos = None
        if "algorithm" in api_params and api_params["algorithm"]:
            algos = api_params["algorithm"].split(',')
        elif self.default_plugin:
            algos = [self.default_plugin.name, ]
        else:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
88
89
            raise Error(
                status=404,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
90
                message="No default plugin found, and None provided")
91

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
92
93
94
95
96
97
98
99
100
        plugins = list()
        for algo in algos:
            if algo not in self.plugins:
                logger.debug(("The algorithm '{}' is not valid\n"
                              "Valid algorithms: {}").format(algo,
                                                             self.plugins.keys()))
                raise Error(
                    status=404,
                    message="The algorithm '{}' is not valid".format(algo))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
101

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
102
103
104
105
106
107
108
109
110
111
            if not self.plugins[algo].is_activated:
                logger.debug("Plugin not activated: {}".format(algo))
                raise Error(
                    status=400,
                    message=("The algorithm '{}'"
                             " is not activated yet").format(algo))
            plugins.append(self.plugins[algo])
        return plugins

    def _get_params(self, params, plugin=None):
112
        nif_params = parse_params(params, spec=NIF_PARAMS)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
113
114
115
116
        if plugin:
            extra_params = plugin.get('extra_params', {})
            specific_params = parse_params(params, spec=extra_params)
            nif_params.update(specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
117
118
119
120
        return nif_params

    def _get_entries(self, params):
        if params['informat'] == 'text':
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
121
            results = Results()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
122
            entry = Entry(text=params['input'])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
123
124
            results.entries.append(entry)
        elif params['informat'] == 'json-ld':
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
125
            results = from_string(params['input'], cls=Results)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
126
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
            raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
        return results

    def _process_entries(self, entries, plugins, nif_params):
        if not plugins:
            for i in entries:
                yield i
            return
        plugin = plugins[0]
        specific_params = self._get_params(nif_params, plugin)
        results = plugin.analyse_entries(entries, specific_params)
        for i in self._process_entries(results, plugins[1:], nif_params):
            yield i

    def _process_response(self, resp, plugins, nif_params):
        entries = resp.entries
        resp.entries = []
        for plug in plugins:
            resp.analysis.append(plug.id)
        for i in self._process_entries(entries, plugins, nif_params):
            resp.entries.append(i)
        return resp
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
149
150

    def analyse(self, **api_params):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
151
152
153
154
        """
        Main method that analyses a request, either from CLI or HTTP.
        It uses a dictionary of parameters, provided by the user.
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
155
        logger.debug("analysing with params: {}".format(api_params))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
156
157
158
        plugins = self._find_plugins(api_params)
        nif_params = self._get_params(api_params)
        resp = self._get_entries(nif_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
159
160
        if 'with_parameters' in api_params:
            resp.parameters = nif_params
161
        try:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
162
163
            resp = self._process_response(resp, plugins, nif_params)
            self.convert_emotions(resp, plugins, nif_params)
164
            logger.debug("Returning analysis result: {}".format(resp))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
165
166
167
        except (Error, Exception) as ex:
            if not isinstance(ex, Error):
                ex = Error(message=str(ex), status=500)
168
            logger.exception('Error returning analysis result')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
169
            raise ex
170
        return resp
171

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
172
    def _conversion_candidates(self, fromModel, toModel):
173
        candidates = self.filter_plugins(plugin_type='emotionConversionPlugin')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
174
175
176
177
178
179
180
181
182
        for name, candidate in candidates.items():
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    # logging.debug('Found candidate: {}'.format(candidate))
                    yield candidate

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
183
    def convert_emotions(self, resp, plugins, params):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
184
        """
185
        Conversion of all emotions in a response **in place**.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
186
187
188
189
190
191
192
193
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
        toModel = params.get('emotionModel', None)
        if not toModel:
            return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208

        logger.debug('Asked for model: {}'.format(toModel))
        output = params.get('conversion', None)
        candidates = {}
        for plugin in plugins:
            try:
                fromModel = plugin.get('onyx:usesEmotionModel', None)
                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
            except StopIteration:
                e = Error(('No conversion plugin found for: '
                           '{} -> {}'.format(fromModel, toModel)))
                e.original_response = resp
                e.parameters = params
                raise e
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
209
210
211
        newentries = []
        for i in resp.entries:
            if output == "full":
212
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
213
214
215
            else:
                newemotions = []
            for j in i.emotions:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
216
217
                plugname = j['prov:wasGeneratedBy']
                candidate = candidates[plugname]
218
                resp.analysis.append(candidate.id)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
219
220
221
222
223
224
225
226
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries
227
        resp.analysis = list(set(resp.analysis))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
228

229
230
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
231
232
        candidate = self._default
        if not candidate:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
233
234
            candidates = self.filter_plugins(plugin_type='analysisPlugin',
                                             is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
235
236
237
238
239
240
241
242
243
            if len(candidates) > 0:
                candidate = list(candidates.values())[0]
        logger.debug("Default: {}".format(candidate))
        return candidate

    @default_plugin.setter
    def default_plugin(self, value):
        if isinstance(value, SenpyPlugin):
            self._default = value
244
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
245
            self._default = self.plugins[value]
246

247
248
249
250
251
252
253
254
255
256
257
258
259
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
260
261
262
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
263
264
265
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
266
267
268
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
269
270
271
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

272
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
273

274
        def act():
275
            success = False
276
277
            try:
                plugin.activate()
278
279
280
281
                msg = "Plugin activated: {}".format(plugin.name)
                logger.info(msg)
                success = True
                self._set_active_plugin(plugin_name, success)
282
            except Exception as ex:
283
                msg = "Error activating plugin {} - {} : \n\t{}".format(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
284
                    plugin.name, ex, traceback.format_exc())
285
286
                logger.error(msg)
                raise Error(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
287

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
288
        if sync or 'async' in plugin and not plugin.async:
289
            act()
290
        else:
291
292
            th = Thread(target=act)
            th.start()
293
            return th
294
295

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
296
297
298
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
299
300
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
301

302
303
        self._set_active_plugin(plugin_name, False)

304
305
306
307
308
        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
309
310
                logger.error(
                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
311
312
                logger.error("Trace: {}".format(traceback.format_exc()))

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
313
        if sync or 'async' in plugin and not plugin.async:
314
            deact()
315
        else:
316
317
            th = Thread(target=deact)
            th.start()
318
            return th
319

320
321
322
323
324
    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
325
        """ Return the plugins registered for a given application.  """
326
        if self._outdated:
327
328
            self._plugin_list = plugins.load_plugins(self._search_folders)
            self._outdated = False
329
        return self._plugin_list
330

331
    def filter_plugins(self, **kwargs):
332
        return plugins.pfilter(self.plugins, **kwargs)
333
334
335
336
337

    @property
    def analysis_plugins(self):
        """ Return only the analysis plugins """
        return self.filter_plugins(plugin_type='analysisPlugin')