extensions.py 11.4 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
"""
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
2
3
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
"""
5
6
from future import standard_library
standard_library.install_aliases()
7

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8
from . import plugins, api
9
from .plugins import SenpyPlugin
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
10
from .models import Error
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
11
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
12

13
from threading import Thread
14

15
import os
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
16
import copy
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
17
import logging
18
import traceback
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
19
20

logger = logging.getLogger(__name__)
21

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
22

23
class Senpy(object):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
24
    """ Default Senpy extension for Flask """
25

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
26
27
    def __init__(self,
                 app=None,
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
28
                 plugin_folder=".",
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
29
                 default_plugins=False):
30
        self.app = app
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
31
        self._search_folders = set()
32
        self._plugin_list = []
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
33
        self._outdated = True
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
34
        self._default = None
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
35

36
        self.add_folder(plugin_folder)
37
        if default_plugins:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
39
40
41
42
            self.add_folder('plugins', from_root=True)
        else:
            # Add only conversion plugins
            self.add_folder(os.path.join('plugins', 'conversion'),
                            from_root=True)
43
44
45
46

        if app is not None:
            self.init_app(app)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
47
48
49
    def init_app(self, app):
        """ Initialise a flask app to add plugins to its context """
        """
50
51
        Note: I'm not particularly fond of adding self.app and app.senpy, but
        I can't think of a better way to do it.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
52
        """
53
54
55
56
57
58
59
        app.senpy = self
        # Use the newstyle teardown_appcontext if it's available,
        # otherwise fall back to the request context
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)
60
        app.register_blueprint(api_blueprint, url_prefix="/api")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
61
        app.register_blueprint(ns_blueprint, url_prefix="/ns")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
62
        app.register_blueprint(demo_blueprint, url_prefix="/")
63

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
64
65
66
    def add_folder(self, folder, from_root=False):
        if from_root:
            folder = os.path.join(os.path.dirname(__file__), folder)
67
        logger.debug("Adding folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
68
69
70
71
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
        else:
72
            logger.debug("Not a folder: %s", folder)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
73

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
74
    def _get_plugins(self, request):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75
        if not self.analysis_plugins:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
76
77
78
            raise Error(
                status=404,
                message=("No plugins found."
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
79
                         " Please install one."))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
80
81
82
83
84
85
86
87
        algos = request.parameters.get('algorithm', None)
        if not algos:
            if self.default_plugin:
                algos = [self.default_plugin.name, ]
            else:
                raise Error(
                    status=404,
                    message="No default plugin found, and None provided")
88

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
89
90
91
92
93
94
95
96
97
        plugins = list()
        for algo in algos:
            if algo not in self.plugins:
                logger.debug(("The algorithm '{}' is not valid\n"
                              "Valid algorithms: {}").format(algo,
                                                             self.plugins.keys()))
                raise Error(
                    status=404,
                    message="The algorithm '{}' is not valid".format(algo))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
98

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
99
100
101
102
103
104
105
106
107
            if not self.plugins[algo].is_activated:
                logger.debug("Plugin not activated: {}".format(algo))
                raise Error(
                    status=400,
                    message=("The algorithm '{}'"
                             " is not activated yet").format(algo))
            plugins.append(self.plugins[algo])
        return plugins

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
108
    def _process_entries(self, entries, req, plugins):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109
110
111
112
113
        if not plugins:
            for i in entries:
                yield i
            return
        plugin = plugins[0]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
114
115
116
        specific_params = api.get_extra_params(req, plugin)
        req.analysis.append({'plugin': plugin,
                             'parameters': specific_params})
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
117
        results = plugin.analyse_entries(entries, specific_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
118
        for i in self._process_entries(results, req, plugins[1:]):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
119
120
            yield i

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
121
    def analyse(self, request):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
122
123
        """
        Main method that analyses a request, either from CLI or HTTP.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
124
125
        It takes a processed request, provided by the user, as returned
        by api.parse_call().
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
126
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
127
        logger.debug("analysing request: {}".format(request))
128
        try:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
129
130
131
132
133
134
135
136
137
138
            entries = request.entries
            request.entries = []
            plugins = self._get_plugins(request)
            results = request
            for i in self._process_entries(entries, results, plugins):
                results.entries.append(i)
            self.convert_emotions(results)
            if 'with_parameters' not in results.parameters:
                del results.parameters
            logger.debug("Returning analysis result: {}".format(results))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
139
140
        except (Error, Exception) as ex:
            if not isinstance(ex, Error):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
141
142
143
                msg = "Error during analysis: {} \n\t{}".format(ex,
                                                                traceback.format_exc())
                ex = Error(message=msg, status=500)
144
            logger.exception('Error returning analysis result')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
145
            raise ex
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
146
147
        results.analysis = [i['plugin'].id for i in results.analysis]
        return results
148

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
149
    def _conversion_candidates(self, fromModel, toModel):
150
        candidates = self.filter_plugins(plugin_type='emotionConversionPlugin')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
151
152
153
154
155
156
157
158
159
        for name, candidate in candidates.items():
            for pair in candidate.onyx__doesConversion:
                logging.debug(pair)

                if pair['onyx:conversionFrom'] == fromModel \
                   and pair['onyx:conversionTo'] == toModel:
                    # logging.debug('Found candidate: {}'.format(candidate))
                    yield candidate

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
160
    def convert_emotions(self, resp):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
161
        """
162
        Conversion of all emotions in a response **in place**.
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
163
164
165
166
167
        In addition to converting from one model to another, it has
        to include the conversion plugin to the analysis list.
        Needless to say, this is far from an elegant solution, but it works.
        @todo refactor and clean up
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
168
169
        plugins = [i['plugin'] for i in resp.analysis]
        params = resp.parameters
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
170
171
172
        toModel = params.get('emotionModel', None)
        if not toModel:
            return
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187

        logger.debug('Asked for model: {}'.format(toModel))
        output = params.get('conversion', None)
        candidates = {}
        for plugin in plugins:
            try:
                fromModel = plugin.get('onyx:usesEmotionModel', None)
                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
            except StopIteration:
                e = Error(('No conversion plugin found for: '
                           '{} -> {}'.format(fromModel, toModel)))
                e.original_response = resp
                e.parameters = params
                raise e
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
188
189
190
        newentries = []
        for i in resp.entries:
            if output == "full":
191
                newemotions = copy.deepcopy(i.emotions)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
192
193
194
            else:
                newemotions = []
            for j in i.emotions:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
195
196
                plugname = j['prov:wasGeneratedBy']
                candidate = candidates[plugname]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
197
198
                resp.analysis.append({'plugin': candidate,
                                      'parameters': params})
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
199
200
201
202
203
204
205
206
207
                for k in candidate.convert(j, fromModel, toModel, params):
                    k.prov__wasGeneratedBy = candidate.id
                    if output == 'nested':
                        k.prov__wasDerivedFrom = j
                    newemotions.append(k)
            i.emotions = newemotions
            newentries.append(i)
        resp.entries = newentries

208
209
    @property
    def default_plugin(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
210
211
        candidate = self._default
        if not candidate:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
212
213
            candidates = self.filter_plugins(plugin_type='analysisPlugin',
                                             is_activated=True)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
214
215
216
217
218
219
220
221
222
            if len(candidates) > 0:
                candidate = list(candidates.values())[0]
        logger.debug("Default: {}".format(candidate))
        return candidate

    @default_plugin.setter
    def default_plugin(self, value):
        if isinstance(value, SenpyPlugin):
            self._default = value
223
        else:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
224
            self._default = self.plugins[value]
225

226
227
228
229
230
231
232
233
234
235
236
237
238
    def activate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.activate_plugin(plug, sync=sync))
        return ps

    def deactivate_all(self, sync=False):
        ps = []
        for plug in self.plugins.keys():
            ps.append(self.deactivate_plugin(plug, sync=sync))
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
239
240
241
        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
        Note that plugins may activate themselves by setting this variable.
        '''
242
243
244
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
245
246
247
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
248
249
250
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)

251
        logger.info("Activating plugin: {}".format(plugin.name))
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
252

253
        def act():
254
            success = False
255
256
            try:
                plugin.activate()
257
258
259
260
                msg = "Plugin activated: {}".format(plugin.name)
                logger.info(msg)
                success = True
                self._set_active_plugin(plugin_name, success)
261
            except Exception as ex:
262
                msg = "Error activating plugin {} - {} : \n\t{}".format(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
263
                    plugin.name, ex, traceback.format_exc())
264
265
                logger.error(msg)
                raise Error(msg)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
266

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
267
        if sync or 'async' in plugin and not plugin.async:
268
            act()
269
        else:
270
271
            th = Thread(target=act)
            th.start()
272
            return th
273
274

    def deactivate_plugin(self, plugin_name, sync=False):
J. Fernando Sánchez's avatar
--amend    
J. Fernando Sánchez committed
275
276
277
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
278
279
            raise Error(
                message="Plugin not found: {}".format(plugin_name), status=404)
280

281
282
        self._set_active_plugin(plugin_name, False)

283
284
285
286
287
        def deact():
            try:
                plugin.deactivate()
                logger.info("Plugin deactivated: {}".format(plugin.name))
            except Exception as ex:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
288
289
                logger.error(
                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
290
291
                logger.error("Trace: {}".format(traceback.format_exc()))

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
292
        if sync or 'async' in plugin and not plugin.async:
293
            deact()
294
        else:
295
296
            th = Thread(target=deact)
            th.start()
297
            return th
298

299
300
301
302
303
    def teardown(self, exception):
        pass

    @property
    def plugins(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
304
        """ Return the plugins registered for a given application.  """
305
        if self._outdated:
306
307
            self._plugin_list = plugins.load_plugins(self._search_folders)
            self._outdated = False
308
        return self._plugin_list
309

310
    def filter_plugins(self, **kwargs):
311
        return plugins.pfilter(self.plugins, **kwargs)
312
313
314
315
316

    @property
    def analysis_plugins(self):
        """ Return only the analysis plugins """
        return self.filter_plugins(plugin_type='analysisPlugin')