__init__.py 9.42 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
from future import standard_library
standard_library.install_aliases()
3

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
import os.path
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
5
import os
6
import pickle
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
7
import logging
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8
import copy
9
10
11
12
13
14
15

import fnmatch
import inspect
import sys
import subprocess
import importlib
import yaml
16
import threading
17

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
18
from .. import models, utils
19
from ..api import API_PARAMS
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
20
21
22

logger = logging.getLogger(__name__)

23

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
24
class Plugin(models.Plugin):
25
    def __init__(self, info=None, data_folder=None):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
26
27
28
29
        """
        Provides a canonical name for plugins and serves as base for other
        kinds of plugins.
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
30
31
32
33
34
        logger.debug("Initialising {}".format(info))
        if info:
            self.update(info)
        super(Plugin, self).__init__(**self)
        if not self.validate():
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
35
36
            raise models.Error(message=("You need to provide configuration"
                                        "information for the plugin."))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
37
        self.id = 'plugins/{}_{}'.format(self['name'], self['version'])
38
        self.is_activated = False
39
        self._lock = threading.Lock()
40
        self.data_folder = data_folder or os.getcwd()
41

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
42
43
44
    def validate(self):
        return all(x in self for x in ('name', 'description', 'version'))

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
45
46
47
    def get_folder(self):
        return os.path.dirname(inspect.getfile(self.__class__))

48
49
50
51
52
53
    def activate(self):
        pass

    def deactivate(self):
        pass

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
54
    def test(self):
55
56
57
        if not hasattr(self, 'test_cases'):
            raise AttributeError(('Plugin {} [{}] does not have any defined '
                                  'test cases').format(self.id, inspect.getfile(self.__class__)))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
58
        for case in self.test_cases:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
59
60
61
62
63
64
65
66
67
68
69
70
            entry = models.Entry(case['entry'])
            params = case.get('params', {})
            fails = case.get('fails', False)
            try:
                res = list(self.analyse_entry(entry, params))
            except models.Error:
                if fails:
                    continue
                raise
            if fails:
                raise Exception('This test should have raised an exception.')

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
71
72
73
            exp = case['expected']
            if not isinstance(exp, list):
                exp = [exp]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
74
            utils.check_template(res, exp)
75
76
            for r in res:
                r.validate()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
77

78
79
80
    def open(self, fpath, *args, **kwargs):
        if not os.path.isabs(fpath):
            fpath = os.path.join(self.data_folder, fpath)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
81
82
83
84
        return open(fpath, *args, **kwargs)

    def serve(self, **kwargs):
        utils.serve(plugin=self, **kwargs)
85

86

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
87
88
89
90
SenpyPlugin = Plugin


class AnalysisPlugin(Plugin):
91

92
    def analyse(self, *args, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
93
94
95
96
97
98
99
100
101
102
        raise NotImplemented(
            'Your method should implement either analyse or analyse_entry')

    def analyse_entry(self, entry, parameters):
        """ An implemented plugin should override this method.
        This base method is here to adapt old style plugins which only
        implement the *analyse* function.
        Note that this method may yield an annotated entry or a list of
        entries (e.g. in a tokenizer)
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
103
        text = entry['nif:isString']
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
104
105
106
107
108
        params = copy.copy(parameters)
        params['input'] = text
        results = self.analyse(**params)
        for i in results.entries:
            yield i
109

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
110
111
112
113
114
115
    def analyse_entries(self, entries, parameters):
        for entry in entries:
            logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
            for result in self.analyse_entry(entry, parameters):
                yield result

116

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
117
class ConversionPlugin(Plugin):
118
    pass
119

120

121
class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
122
123
    minPolarityValue = 0
    maxPolarityValue = 1
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
124

125

126
class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
127
128
    minEmotionValue = 0
    maxEmotionValue = 1
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
129
130


131
132
class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin):
    pass
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
133
134
135
136
137


class ShelfMixin(object):
    @property
    def sh(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
138
        if not hasattr(self, '_sh') or self._sh is None:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
139
            self._sh = {}
140
            if os.path.isfile(self.shelf_file):
141
                try:
142
                    with self.open(self.shelf_file, 'rb') as p:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
143
                        self._sh = pickle.load(p)
144
                except (IndexError, EOFError, pickle.UnpicklingError):
145
146
147
                    logger.warning('{} has a corrupted shelf file!'.format(self.id))
                    if not self.get('force_shelf', False):
                        raise
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
148
149
150
151
        return self._sh

    @sh.deleter
    def sh(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
152
153
        if os.path.isfile(self.shelf_file):
            os.remove(self.shelf_file)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
154
            del self._sh
155
        self.save()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
156

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
157
158
159
160
    @sh.setter
    def sh(self, value):
        self._sh = value

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
161
162
    @property
    def shelf_file(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
163
        if 'shelf_file' not in self or not self['shelf_file']:
164
            self.shelf_file = os.path.join(self.data_folder, self.name + '.p')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
165
        return self['shelf_file']
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
166

167
    def save(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
168
        logger.debug('saving pickle')
169
        if hasattr(self, '_sh') and self._sh is not None:
170
            with self.open(self.shelf_file, 'wb') as f:
171
                pickle.dump(self._sh, f)
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208


default_plugin_type = API_PARAMS['plugin_type']['default']


def pfilter(plugins, **kwargs):
    """ Filter plugins by different criteria """
    if isinstance(plugins, models.Plugins):
        plugins = plugins.plugins
    elif isinstance(plugins, dict):
        plugins = plugins.values()
    ptype = kwargs.pop('plugin_type', default_plugin_type)
    logger.debug('#' * 100)
    logger.debug('ptype {}'.format(ptype))
    if ptype:
        try:
            ptype = ptype[0].upper() + ptype[1:]
            pclass = globals()[ptype]
            logger.debug('Class: {}'.format(pclass))
            candidates = filter(lambda x: isinstance(x, pclass),
                                plugins)
        except KeyError:
            raise models.Error('{} is not a valid type'.format(ptype))
    else:
        candidates = plugins

    logger.debug(candidates)

    def matches(plug):
        res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
        logger.debug(
            "matching {} with {}: {}".format(plug.name, kwargs, res))
        return res

    if kwargs:
        candidates = filter(matches, candidates)
    return {p.name: p for p in candidates}
209
210
211


def validate_info(info):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
212
    return all(x in info for x in ('name',))
213
214


215
216
217
def load_module(name, root=None):
    if root:
        sys.path.append(root)
218
    tmp = importlib.import_module(name)
219
220
    if root:
        sys.path.remove(root)
221
222
223
224
225
226
227
228
229
230
231
    return tmp


def log_subprocess_output(process):
    for line in iter(process.stdout.readline, b''):
        logger.info('%r', line)
    for line in iter(process.stderr.readline, b''):
        logger.error('%r', line)


def install_deps(*plugins):
232
    installed = False
233
234
235
    for info in plugins:
        requirements = info.get('requirements', [])
        if requirements:
236
            pip_args = [sys.executable, '-m', 'pip', 'install', '--use-wheel']
237
238
239
240
241
242
243
244
            for req in requirements:
                pip_args.append(req)
            logger.info('Installing requirements: ' + str(requirements))
            process = subprocess.Popen(pip_args,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
            log_subprocess_output(process)
            exitcode = process.wait()
245
            installed = True
246
247
            if exitcode != 0:
                raise models.Error("Dependencies not properly installed")
248
    return installed
249
250


J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
251
252
253
254
255
256
257
258
259
260
261
def get_plugin_class(module):
    candidate = None
    for _, obj in inspect.getmembers(module):
        if inspect.isclass(obj) and inspect.getmodule(obj) == module:
            logger.debug(("Found plugin class:"
                          " {}@{}").format(obj, inspect.getmodule(obj)))
            candidate = obj
            break
    return candidate


262
def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs):
263
264
    if not root and '_path' in info:
        root = os.path.dirname(info['_path'])
265
    if not validator(info):
266
        raise ValueError('Plugin info is not valid: {}'.format(info))
267
268
    module = info["module"]

269
270
271
272
273
274
275
    try:
        tmp = load_module(module, root)
    except ImportError:
        if not install:
            raise
        install_deps(info)
        tmp = load_module(module, root)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
276
277
278
279
280
281
    cls = None
    if '@type' not in info:
        cls = get_plugin_class(tmp)
    if not cls:
        raise Exception("No valid plugin for: {}".format(module))
    return cls(info=info, *args, **kwargs)
282
283


284
def parse_plugin_info(fpath):
285
286
287
    logger.debug("Loading plugin: {}".format(fpath))
    with open(fpath, 'r') as f:
        info = yaml.load(f)
288
289
290
291
292
    info['_path'] = fpath
    name = info['name']
    return name, info


293
def load_plugin(fpath, *args, **kwargs):
294
    name, info = parse_plugin_info(fpath)
295
    logger.debug("Info: {}".format(info))
296
    plugin = load_plugin_from_info(info, *args, **kwargs)
297
    return name, plugin
298
299


300
def load_plugins(folders, loader=load_plugin, *args, **kwargs):
301
302
303
    plugins = {}
    for search_folder in folders:
        for root, dirnames, filenames in os.walk(search_folder):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
304
305
            # Do not look for plugins in hidden or special folders
            dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
306
            for filename in fnmatch.filter(filenames, '*.senpy'):
307
                fpath = os.path.join(root, filename)
308
                name, plugin = loader(fpath, *args, **kwargs)
309
310
311
                if plugin and name:
                    plugins[name] = plugin
    return plugins