__init__.py 9.33 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
from future import standard_library
standard_library.install_aliases()
3

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
import os.path
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
5
import os
6
import pickle
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
7
import logging
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8
import copy
9
10
11
12
13
14
15

import fnmatch
import inspect
import sys
import subprocess
import importlib
import yaml
16
import threading
17

18
19
from contextlib import contextmanager

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
20
from .. import models, utils
21
from ..api import API_PARAMS
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
22
23
24

logger = logging.getLogger(__name__)

25

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
26
class Plugin(models.Plugin):
27
    def __init__(self, info=None, data_folder=None):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
28
29
30
31
        """
        Provides a canonical name for plugins and serves as base for other
        kinds of plugins.
        """
32
        if not info:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
33
34
            raise models.Error(message=("You need to provide configuration"
                                        "information for the plugin."))
35
        logger.debug("Initialising {}".format(info))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
36
        id = 'plugins/{}_{}'.format(info['name'], info['version'])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
37
        super(Plugin, self).__init__(id=id, **info)
38
        self.is_activated = False
39
        self._lock = threading.Lock()
40
        self.data_folder = data_folder or os.getcwd()
41

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
42
43
44
    def get_folder(self):
        return os.path.dirname(inspect.getfile(self.__class__))

45
46
47
48
49
50
    def activate(self):
        pass

    def deactivate(self):
        pass

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
51
    def test(self):
52
53
54
55
        if not hasattr(self, 'test_cases'):
            import inspect
            raise AttributeError(('Plugin {} [{}] does not have any defined '
                                  'test cases').format(self.id, inspect.getfile(self.__class__)))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
56
57
58
59
60
61
        for case in self.test_cases:
            res = list(self.analyse_entry(models.Entry(case['entry']),
                                          case['params']))
            exp = case['expected']
            if not isinstance(exp, list):
                exp = [exp]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
62
            utils.check_template(res, exp)
63
64
            for r in res:
                r.validate()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
65

66
67
68
69
70
71
72
    @contextmanager
    def open(self, fpath, *args, **kwargs):
        if not os.path.isabs(fpath):
            fpath = os.path.join(self.data_folder, fpath)
        with open(fpath, *args, **kwargs) as f:
            yield f

73

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
74
75
76
77
SenpyPlugin = Plugin


class AnalysisPlugin(Plugin):
78

79
    def analyse(self, *args, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
80
81
82
83
84
85
86
87
88
89
        raise NotImplemented(
            'Your method should implement either analyse or analyse_entry')

    def analyse_entry(self, entry, parameters):
        """ An implemented plugin should override this method.
        This base method is here to adapt old style plugins which only
        implement the *analyse* function.
        Note that this method may yield an annotated entry or a list of
        entries (e.g. in a tokenizer)
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
90
        text = entry['nif:isString']
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
91
92
93
94
95
        params = copy.copy(parameters)
        params['input'] = text
        results = self.analyse(**params)
        for i in results.entries:
            yield i
96

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
97
98
99
100
101
102
    def analyse_entries(self, entries, parameters):
        for entry in entries:
            logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
            for result in self.analyse_entry(entry, parameters):
                yield result

103

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
104
class ConversionPlugin(Plugin):
105
    pass
106

107

108
class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin):
109
110
111
112
    def __init__(self, info, *args, **kwargs):
        super(SentimentPlugin, self).__init__(info, *args, **kwargs)
        self.minPolarityValue = float(info.get("minPolarityValue", 0))
        self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
113

114

115
class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin):
116
    def __init__(self, info, *args, **kwargs):
117
        super(EmotionPlugin, self).__init__(info, *args, **kwargs)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
118
119
120
121
        self.minEmotionValue = float(info.get("minEmotionValue", -1))
        self.maxEmotionValue = float(info.get("maxEmotionValue", 1))


122
123
class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin):
    pass
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
124
125
126
127
128


class ShelfMixin(object):
    @property
    def sh(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
129
        if not hasattr(self, '_sh') or self._sh is None:
130
131
            self.__dict__['_sh'] = {}
            if os.path.isfile(self.shelf_file):
132
                try:
133
134
                    with self.open(self.shelf_file, 'rb') as p:
                        self.__dict__['_sh'] = pickle.load(p)
135
                except (IndexError, EOFError, pickle.UnpicklingError):
136
137
138
                    logger.warning('{} has a corrupted shelf file!'.format(self.id))
                    if not self.get('force_shelf', False):
                        raise
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
139
140
141
142
        return self._sh

    @sh.deleter
    def sh(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
143
144
        if os.path.isfile(self.shelf_file):
            os.remove(self.shelf_file)
145
146
            del self.__dict__['_sh']
        self.save()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
147

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
148
149
    @property
    def shelf_file(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
150
        if 'shelf_file' not in self or not self['shelf_file']:
151
            self.shelf_file = os.path.join(self.data_folder, self.name + '.p')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
152
        return self['shelf_file']
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
153

154
    def save(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
155
        logger.debug('saving pickle')
156
        if hasattr(self, '_sh') and self._sh is not None:
157
            with self.open(self.shelf_file, 'wb') as f:
158
                pickle.dump(self._sh, f)
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195


default_plugin_type = API_PARAMS['plugin_type']['default']


def pfilter(plugins, **kwargs):
    """ Filter plugins by different criteria """
    if isinstance(plugins, models.Plugins):
        plugins = plugins.plugins
    elif isinstance(plugins, dict):
        plugins = plugins.values()
    ptype = kwargs.pop('plugin_type', default_plugin_type)
    logger.debug('#' * 100)
    logger.debug('ptype {}'.format(ptype))
    if ptype:
        try:
            ptype = ptype[0].upper() + ptype[1:]
            pclass = globals()[ptype]
            logger.debug('Class: {}'.format(pclass))
            candidates = filter(lambda x: isinstance(x, pclass),
                                plugins)
        except KeyError:
            raise models.Error('{} is not a valid type'.format(ptype))
    else:
        candidates = plugins

    logger.debug(candidates)

    def matches(plug):
        res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
        logger.debug(
            "matching {} with {}: {}".format(plug.name, kwargs, res))
        return res

    if kwargs:
        candidates = filter(matches, candidates)
    return {p.name: p for p in candidates}
196
197
198
199
200
201


def validate_info(info):
    return all(x in info for x in ('name', 'module', 'description', 'version'))


202
203
204
def load_module(name, root=None):
    if root:
        sys.path.append(root)
205
    tmp = importlib.import_module(name)
206
207
    if root:
        sys.path.remove(root)
208
209
210
211
212
213
214
215
216
217
218
    return tmp


def log_subprocess_output(process):
    for line in iter(process.stdout.readline, b''):
        logger.info('%r', line)
    for line in iter(process.stderr.readline, b''):
        logger.error('%r', line)


def install_deps(*plugins):
219
    installed = False
220
221
222
    for info in plugins:
        requirements = info.get('requirements', [])
        if requirements:
223
            pip_args = [sys.executable, '-m', 'pip', 'install', '--use-wheel']
224
225
226
227
228
229
230
231
            for req in requirements:
                pip_args.append(req)
            logger.info('Installing requirements: ' + str(requirements))
            process = subprocess.Popen(pip_args,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
            log_subprocess_output(process)
            exitcode = process.wait()
232
            installed = True
233
234
            if exitcode != 0:
                raise models.Error("Dependencies not properly installed")
235
    return installed
236
237


238
def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs):
239
240
    if not root and '_path' in info:
        root = os.path.dirname(info['_path'])
241
    if not validator(info):
242
        raise ValueError('Plugin info is not valid: {}'.format(info))
243
244
    module = info["module"]

245
246
247
248
249
250
251
    try:
        tmp = load_module(module, root)
    except ImportError:
        if not install:
            raise
        install_deps(info)
        tmp = load_module(module, root)
252
253
254
255
256
257
258
259
260
261
    candidate = None
    for _, obj in inspect.getmembers(tmp):
        if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
            logger.debug(("Found plugin class:"
                          " {}@{}").format(obj, inspect.getmodule(obj)))
            candidate = obj
            break
    if not candidate:
        logger.debug("No valid plugin for: {}".format(module))
        return
262
    module = candidate(info=info, *args, **kwargs)
263
    return module
264
265


266
def parse_plugin_info(fpath):
267
268
269
    logger.debug("Loading plugin: {}".format(fpath))
    with open(fpath, 'r') as f:
        info = yaml.load(f)
270
271
272
273
274
    info['_path'] = fpath
    name = info['name']
    return name, info


275
def load_plugin(fpath, *args, **kwargs):
276
    name, info = parse_plugin_info(fpath)
277
    logger.debug("Info: {}".format(info))
278
    plugin = load_plugin_from_info(info, *args, **kwargs)
279
    return name, plugin
280
281


282
def load_plugins(folders, loader=load_plugin, *args, **kwargs):
283
284
285
    plugins = {}
    for search_folder in folders:
        for root, dirnames, filenames in os.walk(search_folder):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
286
287
            # Do not look for plugins in hidden or special folders
            dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
288
            for filename in fnmatch.filter(filenames, '*.senpy'):
289
                fpath = os.path.join(root, filename)
290
                name, plugin = loader(fpath, *args, **kwargs)
291
292
293
                if plugin and name:
                    plugins[name] = plugin
    return plugins