test_plugins.py 11.4 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1
2
#!/bin/env python

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
import os
4
import pickle
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
5
6
import shutil
import tempfile
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
7

8
from unittest import TestCase, skipIf
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
9
from senpy.models import Results, Entry, EmotionSet, Emotion, Plugins
10
from senpy import plugins
11
from senpy.plugins.conversion.emotion.centroids import CentroidConversion
12
from senpy.gsitk_compat import GSITK_AVAILABLE
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
13

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
14
15
import pandas as pd

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
16

17
class ShelfDummyPlugin(plugins.SentimentPlugin, plugins.ShelfMixin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
18
    '''Dummy plugin for tests.'''
19
20
21
22
    name = 'Shelf'
    version = 0
    author = 'the senpy community'

23
24
25
26
    def activate(self, *args, **kwargs):
        if 'counter' not in self.sh:
            self.sh['counter'] = 0
        self.save()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
27

28
29
    def deactivate(self, *args, **kwargs):
        self.save()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
30

31
    def analyse(self, *args, **kwargs):
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
32
        self.sh['counter'] = self.sh['counter'] + 1
33
34
35
36
37
        e = Entry()
        e.nif__isString = self.sh['counter']
        r = Results()
        r.entries.append(e)
        return r
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
38
39


J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
40
class PluginsTest(TestCase):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
41
    def tearDown(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
42
43
        if os.path.exists(self.shelf_dir):
            shutil.rmtree(self.shelf_dir)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
44
45
        if os.path.isfile(self.shelf_file):
            os.remove(self.shelf_file)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
46

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
47
48
49
    def setUp(self):
        self.shelf_dir = tempfile.mkdtemp()
        self.shelf_file = os.path.join(self.shelf_dir, "shelf")
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
50

51
52
53
54
55
56
57
58
59
60
    def test_serialize(self):
        '''A plugin should be serializable and de-serializable'''
        dummy = ShelfDummyPlugin()
        dummy.serialize()

    def test_jsonld(self):
        '''A plugin should be serializable and de-serializable'''
        dummy = ShelfDummyPlugin()
        dummy.jsonld()

61
    def test_shelf_file(self):
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
62
63
        a = ShelfDummyPlugin(
            info={'name': 'default_shelve_file',
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
64
                  'description': 'Dummy plugin for tests',
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
65
                  'version': 'test'})
66
67
68
69
        a.activate()
        assert os.path.isfile(a.shelf_file)
        os.remove(a.shelf_file)

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
    def test_plugin_filter(self):
        ps = Plugins()
        for i in (plugins.SentimentPlugin,
                  plugins.EmotionPlugin,
                  plugins.AnalysisPlugin):
            p = i(name='Plugin_{}'.format(i.__name__),
                  description='TEST',
                  version=0,
                  author='NOBODY')
            ps.plugins.append(p)
        assert len(ps.plugins) == 3
        cases = [('AnalysisPlugin', 3),
                 ('SentimentPlugin', 1),
                 ('EmotionPlugin', 1)]

        for name, num in cases:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
86
            res = list(plugins.pfilter(ps.plugins, plugin_type=name))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
87
88
            assert len(res) == num

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
89
    def test_shelf(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
90
        ''' A shelf is created and the value is stored '''
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
91
        newfile = self.shelf_file + "new"
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
92
93
        a = ShelfDummyPlugin(info={
            'name': 'shelve',
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
94
            'description': 'Shelf plugin for tests',
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
95
            'version': 'test',
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
96
            'shelf_file': newfile
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
97
        })
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
98
        assert a.sh == {}
99
100
        a.activate()
        assert a.sh == {'counter': 0}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
101
        assert a.shelf_file == newfile
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
102
103

        a.sh['a'] = 'fromA'
104
        assert a.sh['a'] == 'fromA'
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
105

106
107
        a.save()

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
108
        sh = pickle.load(open(newfile, 'rb'))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
110
111
        assert sh['a'] == 'fromA'

112
    def test_dummy_shelf(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
113
114
        with open(self.shelf_file, 'wb') as f:
            pickle.dump({'counter': 99}, f)
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
115
116
        a = ShelfDummyPlugin(info={
            'name': 'DummyShelf',
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
117
            'description': 'Dummy plugin for tests',
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
118
119
120
            'shelf_file': self.shelf_file,
            'version': 'test'
        })
121
122
        a.activate()

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
123
        assert a.shelf_file == self.shelf_file
124
        res1 = a.analyse(input=1)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
125
126
127
128
129
130
131
        assert res1.entries[0].nif__isString == 100
        a.deactivate()
        del a

        with open(self.shelf_file, 'rb') as f:
            sh = pickle.load(f)
            assert sh['counter'] == 100
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
132

133
134
135
136
137
138
139
140
    def test_corrupt_shelf(self):
        ''' Reusing the values of a previous shelf '''

        emptyfile = os.path.join(self.shelf_dir, "emptyfile")
        invalidfile = os.path.join(self.shelf_dir, "invalid_file")
        with open(emptyfile, 'w+b'), open(invalidfile, 'w+b') as inf:
            inf.write(b'ohno')

141
142
        files = {emptyfile: ['empty file', (EOFError, IndexError)],
                 invalidfile: ['invalid file', (pickle.UnpicklingError, IndexError)]}
143
144
145
146
147

        for fn in files:
            with open(fn, 'rb') as f:
                msg, error = files[fn]
                a = ShelfDummyPlugin(info={
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
148
149
                    'name': 'test_corrupt_shelf_{}'.format(msg),
                    'description': 'Dummy plugin for tests',
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
                    'version': 'test',
                    'shelf_file': f.name
                })
                assert os.path.isfile(a.shelf_file)
                print('Shelf file: %s' % a.shelf_file)
                with self.assertRaises(error):
                    a.sh['a'] = 'fromA'
                    a.save()
                del a._sh
                assert os.path.isfile(a.shelf_file)
                a.force_shelf = True
                a.sh['a'] = 'fromA'
                a.save()
                b = pickle.load(f)
                assert b['a'] == 'fromA'

    def test_reuse_shelf(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
167
        ''' Reusing the values of a previous shelf '''
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
168
169
        a = ShelfDummyPlugin(info={
            'name': 'shelve',
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
170
            'description': 'Dummy plugin for tests',
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
171
172
173
            'version': 'test',
            'shelf_file': self.shelf_file
        })
174
        a.activate()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
175
176
        print('Shelf file: %s' % a.shelf_file)
        a.sh['a'] = 'fromA'
177
        a.save()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
178

J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
179
180
        b = ShelfDummyPlugin(info={
            'name': 'shelve',
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
181
            'description': 'Dummy plugin for tests',
J. Fernando Sánchez's avatar
YAPFed    
J. Fernando Sánchez committed
182
183
184
            'version': 'test',
            'shelf_file': self.shelf_file
        })
185
186
        b.activate()
        assert b.sh['a'] == 'fromA'
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
187
188
        b.sh['a'] = 'fromB'
        assert b.sh['a'] == 'fromB'
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
189
190
191
192
193

    def test_extra_params(self):
        ''' Should be able to set extra parameters'''
        a = ShelfDummyPlugin(info={
            'name': 'shelve',
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
194
            'description': 'Dummy shelf plugin for tests',
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
195
196
197
198
199
200
201
202
203
204
205
            'version': 'test',
            'shelf_file': self.shelf_file,
            'extra_params': {
                'example': {
                    'aliases': ['example', 'ex'],
                    'required': True,
                    'default': 'nonsense'
                }
            }
        })
        assert 'example' in a.extra_params
206

207
208
209
210
211
212
213
214
215
216
217
    def test_box(self):

        class MyBox(plugins.Box):
            ''' Vague description'''

            author = 'me'
            version = 0

            def input(self, entry, **kwargs):
                return entry.text

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
218
            def predict_one(self, input):
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
                return 'SIGN' in input

            def output(self, output, entry, **kwargs):
                if output:
                    entry.myAnnotation = 'DETECTED'
                return entry

            test_cases = [
                {
                    'input': "nothing here",
                    'expected': {'myAnnotation': 'DETECTED'},
                    'should_fail': True
                }, {
                    'input': "SIGN",
                    'expected': {'myAnnotation': 'DETECTED'}
                }]

        MyBox().test()

    def test_sentimentbox(self):

        class SentimentBox(plugins.MappingMixin, plugins.SentimentBox):
            ''' Vague description'''

            author = 'me'
            version = 0

            mappings = {'happy': 'marl:Positive', 'sad': 'marl:Negative'}

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
248
            def predict_one(self, input, **kwargs):
249
250
251
252
253
254
255
256
257
258
259
260
261
                return 'happy' if ':)' in input else 'sad'

            test_cases = [
                {
                    'input': 'a happy face :)',
                    'polarity': 'marl:Positive'
                }, {
                    'input': "Nothing",
                    'polarity': 'marl:Negative'
                }]

        SentimentBox().test()

262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
    def test_conversion_centroids(self):
        info = {
            "name": "CentroidTest",
            "description": "Centroid test",
            "version": 0,
            "centroids": {
                "c1": {"V1": 0.5,
                       "V2": 0.5},
                "c2": {"V1": -0.5,
                       "V2": 0.5},
                "c3": {"V1": -0.5,
                       "V2": -0.5},
                "c4": {"V1": 0.5,
                       "V2": -0.5}},
            "aliases": {
                "V1": "X-dimension",
                "V2": "Y-dimension"
            },
            "centroids_direction": ["emoml:big6", "emoml:fsre-dimensions"]
        }
        c = CentroidConversion(info)
283
        print(c.serialize())
284
285
286
287
288
289
290
291

        es1 = EmotionSet()
        e1 = Emotion()
        e1.onyx__hasEmotionCategory = "c1"
        es1.onyx__hasEmotion.append(e1)
        res = c._forward_conversion(es1)
        assert res["X-dimension"] == 0.5
        assert res["Y-dimension"] == 0.5
292
        print(res)
293
294
295
296
297
298
299

        e2 = Emotion()
        e2.onyx__hasEmotionCategory = "c2"
        es1.onyx__hasEmotion.append(e2)
        res = c._forward_conversion(es1)
        assert res["X-dimension"] == 0
        assert res["Y-dimension"] == 1
300
        print(res)
301
302
303
304
305
306

        e = Emotion()
        e["X-dimension"] = -0.2
        e["Y-dimension"] = -0.3
        res = c._backwards_conversion(e)
        assert res["onyx:hasEmotionCategory"] == "c3"
307
        print(res)
308
309
310
311
312
313

        e = Emotion()
        e["X-dimension"] = -0.2
        e["Y-dimension"] = 0.3
        res = c._backwards_conversion(e)
        assert res["onyx:hasEmotionCategory"] == "c2"
314

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
315
    def _test_evaluation(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
        testdata = []
        for i in range(50):
            testdata.append(["good", 1])
        for i in range(50):
            testdata.append(["bad", 0])
        dataset = pd.DataFrame(testdata, columns=['text', 'polarity'])

        class DummyPlugin(plugins.TextBox):
            description = 'Plugin to test evaluation'
            version = 0

            def predict_one(self, input):
                return 0

        class SmartPlugin(plugins.TextBox):
            description = 'Plugin to test evaluation'
            version = 0

            def predict_one(self, input):
                if input == 'good':
                    return 1
                return 0

        dpipe = DummyPlugin()
        results = plugins.evaluate(datasets={'testdata': dataset}, plugins=[dpipe], flatten=True)
        dumb_metrics = results[0].metrics[0]
        assert abs(dumb_metrics['accuracy'] - 0.5) < 0.01

        spipe = SmartPlugin()
        results = plugins.evaluate(datasets={'testdata': dataset}, plugins=[spipe], flatten=True)
        smart_metrics = results[0].metrics[0]
        assert abs(smart_metrics['accuracy'] - 1) < 0.01

349
    @skipIf(not GSITK_AVAILABLE, "GSITK is not available")
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
350
    def test_evaluation(self):
351
352
353
354
355
        self._test_evaluation()

    @skipIf(GSITK_AVAILABLE, "GSITK is available")
    def test_evaluation_unavailable(self):
        with self.assertRaises(Exception) as context:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
356
            self._test_evaluation()
357
        self.assertTrue('GSITK ' in str(context.exception))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
358

359

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
360
def make_mini_test(fpath):
361
    def mini_test(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
362
363
        for plugin in plugins.from_path(fpath, install=True):
            plugin.test()
364
365
366
    return mini_test


367
def _add_tests():
368
369
    root = os.path.join(os.path.dirname(__file__), '..')
    print(root)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
370
    for fpath in plugins.find_plugins([root, ]):
371
        pass
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
372
373
        t_method = make_mini_test(fpath)
        t_method.__name__ = 'test_plugin_{}'.format(fpath)
374
375
376
377
        setattr(PluginsTest, t_method.__name__, t_method)
        del t_method


378
_add_tests()