Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
senpy
senpy
Commits
4d7e8e75
Commit
4d7e8e75
authored
Jul 12, 2017
by
J. Fernando Sánchez
Browse files
Added tests for all "discoverable" plugins
Closes
#39
parent
8e4578dc
Changes
9
Hide whitespace changes
Inline
Side-by-side
senpy/extensions.py
View file @
4d7e8e75
...
...
@@ -15,25 +15,12 @@ from threading import Thread
import
os
import
copy
import
fnmatch
import
inspect
import
sys
import
importlib
import
logging
import
traceback
import
yaml
import
subprocess
logger
=
logging
.
getLogger
(
__name__
)
def
log_subprocess_output
(
process
):
for
line
in
iter
(
process
.
stdout
.
readline
,
b
''
):
logger
.
info
(
'%r'
,
line
)
for
line
in
iter
(
process
.
stderr
.
readline
,
b
''
):
logger
.
error
(
'%r'
,
line
)
class
Senpy
(
object
):
""" Default Senpy extension for Flask """
...
...
@@ -330,84 +317,6 @@ class Senpy(object):
th
.
start
()
return
th
@
classmethod
def
validate_info
(
cls
,
info
):
return
all
(
x
in
info
for
x
in
(
'name'
,
'module'
,
'description'
,
'version'
))
def
install_deps
(
self
):
for
i
in
self
.
plugins
.
values
():
self
.
_install_deps
(
i
)
@
classmethod
def
_install_deps
(
cls
,
info
=
None
):
requirements
=
info
.
get
(
'requirements'
,
[])
if
requirements
:
pip_args
=
[
'pip'
]
pip_args
.
append
(
'install'
)
pip_args
.
append
(
'--use-wheel'
)
for
req
in
requirements
:
pip_args
.
append
(
req
)
logger
.
info
(
'Installing requirements: '
+
str
(
requirements
))
process
=
subprocess
.
Popen
(
pip_args
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
)
log_subprocess_output
(
process
)
exitcode
=
process
.
wait
()
if
exitcode
!=
0
:
raise
Error
(
"Dependencies not properly installed"
)
@
classmethod
def
_load_module
(
cls
,
name
,
root
):
sys
.
path
.
append
(
root
)
tmp
=
importlib
.
import_module
(
name
)
sys
.
path
.
remove
(
root
)
return
tmp
@
classmethod
def
_load_plugin_from_info
(
cls
,
info
,
root
):
if
not
cls
.
validate_info
(
info
):
logger
.
warn
(
'The module info is not valid.
\n\t
{}'
.
format
(
info
))
return
None
,
None
module
=
info
[
"module"
]
name
=
info
[
"name"
]
cls
.
_install_deps
(
info
)
tmp
=
cls
.
_load_module
(
module
,
root
)
candidate
=
None
for
_
,
obj
in
inspect
.
getmembers
(
tmp
):
if
inspect
.
isclass
(
obj
)
and
inspect
.
getmodule
(
obj
)
==
tmp
:
logger
.
debug
((
"Found plugin class:"
" {}@{}"
).
format
(
obj
,
inspect
.
getmodule
(
obj
)))
candidate
=
obj
break
if
not
candidate
:
logger
.
debug
(
"No valid plugin for: {}"
.
format
(
module
))
return
module
=
candidate
(
info
=
info
)
return
name
,
module
@
classmethod
def
_load_plugin
(
cls
,
root
,
filename
):
fpath
=
os
.
path
.
join
(
root
,
filename
)
logger
.
debug
(
"Loading plugin: {}"
.
format
(
fpath
))
with
open
(
fpath
,
'r'
)
as
f
:
info
=
yaml
.
load
(
f
)
logger
.
debug
(
"Info: {}"
.
format
(
info
))
return
cls
.
_load_plugin_from_info
(
info
,
root
)
def
_load_plugins
(
self
):
plugins
=
{}
for
search_folder
in
self
.
_search_folders
:
for
root
,
dirnames
,
filenames
in
os
.
walk
(
search_folder
):
for
filename
in
fnmatch
.
filter
(
filenames
,
'*.senpy'
):
name
,
plugin
=
self
.
_load_plugin
(
root
,
filename
)
if
plugin
and
name
:
plugins
[
name
]
=
plugin
self
.
_outdated
=
False
return
plugins
def
teardown
(
self
,
exception
):
pass
...
...
@@ -415,7 +324,8 @@ class Senpy(object):
def
plugins
(
self
):
""" Return the plugins registered for a given application. """
if
self
.
_outdated
:
self
.
_plugin_list
=
self
.
_load_plugins
()
self
.
_plugin_list
=
plugins
.
load_plugins
(
self
.
_search_folders
)
self
.
_outdated
=
False
return
self
.
_plugin_list
def
filter_plugins
(
self
,
**
kwargs
):
...
...
senpy/models.py
View file @
4d7e8e75
...
...
@@ -223,12 +223,6 @@ class BaseModel(SenpyMixin, dict):
key
=
key
.
replace
(
"__"
,
":"
,
1
)
return
key
def
__getitem__
(
self
,
key
):
return
dict
.
__getitem__
(
self
,
self
.
_get_key
(
key
))
def
__setitem__
(
self
,
key
,
value
):
dict
.
__setitem__
(
self
,
self
.
_get_key
(
key
),
value
)
def
__delitem__
(
self
,
key
):
dict
.
__delitem__
(
self
,
key
)
...
...
senpy/plugins/__init__.py
View file @
4d7e8e75
from
future
import
standard_library
standard_library
.
install_aliases
()
import
inspect
import
os.path
import
os
import
pickle
import
logging
import
tempfile
import
copy
import
fnmatch
import
inspect
import
sys
import
subprocess
import
importlib
import
yaml
from
..
import
models
from
..api
import
API_PARAMS
...
...
@@ -69,6 +76,8 @@ class Plugin(models.Plugin):
if
not
isinstance
(
exp
,
list
):
exp
=
[
exp
]
check_template
(
res
,
exp
)
for
r
in
res
:
r
.
validate
()
SenpyPlugin
=
Plugin
...
...
@@ -193,3 +202,84 @@ def pfilter(plugins, **kwargs):
if
kwargs
:
candidates
=
filter
(
matches
,
candidates
)
return
{
p
.
name
:
p
for
p
in
candidates
}
def
validate_info
(
info
):
return
all
(
x
in
info
for
x
in
(
'name'
,
'module'
,
'description'
,
'version'
))
def
load_module
(
name
,
root
):
sys
.
path
.
append
(
root
)
tmp
=
importlib
.
import_module
(
name
)
sys
.
path
.
remove
(
root
)
return
tmp
def
log_subprocess_output
(
process
):
for
line
in
iter
(
process
.
stdout
.
readline
,
b
''
):
logger
.
info
(
'%r'
,
line
)
for
line
in
iter
(
process
.
stderr
.
readline
,
b
''
):
logger
.
error
(
'%r'
,
line
)
def
install_deps
(
*
plugins
):
for
info
in
plugins
:
requirements
=
info
.
get
(
'requirements'
,
[])
if
requirements
:
pip_args
=
[
'pip'
]
pip_args
.
append
(
'install'
)
pip_args
.
append
(
'--use-wheel'
)
for
req
in
requirements
:
pip_args
.
append
(
req
)
logger
.
info
(
'Installing requirements: '
+
str
(
requirements
))
process
=
subprocess
.
Popen
(
pip_args
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
)
log_subprocess_output
(
process
)
exitcode
=
process
.
wait
()
if
exitcode
!=
0
:
raise
models
.
Error
(
"Dependencies not properly installed"
)
def
load_plugin_from_info
(
info
,
root
,
validator
=
validate_info
):
if
not
validator
(
info
):
logger
.
warn
(
'The module info is not valid.
\n\t
{}'
.
format
(
info
))
return
None
,
None
module
=
info
[
"module"
]
name
=
info
[
"name"
]
install_deps
(
info
)
tmp
=
load_module
(
module
,
root
)
candidate
=
None
for
_
,
obj
in
inspect
.
getmembers
(
tmp
):
if
inspect
.
isclass
(
obj
)
and
inspect
.
getmodule
(
obj
)
==
tmp
:
logger
.
debug
((
"Found plugin class:"
" {}@{}"
).
format
(
obj
,
inspect
.
getmodule
(
obj
)))
candidate
=
obj
break
if
not
candidate
:
logger
.
debug
(
"No valid plugin for: {}"
.
format
(
module
))
return
module
=
candidate
(
info
=
info
)
return
name
,
module
def
load_plugin
(
root
,
filename
):
fpath
=
os
.
path
.
join
(
root
,
filename
)
logger
.
debug
(
"Loading plugin: {}"
.
format
(
fpath
))
with
open
(
fpath
,
'r'
)
as
f
:
info
=
yaml
.
load
(
f
)
logger
.
debug
(
"Info: {}"
.
format
(
info
))
return
load_plugin_from_info
(
info
,
root
)
def
load_plugins
(
folders
,
loader
=
load_plugin
):
plugins
=
{}
for
search_folder
in
folders
:
for
root
,
dirnames
,
filenames
in
os
.
walk
(
search_folder
):
for
filename
in
fnmatch
.
filter
(
filenames
,
'*.senpy'
):
name
,
plugin
=
loader
(
root
,
filename
)
if
plugin
and
name
:
plugins
[
name
]
=
plugin
return
plugins
senpy/plugins/example/emoRand/emoRand.py
View file @
4d7e8e75
...
...
@@ -21,6 +21,6 @@ class RmoRandPlugin(EmotionPlugin):
params
=
dict
()
results
=
list
()
for
i
in
range
(
100
):
res
=
next
(
self
.
analyse_entry
(
Entry
(
text
=
"Hello"
),
params
))
res
=
next
(
self
.
analyse_entry
(
Entry
(
nif__isString
=
"Hello"
),
params
))
res
.
validate
()
results
.
append
(
res
.
emotions
[
0
][
'onyx:hasEmotion'
][
0
][
'onyx:hasEmotionCategory'
])
senpy/plugins/example/rand/rand.py
View file @
4d7e8e75
...
...
@@ -27,7 +27,7 @@ class RandPlugin(SentimentPlugin):
params
=
dict
()
results
=
list
()
for
i
in
range
(
100
):
res
=
next
(
self
.
analyse_entry
(
Entry
(
text
=
"Hello"
),
params
))
res
=
next
(
self
.
analyse_entry
(
Entry
(
nif__isString
=
"Hello"
),
params
))
res
.
validate
()
results
.
append
(
res
.
sentiments
[
0
][
'marl:hasPolarity'
])
assert
'marl:Positive'
in
results
...
...
senpy/plugins/sentiment/sentiment140/sentiment140.py
View file @
4d7e8e75
...
...
@@ -12,7 +12,7 @@ class Sentiment140Plugin(SentimentPlugin):
json
.
dumps
({
"language"
:
lang
,
"data"
:
[{
"text"
:
entry
.
text
"text"
:
entry
.
nif__isString
}]
}))
p
=
params
.
get
(
"prefix"
,
None
)
...
...
@@ -38,11 +38,11 @@ class Sentiment140Plugin(SentimentPlugin):
test_cases
=
[
{
'entry'
:
{
'
text
'
:
'I love Titanic'
'
nif:isString
'
:
'I love Titanic'
},
'params'
:
{},
'expected'
:
{
"
text
"
:
"I love Titanic"
,
"
nif:isString
"
:
"I love Titanic"
,
'sentiments'
:
[
{
'marl:hasPolarity'
:
'marl:Positive'
,
...
...
senpy/schemas/context.jsonld
View file @
4d7e8e75
...
...
@@ -20,6 +20,9 @@
"@id": "me:hasSuggestions",
"@container": "@set"
},
"onyx:hasEmotion": {
"@container": "@set"
},
"emotions": {
"@id": "onyx:hasEmotionSet",
"@container": "@set"
...
...
tests/test_extensions.py
View file @
4d7e8e75
...
...
@@ -10,6 +10,7 @@ except ImportError:
from
functools
import
partial
from
senpy.extensions
import
Senpy
from
senpy
import
plugins
from
senpy.models
import
Error
,
Results
,
Entry
,
EmotionSet
,
Emotion
,
Plugin
from
flask
import
Flask
from
unittest
import
TestCase
...
...
@@ -18,7 +19,7 @@ from unittest import TestCase
class
ExtensionsTest
(
TestCase
):
def
setUp
(
self
):
self
.
app
=
Flask
(
'test_extensions'
)
self
.
dir
=
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
)
)
self
.
dir
=
os
.
path
.
dirname
(
__file__
)
self
.
senpy
=
Senpy
(
plugin_folder
=
self
.
dir
,
app
=
self
.
app
,
default_plugins
=
False
)
...
...
@@ -38,8 +39,8 @@ class ExtensionsTest(TestCase):
print
(
self
.
senpy
.
plugins
)
assert
"Dummy"
in
self
.
senpy
.
plugins
def
test_
enab
ling
(
self
):
"""
Enab
ling a plugin """
def
test_
instal
ling
(
self
):
"""
Instal
ling a plugin """
info
=
{
'name'
:
'TestPip'
,
'module'
:
'dummy'
,
...
...
@@ -48,14 +49,13 @@ class ExtensionsTest(TestCase):
'version'
:
0
}
root
=
os
.
path
.
join
(
self
.
dir
,
'plugins'
,
'dummy_plugin'
)
name
,
module
=
self
.
senpy
.
_
load_plugin_from_info
(
info
,
root
=
root
)
name
,
module
=
plugins
.
load_plugin_from_info
(
info
,
root
=
root
)
assert
name
==
'TestPip'
assert
module
import
noop
dir
(
noop
)
self
.
senpy
.
install_deps
()
def
test_
instal
ling
(
self
):
def
test_
enab
ling
(
self
):
""" Enabling a plugin """
self
.
senpy
.
activate_all
(
sync
=
True
)
assert
len
(
self
.
senpy
.
plugins
)
>=
3
...
...
@@ -72,7 +72,7 @@ class ExtensionsTest(TestCase):
}
root
=
os
.
path
.
join
(
self
.
dir
,
'plugins'
,
'dummy_plugin'
)
with
self
.
assertRaises
(
Error
):
name
,
module
=
self
.
senpy
.
_
load_plugin_from_info
(
info
,
root
=
root
)
name
,
module
=
plugins
.
load_plugin_from_info
(
info
,
root
=
root
)
def
test_disabling
(
self
):
""" Disabling a plugin """
...
...
@@ -173,7 +173,7 @@ class ExtensionsTest(TestCase):
'onyx:usesEmotionModel'
:
'emoml:fsre-dimensions'
})
eSet1
=
EmotionSet
()
eSet1
.
prov__wasGeneratedBy
=
plugin
[
'id'
]
eSet1
.
prov__wasGeneratedBy
=
plugin
[
'
@
id'
]
eSet1
[
'onyx:hasEmotion'
].
append
(
Emotion
({
'emoml:arousal'
:
1
,
'emoml:potency'
:
0
,
...
...
tests/test_plugins.py
View file @
4d7e8e75
...
...
@@ -7,11 +7,11 @@ import tempfile
from
unittest
import
TestCase
from
senpy.models
import
Results
,
Entry
,
EmotionSet
,
Emotion
from
senpy
.plugins
import
SentimentPlugin
,
ShelfMixin
from
senpy
import
plugins
from
senpy.plugins.conversion.emotion.centroids
import
CentroidConversion
class
ShelfDummyPlugin
(
SentimentPlugin
,
ShelfMixin
):
class
ShelfDummyPlugin
(
plugins
.
SentimentPlugin
,
plugins
.
ShelfMixin
):
def
activate
(
self
,
*
args
,
**
kwargs
):
if
'counter'
not
in
self
.
sh
:
self
.
sh
[
'counter'
]
=
0
...
...
@@ -202,3 +202,22 @@ class PluginsTest(TestCase):
e
[
"Y-dimension"
]
=
0.3
res
=
c
.
_backwards_conversion
(
e
)
assert
res
[
"onyx:hasEmotionCategory"
]
==
"c2"
def
make_mini_test
(
plugin
):
def
mini_test
(
self
):
plugin
.
test
()
return
mini_test
def
add_tests
():
root
=
os
.
path
.
dirname
(
__file__
)
plugs
=
plugins
.
load_plugins
(
os
.
path
.
join
(
root
,
".."
))
for
k
,
v
in
plugs
.
items
():
t_method
=
make_mini_test
(
v
)
t_method
.
__name__
=
'test_plugin_{}'
.
format
(
k
)
setattr
(
PluginsTest
,
t_method
.
__name__
,
t_method
)
del
t_method
add_tests
()
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment