Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
senpy
senpy
Commits
48d7d1d0
Commit
48d7d1d0
authored
Feb 20, 2016
by
J. Fernando Sánchez
Browse files
Improved plugins API and loading
Also: * added drone-ci integration: tests for py2.7 and py3
parent
14c9f618
Changes
8
Hide whitespace changes
Inline
Side-by-side
.drone.yml
0 → 100644
View file @
48d7d1d0
build
:
image
:
python:$$PYTHON_VERSION
commands
:
-
python setup.py test
matrix
:
PYTHON_VERSION
:
-
2.7
-
3.4
senpy/__main__.py
View file @
48d7d1d0
...
...
@@ -63,7 +63,9 @@ def main():
default
=
"plugins"
,
help
=
'Where to look for plugins.'
)
args
=
parser
.
parse_args
()
logging
.
basicConfig
(
level
=
getattr
(
logging
,
args
.
level
))
logging
.
basicConfig
()
rl
=
logging
.
getLogger
()
rl
.
setLevel
(
getattr
(
logging
,
args
.
level
))
app
=
Flask
(
__name__
)
app
.
debug
=
args
.
debug
sp
=
Senpy
(
app
,
args
.
plugins_folder
,
default_plugins
=
args
.
default_plugins
)
...
...
senpy/blueprints.py
View file @
48d7d1d0
...
...
@@ -18,7 +18,7 @@
Blueprints for Senpy
"""
from
flask
import
Blueprint
,
request
,
current_app
,
render_template
from
.models
import
Error
,
Response
from
.models
import
Error
,
Response
,
Plugins
from
future.utils
import
iteritems
import
json
...
...
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
nif_blueprint
=
Blueprint
(
"NIF Sentiment Analysis Server"
,
__name__
)
demo_blueprint
=
Blueprint
(
"Demo of the service. It includes an HTML+Javascript playground to test senpy"
,
__name__
)
BASIC
_PARAMS
=
{
API
_PARAMS
=
{
"algorithm"
:
{
"aliases"
:
[
"algorithm"
,
"a"
,
"algo"
],
"required"
:
False
,
...
...
@@ -41,6 +41,63 @@ BASIC_PARAMS = {
}
}
BASIC_PARAMS
=
{
"algorithm"
:
{
"aliases"
:
[
"algorithm"
,
"a"
,
"algo"
],
"required"
:
False
,
},
"inHeaders"
:
{
"aliases"
:
[
"inHeaders"
,
"headers"
],
"required"
:
True
,
"default"
:
"0"
},
"input"
:
{
"@id"
:
"input"
,
"aliases"
:
[
"i"
,
"input"
],
"required"
:
True
,
"help"
:
"Input text"
},
"informat"
:
{
"@id"
:
"informat"
,
"aliases"
:
[
"f"
,
"informat"
],
"required"
:
False
,
"default"
:
"text"
,
"options"
:
[
"turtle"
,
"text"
],
},
"intype"
:
{
"@id"
:
"intype"
,
"aliases"
:
[
"intype"
,
"t"
],
"required"
:
False
,
"default"
:
"direct"
,
"options"
:
[
"direct"
,
"url"
,
"file"
],
},
"outformat"
:
{
"@id"
:
"outformat"
,
"aliases"
:
[
"outformat"
,
"o"
],
"default"
:
"json-ld"
,
"required"
:
False
,
"options"
:
[
"json-ld"
],
},
"language"
:
{
"@id"
:
"language"
,
"aliases"
:
[
"language"
,
"l"
],
"required"
:
False
,
},
"prefix"
:
{
"@id"
:
"prefix"
,
"aliases"
:
[
"prefix"
,
"p"
],
"required"
:
True
,
"default"
:
""
,
},
"urischeme"
:
{
"@id"
:
"urischeme"
,
"aliases"
:
[
"urischeme"
,
"u"
],
"required"
:
False
,
"default"
:
"RFC5147String"
,
"options"
:
"RFC5147String"
},
}
def
get_params
(
req
,
params
=
BASIC_PARAMS
):
if
req
.
method
==
'POST'
:
indict
=
req
.
form
...
...
@@ -119,36 +176,29 @@ def api():
return
ex
.
message
.
flask
()
@
nif_blueprint
.
route
(
"/default"
)
def
default
():
# return current_app.senpy.default_plugin
plug
=
current_app
.
senpy
.
default_plugin
if
plug
:
return
plugins
(
action
=
"list"
,
plugin
=
plug
.
name
)
else
:
error
=
Error
(
status
=
404
,
message
=
"No plugins found"
)
return
error
.
flask
()
@
nif_blueprint
.
route
(
'/plugins/'
,
methods
=
[
'POST'
,
'GET'
])
def
plugins
():
in_headers
=
get_params
(
request
,
API_PARAMS
)[
"inHeaders"
]
!=
"0"
sp
=
current_app
.
senpy
dic
=
Plugins
(
plugins
=
list
(
sp
.
plugins
.
values
()))
return
dic
.
flask
(
in_headers
=
in_headers
)
@
nif_blueprint
.
route
(
'/plugins/<plugin>/'
,
methods
=
[
'POST'
,
'GET'
])
@
nif_blueprint
.
route
(
'/plugins/<plugin>/<action>'
,
methods
=
[
'POST'
,
'GET'
])
def
plugin
s
(
plugin
=
None
,
action
=
"list"
):
def
plugin
(
plugin
=
None
,
action
=
"list"
):
filt
=
{}
sp
=
current_app
.
senpy
if
plugin
:
filt
[
"name"
]
=
plugin
plugs
=
sp
.
filter_plugins
(
**
filt
)
if
plugin
and
not
plugs
:
return
"Plugin not found"
,
400
plugs
=
sp
.
filter_plugins
(
name
=
plugin
)
if
plugin
==
'default'
and
sp
.
default_plugin
:
response
=
sp
.
default_plugin
plugin
=
response
.
name
elif
plugin
in
sp
.
plugins
:
response
=
sp
.
plugins
[
plugin
]
else
:
return
Error
(
message
=
"Plugin not found"
,
status
=
404
).
flask
()
if
action
==
"list"
:
in_headers
=
get_params
(
request
,
BASIC_PARAMS
)[
"inHeaders"
]
!=
"0"
if
plugin
:
dic
=
plugs
[
plugin
]
else
:
dic
=
Response
(
{
plug
:
plugs
[
plug
].
serializable
()
for
plug
in
plugs
})
return
dic
.
flask
(
in_headers
=
in_headers
)
in_headers
=
get_params
(
request
,
API_PARAMS
)[
"inHeaders"
]
!=
"0"
return
response
.
flask
(
in_headers
=
in_headers
)
method
=
"{}_plugin"
.
format
(
action
)
if
(
hasattr
(
sp
,
method
)):
getattr
(
sp
,
method
)(
plugin
)
...
...
@@ -156,7 +206,6 @@ def plugins(plugin=None, action="list"):
else
:
return
Error
(
message
=
"action '{}' not allowed"
.
format
(
action
)).
flask
()
if
__name__
==
'__main__'
:
import
config
...
...
senpy/extensions.py
View file @
48d7d1d0
...
...
@@ -34,6 +34,7 @@ class Senpy(object):
self
.
app
=
app
self
.
_search_folders
=
set
()
self
.
_plugin_list
=
[]
self
.
_outdated
=
True
self
.
add_folder
(
plugin_folder
)
...
...
@@ -65,10 +66,8 @@ class Senpy(object):
if
os
.
path
.
isdir
(
folder
):
self
.
_search_folders
.
add
(
folder
)
self
.
_outdated
=
True
return
True
else
:
logger
.
debug
(
"Not a folder: %s"
,
folder
)
return
False
def
analyse
(
self
,
**
params
):
algo
=
None
...
...
@@ -113,7 +112,7 @@ class Senpy(object):
def
parameters
(
self
,
algo
):
return
getattr
(
self
.
plugins
.
get
(
algo
)
or
self
.
default_plugin
,
"params"
,
"
extra_
params"
,
{})
def
activate_all
(
self
,
sync
=
False
):
...
...
@@ -129,13 +128,18 @@ class Senpy(object):
return
ps
def
_set_active_plugin
(
self
,
plugin_name
,
active
=
True
,
*
args
,
**
kwargs
):
''' We're using a variable in the plugin itself to activate/deactive plugins.
\
Note that plugins may activate themselves by setting this variable.
'''
self
.
plugins
[
plugin_name
].
is_activated
=
active
def
activate_plugin
(
self
,
plugin_name
,
sync
=
False
):
plugin
=
self
.
plugins
[
plugin_name
]
logger
.
info
(
"Activating plugin: {}"
.
format
(
plugin
.
name
))
def
act
():
try
:
plugin
.
activate
()
logger
.
info
(
"Plugin activated: {}"
.
format
(
plugin
.
name
))
except
Exception
as
ex
:
logger
.
error
(
"Error activating plugin {}: {}"
.
format
(
plugin
.
name
,
ex
))
...
...
@@ -149,19 +153,33 @@ class Senpy(object):
def
deactivate_plugin
(
self
,
plugin_name
,
sync
=
False
):
plugin
=
self
.
plugins
[
plugin_name
]
th
=
gevent
.
spawn
(
plugin
.
deactivate
)
def
deact
():
try
:
plugin
.
deactivate
()
logger
.
info
(
"Plugin deactivated: {}"
.
format
(
plugin
.
name
))
except
Exception
as
ex
:
logger
.
error
(
"Error deactivating plugin {}: {}"
.
format
(
plugin
.
name
,
ex
))
logger
.
error
(
"Trace: {}"
.
format
(
traceback
.
format_exc
()))
th
=
gevent
.
spawn
(
deact
)
th
.
link_value
(
partial
(
self
.
_set_active_plugin
,
plugin_name
,
False
))
if
sync
:
th
.
join
()
else
:
return
th
def
reload_plugin
(
self
,
plugin
):
logger
.
debug
(
"Reloading {}"
.
format
(
plugin
))
plug
=
self
.
plugins
[
plugin
]
nplug
=
self
.
_load_plugin
(
plug
.
module
,
plug
.
path
)
del
self
.
plugins
[
plugin
]
self
.
plugins
[
nplug
.
name
]
=
nplug
def
reload_plugin
(
self
,
name
):
logger
.
debug
(
"Reloading {}"
.
format
(
name
))
plugin
=
self
.
plugins
[
name
]
try
:
del
self
.
plugins
[
name
]
nplug
=
self
.
_load_plugin
(
plugin
.
module
,
plugin
.
path
)
self
.
plugins
[
nplug
.
name
]
=
nplug
except
Exception
as
ex
:
logger
.
error
(
'Error reloading {}: {}'
.
format
(
name
,
ex
))
self
.
plugins
[
name
]
=
plugin
@
staticmethod
def
_load_plugin
(
root
,
filename
):
...
...
@@ -206,7 +224,7 @@ class Senpy(object):
for
root
,
dirnames
,
filenames
in
os
.
walk
(
search_folder
):
for
filename
in
fnmatch
.
filter
(
filenames
,
'*.senpy'
):
name
,
plugin
=
self
.
_load_plugin
(
root
,
filename
)
if
plugin
:
if
plugin
and
name
not
in
self
.
_plugin_list
:
plugins
[
name
]
=
plugin
self
.
_outdated
=
False
...
...
@@ -218,9 +236,9 @@ class Senpy(object):
@
property
def
plugins
(
self
):
""" Return the plugins registered for a given application. """
if
not
hasattr
(
self
,
'senpy_plugins'
)
or
self
.
_outdated
:
self
.
senpy
_plugin
s
=
self
.
_load_plugins
()
return
self
.
senpy
_plugin
s
if
self
.
_outdated
:
self
.
_plugin
_list
=
self
.
_load_plugins
()
return
self
.
_plugin
_list
def
filter_plugins
(
self
,
**
kwargs
):
""" Filter plugins by different criteria """
...
...
senpy/models.py
View file @
48d7d1d0
...
...
@@ -117,11 +117,16 @@ class SenpyMixin(object):
sort_keys
=
True
)
return
js
def
validate
(
self
,
obj
=
None
):
if
not
obj
:
obj
=
self
if
hasattr
(
obj
,
"jsonld"
):
obj
=
obj
.
jsonld
()
jsonschema
.
validate
(
obj
,
self
.
schema
)
class
SenpyModel
(
SenpyMixin
,
dict
):
schema
=
base_schema
prefix
=
None
def
__init__
(
self
,
*
args
,
**
kwargs
):
temp
=
dict
(
*
args
,
**
kwargs
)
...
...
@@ -161,14 +166,6 @@ class SenpyModel(SenpyMixin, dict):
def
__delattr__
(
self
,
key
):
self
.
__delitem__
(
self
.
_get_key
(
key
))
def
validate
(
self
,
obj
=
None
):
if
not
obj
:
obj
=
self
if
hasattr
(
obj
,
"jsonld"
):
obj
=
obj
.
jsonld
()
jsonschema
.
validate
(
obj
,
self
.
schema
)
@
classmethod
...
...
senpy/plugins.py
View file @
48d7d1d0
...
...
@@ -9,55 +9,6 @@ from .models import Response, PluginModel, Error
logger
=
logging
.
getLogger
(
__name__
)
PARAMS
=
{
"input"
:
{
"@id"
:
"input"
,
"aliases"
:
[
"i"
,
"input"
],
"required"
:
True
,
"help"
:
"Input text"
},
"informat"
:
{
"@id"
:
"informat"
,
"aliases"
:
[
"f"
,
"informat"
],
"required"
:
False
,
"default"
:
"text"
,
"options"
:
[
"turtle"
,
"text"
],
},
"intype"
:
{
"@id"
:
"intype"
,
"aliases"
:
[
"intype"
,
"t"
],
"required"
:
False
,
"default"
:
"direct"
,
"options"
:
[
"direct"
,
"url"
,
"file"
],
},
"outformat"
:
{
"@id"
:
"outformat"
,
"aliases"
:
[
"outformat"
,
"o"
],
"default"
:
"json-ld"
,
"required"
:
False
,
"options"
:
[
"json-ld"
],
},
"language"
:
{
"@id"
:
"language"
,
"aliases"
:
[
"language"
,
"l"
],
"required"
:
False
,
},
"prefix"
:
{
"@id"
:
"prefix"
,
"aliases"
:
[
"prefix"
,
"p"
],
"required"
:
True
,
"default"
:
""
,
},
"urischeme"
:
{
"@id"
:
"urischeme"
,
"aliases"
:
[
"urischeme"
,
"u"
],
"required"
:
False
,
"default"
:
"RFC5147String"
,
"options"
:
"RFC5147String"
},
}
class
SenpyPlugin
(
PluginModel
):
def
__init__
(
self
,
info
=
None
):
...
...
@@ -65,14 +16,12 @@ class SenpyPlugin(PluginModel):
raise
Error
(
message
=
(
"You need to provide configuration"
"information for the plugin."
))
logger
.
debug
(
"Initialising {}"
.
format
(
info
))
s
elf
.
name
=
info
[
"name"
]
self
.
version
=
info
[
"version"
]
self
.
params
=
info
.
get
(
"params"
,
PARAMS
.
copy
())
s
uper
(
SenpyPlugin
,
self
).
__init__
(
info
)
self
.
params
=
info
.
get
(
"extra_params"
,
{})
self
.
_info
=
info
if
"@id"
not
in
self
.
params
:
self
.
params
[
"@id"
]
=
"params_%s"
%
self
.
id
self
.
is_activated
=
False
self
.
_info
=
info
super
(
SenpyPlugin
,
self
).
__init__
()
def
get_folder
(
self
):
return
os
.
path
.
dirname
(
inspect
.
getfile
(
self
.
__class__
))
...
...
setup.py
View file @
48d7d1d0
...
...
@@ -15,7 +15,7 @@ except AttributeError:
install_reqs
=
[
str
(
ir
.
req
)
for
ir
in
install_reqs
]
test_reqs
=
[
str
(
ir
.
req
)
for
ir
in
test_reqs
]
VERSION
=
"0.5"
VERSION
=
"0.5
.1
"
setup
(
name
=
'senpy'
,
...
...
tests/test_blueprints.py
View file @
48d7d1d0
...
...
@@ -56,7 +56,10 @@ class BlueprintsTest(TestCase):
resp
=
self
.
client
.
get
(
"/api/plugins/"
)
self
.
assert200
(
resp
)
logging
.
debug
(
resp
.
json
)
assert
"Dummy"
in
resp
.
json
assert
'plugins'
in
resp
.
json
plugins
=
resp
.
json
[
'plugins'
]
assert
len
(
plugins
)
>
1
assert
list
(
p
for
p
in
plugins
if
p
[
'name'
]
==
"Dummy"
)
assert
"@context"
in
resp
.
json
def
test_headers
(
self
):
...
...
@@ -98,7 +101,7 @@ class BlueprintsTest(TestCase):
def
test_default
(
self
):
""" Show only one plugin"""
resp
=
self
.
client
.
get
(
"/api/default"
)
resp
=
self
.
client
.
get
(
"/api/
plugins/
default
/
"
)
self
.
assert200
(
resp
)
logging
.
debug
(
resp
.
json
)
assert
"@id"
in
resp
.
json
...
...
@@ -106,5 +109,5 @@ class BlueprintsTest(TestCase):
resp
=
self
.
client
.
get
(
"/api/plugins/Dummy/deactivate"
)
self
.
assert200
(
resp
)
sleep
(
0.5
)
resp
=
self
.
client
.
get
(
"/api/default"
)
resp
=
self
.
client
.
get
(
"/api/
plugins/
default
/
"
)
self
.
assert404
(
resp
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment