Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
senpy
senpy
Commits
00ffbb38
Commit
00ffbb38
authored
Jul 04, 2018
by
J. Fernando Sánchez
Browse files
Several changes
* Add flag to run tests * Add ntriples outformat
parent
13cf0c71
Changes
11
Hide whitespace changes
Inline
Side-by-side
senpy/__main__.py
View file @
00ffbb38
...
...
@@ -78,10 +78,15 @@ def main():
help
=
'Do not run a server, only install plugin dependencies'
)
parser
.
add_argument
(
'--only-test'
,
'-t'
,
action
=
'store_true'
,
default
=
False
,
help
=
'Do not run a server, just test all plugins'
)
parser
.
add_argument
(
'--test'
,
'-t'
,
action
=
'store_true'
,
default
=
False
,
help
=
'Test all plugins before launching the server'
)
parser
.
add_argument
(
'--only-list'
,
'--list'
,
...
...
@@ -99,6 +104,12 @@ def main():
action
=
'store_false'
,
default
=
True
,
help
=
'Run a threaded server'
)
parser
.
add_argument
(
'--no-deps'
,
'-n'
,
action
=
'store_true'
,
default
=
False
,
help
=
'Skip installing dependencies'
)
parser
.
add_argument
(
'--version'
,
'-v'
,
...
...
@@ -125,19 +136,27 @@ def main():
data_folder
=
args
.
data_folder
)
if
args
.
only_list
:
plugins
=
sp
.
plugins
()
maxwidth
=
max
(
len
(
x
.
id
)
for
x
in
plugins
)
maxname
=
max
(
len
(
x
.
name
)
for
x
in
plugins
)
maxversion
=
max
(
len
(
x
.
version
)
for
x
in
plugins
)
print
(
'Found {} plugins:'
.
format
(
len
(
plugins
)))
for
plugin
in
plugins
:
import
inspect
fpath
=
inspect
.
getfile
(
plugin
.
__class__
)
print
(
'{: <{width}} @ {}'
.
format
(
plugin
.
id
,
fpath
,
width
=
maxwidth
))
print
(
'
\t
{: <{maxname}} @ {: <{maxversion}} -> {}'
.
format
(
plugin
.
name
,
plugin
.
version
,
fpath
,
maxname
=
maxname
,
maxversion
=
maxversion
))
return
sp
.
install_deps
()
if
not
args
.
no_deps
:
sp
.
install_deps
()
if
args
.
only_install
:
return
sp
.
activate_all
(
allow_fail
=
args
.
allow_fail
)
if
args
.
only_test
:
if
args
.
test
or
args
.
only_test
:
easy_test
(
sp
.
plugins
(),
debug
=
args
.
debug
)
return
if
args
.
only_test
:
return
print
(
'Senpy version {}'
.
format
(
senpy
.
__version__
))
print
(
'Server running on port %s:%d. Ctrl+C to quit'
%
(
args
.
host
,
args
.
port
))
...
...
senpy/api.py
View file @
00ffbb38
...
...
@@ -4,7 +4,7 @@ import logging
logger
=
logging
.
getLogger
(
__name__
)
boolean
=
(
True
,
False
)
boolean
=
[
True
,
False
]
API_PARAMS
=
{
...
...
@@ -33,7 +33,7 @@ API_PARAMS = {
"aliases"
:
[
"o"
],
"default"
:
"json-ld"
,
"required"
:
True
,
"options"
:
[
"json-ld"
,
"turtle"
],
"options"
:
[
"json-ld"
,
"turtle"
,
"ntriples"
],
},
"help"
:
{
"@id"
:
"help"
,
...
...
@@ -175,8 +175,8 @@ def parse_params(indict, *specs):
parameters
=
outdict
,
errors
=
wrong_params
)
raise
message
if
'algorithm'
in
outdict
and
not
isinstance
(
outdict
[
'algorithm'
],
tuple
):
outdict
[
'algorithm'
]
=
tuple
(
outdict
[
'algorithm'
].
split
(
','
))
if
'algorithm'
in
outdict
and
not
isinstance
(
outdict
[
'algorithm'
],
list
):
outdict
[
'algorithm'
]
=
list
(
outdict
[
'algorithm'
].
split
(
','
))
return
outdict
...
...
@@ -194,7 +194,8 @@ def parse_call(params):
params
=
parse_params
(
params
,
NIF_PARAMS
)
if
params
[
'informat'
]
==
'text'
:
results
=
Results
()
entry
=
Entry
(
nif__isString
=
params
[
'input'
])
entry
=
Entry
(
nif__isString
=
params
[
'input'
],
id
=
'#'
)
# Use @base
results
.
entries
.
append
(
entry
)
elif
params
[
'informat'
]
==
'json-ld'
:
results
=
from_string
(
params
[
'input'
],
cls
=
Results
)
...
...
senpy/blueprints.py
View file @
00ffbb38
...
...
@@ -25,7 +25,6 @@ from .version import __version__
from
functools
import
wraps
import
logging
import
traceback
import
json
import
base64
...
...
@@ -37,12 +36,17 @@ ns_blueprint = Blueprint("ns", __name__)
_mimetypes_r
=
{
'json-ld'
:
[
'application/ld+json'
],
'turtle'
:
[
'text/turtle'
],
'ntriples'
:
[
'application/n-triples'
],
'text'
:
[
'text/plain'
]}
MIMETYPES
=
{}
for
k
,
vs
in
_mimetypes_r
.
items
():
for
v
in
vs
:
if
v
in
MIMETYPES
:
raise
Exception
(
'MIMETYPE {} specified for two formats: {} and {}'
.
format
(
v
,
v
,
MIMETYPES
[
v
]))
MIMETYPES
[
v
]
=
k
DEFAULT_MIMETYPE
=
'application/ld+json'
...
...
@@ -147,16 +151,14 @@ def basic_api(f):
request
.
parameters
=
params
response
=
f
(
*
args
,
**
kwargs
)
except
(
Exception
)
as
ex
:
if
current_app
.
debug
:
if
current_app
.
debug
or
current_app
.
config
[
'TESTING'
]
:
raise
if
not
isinstance
(
ex
,
Error
):
msg
=
"{}:
\n\t
{}"
.
format
(
ex
,
traceback
.
format_exc
())
msg
=
"{}"
.
format
(
ex
)
ex
=
Error
(
message
=
msg
,
status
=
500
)
logger
.
exception
(
'Error returning analysis result'
)
response
=
ex
response
.
parameters
=
raw_params
logger
.
e
rror
(
ex
)
logger
.
e
xception
(
ex
)
if
'parameters'
in
response
and
not
params
[
'with_parameters'
]:
del
response
.
parameters
...
...
senpy/extensions.py
View file @
00ffbb38
...
...
@@ -18,14 +18,9 @@ import errno
import
logging
logger
=
logging
.
getLogger
(
__name__
)
from
.
import
gsitk_compat
try
:
from
gsitk.datasets.datasets
import
DatasetManager
GSITK_AVAILABLE
=
True
except
ImportError
:
logger
.
warn
(
'GSITK is not installed. Some functions will be unavailable.'
)
GSITK_AVAILABLE
=
False
logger
=
logging
.
getLogger
(
__name__
)
class
Senpy
(
object
):
...
...
@@ -167,8 +162,7 @@ class Senpy(object):
yield
i
def
install_deps
(
self
):
for
plugin
in
self
.
plugins
(
is_activated
=
True
):
plugins
.
install_deps
(
plugin
)
plugins
.
install_deps
(
*
self
.
plugins
())
def
analyse
(
self
,
request
):
"""
...
...
@@ -203,16 +197,14 @@ class Senpy(object):
raise
Error
(
status
=
404
,
message
=
"The dataset '{}' is not valid"
.
format
(
dataset
))
dm
=
DatasetManager
()
dm
=
gsitk_compat
.
DatasetManager
()
datasets
=
dm
.
prepare_datasets
(
datasets_name
)
return
datasets
@
property
def
datasets
(
self
):
if
not
GSITK_AVAILABLE
:
raise
Exception
(
'GSITK is not available. Install it to use this function.'
)
self
.
_dataset_list
=
{}
dm
=
DatasetManager
()
dm
=
gsitk_compat
.
DatasetManager
()
for
item
in
dm
.
get_datasets
():
for
key
in
item
:
if
key
in
self
.
_dataset_list
:
...
...
@@ -223,8 +215,6 @@ class Senpy(object):
return
self
.
_dataset_list
def
evaluate
(
self
,
params
):
if
not
GSITK_AVAILABLE
:
raise
Exception
(
'GSITK is not available. Install it to use this function.'
)
logger
.
debug
(
"evaluating request: {}"
.
format
(
params
))
results
=
AggregatedEvaluation
()
results
.
parameters
=
params
...
...
@@ -351,6 +341,7 @@ class Senpy(object):
logger
.
info
(
msg
)
success
=
True
self
.
_set_active
(
plugin
,
success
)
return
success
def
activate_plugin
(
self
,
plugin_name
,
sync
=
True
):
plugin_name
=
plugin_name
.
lower
()
...
...
@@ -362,7 +353,7 @@ class Senpy(object):
logger
.
info
(
"Activating plugin: {}"
.
format
(
plugin
.
name
))
if
sync
or
'async'
in
plugin
and
not
plugin
.
async
:
self
.
_activate
(
plugin
)
return
self
.
_activate
(
plugin
)
else
:
th
=
Thread
(
target
=
partial
(
self
.
_activate
,
plugin
))
th
.
start
()
...
...
senpy/gsitk_compat.py
0 → 100644
View file @
00ffbb38
import
logging
logger
=
logging
.
getLogger
(
__name__
)
MSG
=
'GSITK is not (properly) installed.'
IMPORTMSG
=
'{} Some functions will be unavailable.'
.
format
(
MSG
)
RUNMSG
=
'{} Install it to use this function.'
.
format
(
MSG
)
def
raise_exception
(
*
args
,
**
kwargs
):
raise
Exception
(
RUNMSG
)
try
:
from
gsitk.datasets.datasets
import
DatasetManager
from
gsitk.evaluation.evaluation
import
Evaluation
as
Eval
from
sklearn.pipeline
import
Pipeline
GSITK_AVAILABLE
=
True
modules
=
locals
()
except
ImportError
:
logger
.
warn
(
IMPORTMSG
)
GSITK_AVAILABLE
=
False
DatasetManager
=
Eval
=
Pipeline
=
raise_exception
senpy/models.py
View file @
00ffbb38
...
...
@@ -179,17 +179,18 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
content
=
json
.
dumps
(
js
,
indent
=
2
,
sort_keys
=
True
)
if
format
==
'json-ld'
:
mimetype
=
"application/json"
elif
format
in
[
'turtle'
,
]:
elif
format
in
[
'turtle'
,
'ntriples'
]:
logger
.
debug
(
js
)
base
=
kwargs
.
get
(
'prefix'
)
g
=
Graph
().
parse
(
data
=
content
,
format
=
'json-ld'
,
base
=
base
,
context
=
self
.
_context
)
context
=
[
self
.
_context
,
{
'@base'
:
base
}])
logger
.
debug
(
'Parsing with prefix: {}'
.
format
(
kwargs
.
get
(
'prefix'
)))
content
=
g
.
serialize
(
format
=
'turtle'
,
content
=
g
.
serialize
(
format
=
format
,
base
=
base
).
decode
(
'utf-8'
)
mimetype
=
'text/{}'
.
format
(
format
)
else
:
...
...
@@ -207,26 +208,20 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
result
=
self
.
serializable
()
ctx
=
context_uri
or
self
.
_context
result
[
'@context'
]
=
ctx
# result = jsonld.compact(result,
# ctx,
# options={
# 'base': prefix,
# 'expandContext': self._context,
# 'senpy': prefix
# })
if
expanded
:
result
=
jsonld
.
expand
(
result
,
options
=
{
'base'
:
prefix
,
'expandContext'
:
ctx
})
'expandContext'
:
self
.
_context
})[
0
]
if
not
with_context
:
try
:
del
result
[
'@context'
]
except
KeyError
:
pass
elif
context_uri
:
result
[
'@context'
]
=
context_uri
else
:
result
[
'@context'
]
=
self
.
_context
return
result
def
validate
(
self
,
obj
=
None
):
...
...
senpy/plugins/__init__.py
View file @
00ffbb38
...
...
@@ -23,18 +23,11 @@ import nltk
from
..
import
models
,
utils
from
..
import
api
from
..
import
gsitk_compat
logger
=
logging
.
getLogger
(
__name__
)
try
:
from
gsitk.evaluation.evaluation
import
Evaluation
as
Eval
from
sklearn.pipeline
import
Pipeline
GSITK_AVAILABLE
=
True
except
ImportError
:
logger
.
warn
(
'GSITK is not installed. Some functions will be unavailable.'
)
GSITK_AVAILABLE
=
False
class
PluginMeta
(
models
.
BaseMeta
):
_classes
=
{}
...
...
@@ -333,7 +326,7 @@ class Box(AnalysisPlugin):
return
self
.
transform
(
X
)
def
as_pipe
(
self
):
pipe
=
Pipeline
([(
'plugin'
,
self
)])
pipe
=
gsitk_compat
.
Pipeline
([(
'plugin'
,
self
)])
pipe
.
name
=
self
.
name
return
pipe
...
...
@@ -626,12 +619,9 @@ def _from_loaded_module(module, info=None, **kwargs):
def
evaluate
(
plugins
,
datasets
,
**
kwargs
):
if
not
GSITK_AVAILABLE
:
raise
Exception
(
'GSITK is not available. Install it to use this function.'
)
ev
=
Eval
(
tuples
=
None
,
datasets
=
datasets
,
pipelines
=
[
plugin
.
as_pipe
()
for
plugin
in
plugins
])
ev
=
gsitk_compat
.
Eval
(
tuples
=
None
,
datasets
=
datasets
,
pipelines
=
[
plugin
.
as_pipe
()
for
plugin
in
plugins
])
ev
.
evaluate
()
results
=
ev
.
results
evaluations
=
evaluations_to_JSONLD
(
results
,
**
kwargs
)
...
...
senpy/static/js/main.js
View file @
00ffbb38
...
...
@@ -413,7 +413,7 @@ function evaluate_JSON(){
url
+=
"
?algo=
"
+
plugin
+
"
&dataset=
"
+
datasets
$
(
'
#doevaluate
'
).
attr
(
"
disabled
"
,
true
);
$
.
ajax
({
type
:
"
GET
"
,
url
:
url
,
dataType
:
'
json
'
}).
done
(
function
(
resp
)
{
$
.
ajax
({
type
:
"
GET
"
,
url
:
url
,
dataType
:
'
json
'
}).
always
(
function
(
resp
)
{
$
(
'
#doevaluate
'
).
attr
(
"
disabled
"
,
false
);
response
=
resp
.
responseText
;
...
...
senpy/utils.py
View file @
00ffbb38
...
...
@@ -80,7 +80,7 @@ def easy_test(plugin_list=None, debug=True):
for
plug
in
plugin_list
:
plug
.
test
()
plug
.
log
.
info
(
'My tests passed!'
)
logger
.
info
(
'All tests passed
!'
)
logger
.
info
(
'All tests passed
for {} plugins!'
.
format
(
len
(
plugin_list
))
)
except
Exception
:
if
not
debug
:
raise
...
...
tests/test_blueprints.py
View file @
00ffbb38
...
...
@@ -21,7 +21,6 @@ class BlueprintsTest(TestCase):
def
setUpClass
(
cls
):
"""Set up only once, and re-use in every individual test"""
cls
.
app
=
Flask
(
"test_extensions"
)
cls
.
app
.
debug
=
False
cls
.
client
=
cls
.
app
.
test_client
()
cls
.
senpy
=
Senpy
(
default_plugins
=
True
)
cls
.
senpy
.
init_app
(
cls
.
app
)
...
...
@@ -31,6 +30,9 @@ class BlueprintsTest(TestCase):
cls
.
senpy
.
activate_plugin
(
"DummyRequired"
,
sync
=
True
)
cls
.
senpy
.
default_plugin
=
'Dummy'
def
setUp
(
self
):
self
.
app
.
config
[
'TESTING'
]
=
True
# Tell Flask not to catch Exceptions
def
assertCode
(
self
,
resp
,
code
):
self
.
assertEqual
(
resp
.
status_code
,
code
)
...
...
@@ -42,6 +44,7 @@ class BlueprintsTest(TestCase):
"""
Calling with no arguments should ask the user for more arguments
"""
self
.
app
.
config
[
'TESTING'
]
=
False
# Errors are expected in this case
resp
=
self
.
client
.
get
(
"/api/"
)
self
.
assertCode
(
resp
,
400
)
js
=
parse_resp
(
resp
)
...
...
@@ -81,7 +84,7 @@ class BlueprintsTest(TestCase):
Extra params that have a required argument that does not
have a default should raise an error.
"""
self
.
app
.
debug
=
Fal
se
self
.
app
.
config
[
'TESTING'
]
=
False
# Errors are expected in this ca
se
resp
=
self
.
client
.
get
(
"/api/?i=My aloha mohame&algo=DummyRequired"
)
self
.
assertCode
(
resp
,
400
)
js
=
parse_resp
(
resp
)
...
...
@@ -97,7 +100,7 @@ class BlueprintsTest(TestCase):
The dummy plugin returns an empty response,
\
it should contain the context
"""
self
.
app
.
debug
=
Fal
se
self
.
app
.
config
[
'TESTING'
]
=
False
# Errors are expected in this ca
se
resp
=
self
.
client
.
get
(
"/api/?i=My aloha mohame&algo=DOESNOTEXIST"
)
self
.
assertCode
(
resp
,
404
)
js
=
parse_resp
(
resp
)
...
...
@@ -172,5 +175,6 @@ class BlueprintsTest(TestCase):
assert
"help"
in
js
[
"valid_parameters"
]
def
test_conversion
(
self
):
self
.
app
.
config
[
'TESTING'
]
=
False
# Errors are expected in this case
resp
=
self
.
client
.
get
(
"/api/?input=hello&algo=emoRand&emotionModel=DOES NOT EXIST"
)
self
.
assertCode
(
resp
,
404
)
tests/test_plugins.py
View file @
00ffbb38
...
...
@@ -6,7 +6,7 @@ import pickle
import
shutil
import
tempfile
from
unittest
import
TestCase
,
skipIf
from
unittest
import
TestCase
from
senpy.models
import
Results
,
Entry
,
EmotionSet
,
Emotion
,
Plugins
from
senpy
import
plugins
from
senpy.plugins.conversion.emotion.centroids
import
CentroidConversion
...
...
@@ -312,9 +312,7 @@ class PluginsTest(TestCase):
res
=
c
.
_backwards_conversion
(
e
)
assert
res
[
"onyx:hasEmotionCategory"
]
==
"c2"
@
skipIf
(
sys
.
version_info
<
(
3
,
0
),
reason
=
"requires Python3"
)
def
test_evaluation
(
self
):
def
_test_evaluation
(
self
):
testdata
=
[]
for
i
in
range
(
50
):
testdata
.
append
([
"good"
,
1
])
...
...
@@ -348,6 +346,14 @@ class PluginsTest(TestCase):
smart_metrics
=
results
[
0
].
metrics
[
0
]
assert
abs
(
smart_metrics
[
'accuracy'
]
-
1
)
<
0.01
def
test_evaluation
(
self
):
if
sys
.
version_info
<
(
3
,
0
):
with
self
.
assertRaises
(
Exception
)
as
context
:
self
.
_test_evaluation
()
self
.
assertTrue
(
'GSITK '
in
str
(
context
.
exception
))
else
:
self
.
_test_evaluation
()
def
make_mini_test
(
fpath
):
def
mini_test
(
self
):
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment