Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
senpy
senpy
Commits
1a9dd07f
Commit
1a9dd07f
authored
May 05, 2017
by
Ian Wood
Browse files
Merge branch 'master' 0.8.7 into patch-6
parents
b80b0c79
13cefbed
Changes
16
Hide whitespace changes
Inline
Side-by-side
Makefile
View file @
1a9dd07f
...
...
@@ -73,8 +73,8 @@ pip_test: $(addprefix pip_test-,$(PYVERSIONS))
clean
:
@
docker ps
-a
|
awk
'/
$(REPO)
\/
$(NAME)
/{ split($$2, vers, "-"); if(vers[0] != "
${VERSION}
"){ print $$1;}}'
| xargs docker
rm
-v
2>/dev/null||
true
@
docker images |
awk
'/
$(REPO)
\/
$(NAME)
/{ split($$2, vers, "-"); if(vers[0] != "
${VERSION}
"){ print $$1":"$$2;}}'
| xargs docker rmi 2>/dev/null||
true
@
docker
rmi
$(NAME)
-dev
2>/dev/null
||
true
@
docker
stop
$(
addprefix
$(NAME)
-dev
,
$(PYVERSIONS)
)
2>/dev/null
||
true
@
docker
rm
$(
addprefix
$(NAME)
-dev
,
$(PYVERSIONS)
)
2>/dev/null
||
true
git_commit
:
git commit
-a
...
...
docs/bad-examples/results/example-analysis-as-id-FAIL.json
0 → 100644
View file @
1a9dd07f
{
"@context"
:
"http://mixedemotions-project.eu/ns/context.jsonld"
,
"@id"
:
"me:Result1"
,
"@type"
:
"results"
,
"analysis"
:
[
"me:SAnalysis1"
,
"me:SgAnalysis1"
,
"me:EmotionAnalysis1"
,
"me:NER1"
,
{
"@type"
:
"analysis"
,
"@id"
:
"wrong"
}
],
"entries"
:
[
{
"@id"
:
"http://micro.blog/status1"
,
"@type"
:
[
"nif:RFC5147String"
,
"nif:Context"
],
"nif:isString"
:
"Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource"
,
"entities"
:
[
{
"@id"
:
"http://micro.blog/status1#char=5,13"
,
"nif:beginIndex"
:
5
,
"nif:endIndex"
:
13
,
"nif:anchorOf"
:
"Microsoft"
,
"me:references"
:
"http://dbpedia.org/page/Microsoft"
,
"prov:wasGeneratedBy"
:
"me:NER1"
},
{
"@id"
:
"http://micro.blog/status1#char=25,37"
,
"nif:beginIndex"
:
25
,
"nif:endIndex"
:
37
,
"nif:anchorOf"
:
"Windows Phone"
,
"me:references"
:
"http://dbpedia.org/page/Windows_Phone"
,
"prov:wasGeneratedBy"
:
"me:NER1"
}
],
"suggestions"
:
[
{
"@id"
:
"http://micro.blog/status1#char=16,77"
,
"nif:beginIndex"
:
16
,
"nif:endIndex"
:
77
,
"nif:anchorOf"
:
"put your Windows Phone on your newest #open technology program"
,
"prov:wasGeneratedBy"
:
"me:SgAnalysis1"
}
],
"sentiments"
:
[
{
"@id"
:
"http://micro.blog/status1#char=80,97"
,
"nif:beginIndex"
:
80
,
"nif:endIndex"
:
97
,
"nif:anchorOf"
:
"You'll be awesome."
,
"marl:hasPolarity"
:
"marl:Positive"
,
"marl:polarityValue"
:
0.9
,
"prov:wasGeneratedBy"
:
"me:SAnalysis1"
}
],
"emotions"
:
[
{
"@id"
:
"http://micro.blog/status1#char=0,109"
,
"nif:anchorOf"
:
"Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource"
,
"prov:wasGeneratedBy"
:
"me:EAnalysis1"
,
"onyx:hasEmotion"
:
[
{
"onyx:hasEmotionCategory"
:
"wna:liking"
},
{
"onyx:hasEmotionCategory"
:
"wna:excitement"
}
]
}
]
}
]
}
docs/examples/results/example-analysis-as-id.json
0 → 100644
View file @
1a9dd07f
{
"@context"
:
"http://mixedemotions-project.eu/ns/context.jsonld"
,
"@id"
:
"me:Result1"
,
"@type"
:
"results"
,
"analysis"
:
[
"me:SAnalysis1"
,
"me:SgAnalysis1"
,
"me:EmotionAnalysis1"
,
"me:NER1"
],
"entries"
:
[
{
"@id"
:
"http://micro.blog/status1"
,
"@type"
:
[
"nif:RFC5147String"
,
"nif:Context"
],
"nif:isString"
:
"Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource"
,
"entities"
:
[
{
"@id"
:
"http://micro.blog/status1#char=5,13"
,
"nif:beginIndex"
:
5
,
"nif:endIndex"
:
13
,
"nif:anchorOf"
:
"Microsoft"
,
"me:references"
:
"http://dbpedia.org/page/Microsoft"
,
"prov:wasGeneratedBy"
:
"me:NER1"
},
{
"@id"
:
"http://micro.blog/status1#char=25,37"
,
"nif:beginIndex"
:
25
,
"nif:endIndex"
:
37
,
"nif:anchorOf"
:
"Windows Phone"
,
"me:references"
:
"http://dbpedia.org/page/Windows_Phone"
,
"prov:wasGeneratedBy"
:
"me:NER1"
}
],
"suggestions"
:
[
{
"@id"
:
"http://micro.blog/status1#char=16,77"
,
"nif:beginIndex"
:
16
,
"nif:endIndex"
:
77
,
"nif:anchorOf"
:
"put your Windows Phone on your newest #open technology program"
,
"prov:wasGeneratedBy"
:
"me:SgAnalysis1"
}
],
"sentiments"
:
[
{
"@id"
:
"http://micro.blog/status1#char=80,97"
,
"nif:beginIndex"
:
80
,
"nif:endIndex"
:
97
,
"nif:anchorOf"
:
"You'll be awesome."
,
"marl:hasPolarity"
:
"marl:Positive"
,
"marl:polarityValue"
:
0.9
,
"prov:wasGeneratedBy"
:
"me:SAnalysis1"
}
],
"emotions"
:
[
{
"@id"
:
"http://micro.blog/status1#char=0,109"
,
"nif:anchorOf"
:
"Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource"
,
"prov:wasGeneratedBy"
:
"me:EAnalysis1"
,
"onyx:hasEmotion"
:
[
{
"onyx:hasEmotionCategory"
:
"wna:liking"
},
{
"onyx:hasEmotionCategory"
:
"wna:excitement"
}
]
}
]
}
]
}
requirements.txt
View file @
1a9dd07f
Flask
>=0.10.1
requests
>=2.4.1
gevent
>=1.1rc4
tornado
>=4.4.3
PyLD
>=0.6.5
six
future
...
...
senpy/__main__.py
View file @
1a9dd07f
...
...
@@ -22,15 +22,16 @@ the server.
from
flask
import
Flask
from
senpy.extensions
import
Senpy
from
gevent.wsgi
import
WSGIServer
from
gevent.monkey
import
patch_all
from
tornado.wsgi
import
WSGIContainer
from
tornado.httpserver
import
HTTPServer
from
tornado.ioloop
import
IOLoop
import
logging
import
os
import
argparse
import
senpy
patch_all
(
thread
=
False
)
SERVER_PORT
=
os
.
environ
.
get
(
"PORT"
,
5000
)
...
...
@@ -92,9 +93,10 @@ def main():
print
(
'Server running on port %s:%d. Ctrl+C to quit'
%
(
args
.
host
,
args
.
port
))
if
not
app
.
debug
:
http_server
=
WSGIServer
((
args
.
host
,
args
.
port
),
app
)
http_server
=
HTTPServer
(
WSGIContainer
(
app
))
http_server
.
listen
(
args
.
port
,
address
=
args
.
host
)
try
:
http_server
.
serve_forever
()
IOLoop
.
instance
().
start
()
except
KeyboardInterrupt
:
print
(
'Bye!'
)
http_server
.
stop
()
...
...
senpy/client.py
View file @
1a9dd07f
import
requests
import
logging
from
.
import
models
from
.plugins
import
default_plugin_type
logger
=
logging
.
getLogger
(
__name__
)
...
...
@@ -12,6 +13,10 @@ class Client(object):
def
analyse
(
self
,
input
,
method
=
'GET'
,
**
kwargs
):
return
self
.
request
(
'/'
,
method
=
method
,
input
=
input
,
**
kwargs
)
def
plugins
(
self
,
ptype
=
default_plugin_type
):
resp
=
self
.
request
(
path
=
'/plugins'
,
plugin_type
=
ptype
).
plugins
return
{
p
.
name
:
p
for
p
in
resp
}
def
request
(
self
,
path
=
None
,
method
=
'GET'
,
**
params
):
url
=
'{}{}'
.
format
(
self
.
endpoint
,
path
)
response
=
requests
.
request
(
method
=
method
,
url
=
url
,
params
=
params
)
...
...
senpy/extensions.py
View file @
1a9dd07f
...
...
@@ -183,7 +183,7 @@ class Senpy(object):
return
resp
def
_conversion_candidates
(
self
,
fromModel
,
toModel
):
candidates
=
self
.
filter_plugins
(
**
{
'@
type
'
:
'emotionConversionPlugin'
}
)
candidates
=
self
.
filter_plugins
(
plugin_
type
=
'emotionConversionPlugin'
)
for
name
,
candidate
in
candidates
.
items
():
for
pair
in
candidate
.
onyx__doesConversion
:
logging
.
debug
(
pair
)
...
...
@@ -303,6 +303,7 @@ class Senpy(object):
else
:
th
=
Thread
(
target
=
act
)
th
.
start
()
return
th
def
deactivate_plugin
(
self
,
plugin_name
,
sync
=
False
):
try
:
...
...
@@ -327,6 +328,7 @@ class Senpy(object):
else
:
th
=
Thread
(
target
=
deact
)
th
.
start
()
return
th
@
classmethod
def
validate_info
(
cls
,
info
):
...
...
@@ -417,33 +419,7 @@ class Senpy(object):
return
self
.
_plugin_list
def
filter_plugins
(
self
,
**
kwargs
):
""" Filter plugins by different criteria """
ptype
=
kwargs
.
pop
(
'plugin_type'
,
None
)
logger
.
debug
(
'#'
*
100
)
logger
.
debug
(
'ptype {}'
.
format
(
ptype
))
if
ptype
:
try
:
ptype
=
ptype
[
0
].
upper
()
+
ptype
[
1
:]
pclass
=
getattr
(
plugins
,
ptype
)
logger
.
debug
(
'Class: {}'
.
format
(
pclass
))
candidates
=
filter
(
lambda
x
:
isinstance
(
x
,
pclass
),
self
.
plugins
.
values
())
except
AttributeError
:
raise
Error
(
'{} is not a valid type'
.
format
(
ptype
))
else
:
candidates
=
self
.
plugins
.
values
()
logger
.
debug
(
candidates
)
def
matches
(
plug
):
res
=
all
(
getattr
(
plug
,
k
,
None
)
==
v
for
(
k
,
v
)
in
kwargs
.
items
())
logger
.
debug
(
"matching {} with {}: {}"
.
format
(
plug
.
name
,
kwargs
,
res
))
return
res
if
kwargs
:
candidates
=
filter
(
matches
,
candidates
)
return
{
p
.
name
:
p
for
p
in
candidates
}
return
plugins
.
pfilter
(
self
.
plugins
,
**
kwargs
)
@
property
def
analysis_plugins
(
self
):
...
...
senpy/plugins/__init__.py
View file @
1a9dd07f
...
...
@@ -9,6 +9,7 @@ import logging
import
tempfile
import
copy
from
..
import
models
from
..api
import
API_PARAMS
logger
=
logging
.
getLogger
(
__name__
)
...
...
@@ -117,3 +118,40 @@ class ShelfMixin(object):
if
hasattr
(
self
,
'_sh'
)
and
self
.
_sh
is
not
None
:
with
open
(
self
.
shelf_file
,
'wb'
)
as
f
:
pickle
.
dump
(
self
.
_sh
,
f
)
default_plugin_type
=
API_PARAMS
[
'plugin_type'
][
'default'
]
def
pfilter
(
plugins
,
**
kwargs
):
""" Filter plugins by different criteria """
if
isinstance
(
plugins
,
models
.
Plugins
):
plugins
=
plugins
.
plugins
elif
isinstance
(
plugins
,
dict
):
plugins
=
plugins
.
values
()
ptype
=
kwargs
.
pop
(
'plugin_type'
,
default_plugin_type
)
logger
.
debug
(
'#'
*
100
)
logger
.
debug
(
'ptype {}'
.
format
(
ptype
))
if
ptype
:
try
:
ptype
=
ptype
[
0
].
upper
()
+
ptype
[
1
:]
pclass
=
globals
()[
ptype
]
logger
.
debug
(
'Class: {}'
.
format
(
pclass
))
candidates
=
filter
(
lambda
x
:
isinstance
(
x
,
pclass
),
plugins
)
except
KeyError
:
raise
models
.
Error
(
'{} is not a valid type'
.
format
(
ptype
))
else
:
candidates
=
plugins
logger
.
debug
(
candidates
)
def
matches
(
plug
):
res
=
all
(
getattr
(
plug
,
k
,
None
)
==
v
for
(
k
,
v
)
in
kwargs
.
items
())
logger
.
debug
(
"matching {} with {}: {}"
.
format
(
plug
.
name
,
kwargs
,
res
))
return
res
if
kwargs
:
candidates
=
filter
(
matches
,
candidates
)
return
{
p
.
name
:
p
for
p
in
candidates
}
senpy/schemas/context.jsonld
View file @
1a9dd07f
...
...
@@ -37,12 +37,12 @@
"@type": "@id",
"@container": "@set"
},
"plugins": {
"@container": "@list"
},
"options": {
"@container": "@set"
},
"plugins": {
"@container": "@set"
},
"prov:wasGeneratedBy": {
"@type": "@id"
},
...
...
senpy/schemas/plugins.json
View file @
1a9dd07f
...
...
@@ -10,8 +10,6 @@
"items"
:
{
"$ref"
:
"plugin.json"
}
},
"@type"
:
{
}
}
}
...
...
senpy/schemas/results.json
View file @
1a9dd07f
...
...
@@ -18,10 +18,16 @@
"type"
:
"string"
},
"analysis"
:
{
"type"
:
"array"
,
"default"
:
[],
"type"
:
"array"
,
"items"
:
{
"$ref"
:
"analysis.json"
"anyOf"
:
[
{
"$ref"
:
"analysis.json"
},{
"type"
:
"string"
}
]
}
},
"entries"
:
{
...
...
tests/plugins/async_plugin/asyncplugin.py
0 → 100644
View file @
1a9dd07f
from
senpy.plugins
import
AnalysisPlugin
import
multiprocessing
def
_train
(
process_number
):
return
process_number
class
AsyncPlugin
(
AnalysisPlugin
):
def
_do_async
(
self
,
num_processes
):
pool
=
multiprocessing
.
Pool
(
processes
=
num_processes
)
values
=
pool
.
map
(
_train
,
range
(
num_processes
))
return
values
def
activate
(
self
):
self
.
value
=
self
.
_do_async
(
4
)
def
analyse_entry
(
self
,
entry
,
params
):
values
=
self
.
_do_async
(
2
)
entry
.
async_values
=
values
yield
entry
tests/plugins/async_plugin/asyncplugin.senpy
0 → 100644
View file @
1a9dd07f
---
name: Async
module: asyncplugin
description: I am async
author: "@balkian"
version: '0.1'
async: true
extra_params: {}
\ No newline at end of file
tests/test_client.py
View file @
1a9dd07f
...
...
@@ -4,18 +4,21 @@ try:
except
ImportError
:
from
mock
import
patch
import
json
from
senpy.client
import
Client
from
senpy.models
import
Results
,
Error
from
senpy.models
import
Results
,
Plugins
,
Error
from
senpy.plugins
import
AnalysisPlugin
,
default_plugin_type
class
Call
(
dict
):
def
__init__
(
self
,
obj
):
self
.
obj
=
obj
.
jsonld
()
self
.
obj
=
obj
.
serialize
()
self
.
status_code
=
200
self
.
content
=
self
.
json
()
def
json
(
self
):
return
self
.
obj
return
json
.
loads
(
self
.
obj
)
class
ModelsTest
(
TestCase
):
...
...
@@ -44,3 +47,19 @@ class ModelsTest(TestCase):
method
=
'GET'
,
params
=
{
'input'
:
'hello'
,
'algorithm'
:
'NONEXISTENT'
})
def
test_plugins
(
self
):
endpoint
=
'http://dummy/'
client
=
Client
(
endpoint
)
plugins
=
Plugins
()
p1
=
AnalysisPlugin
({
'name'
:
'AnalysisP1'
,
'version'
:
0
,
'description'
:
'No'
})
plugins
.
plugins
=
[
p1
,
]
success
=
Call
(
plugins
)
with
patch
(
'requests.request'
,
return_value
=
success
)
as
patched
:
response
=
client
.
plugins
()
assert
isinstance
(
response
,
dict
)
assert
len
(
response
)
==
1
assert
"AnalysisP1"
in
response
patched
.
assert_called_with
(
url
=
endpoint
+
'/plugins'
,
method
=
'GET'
,
params
=
{
'plugin_type'
:
default_plugin_type
})
tests/test_extensions.py
View file @
1a9dd07f
...
...
@@ -167,7 +167,7 @@ class ExtensionsTest(TestCase):
assert
len
(
senpy
.
plugins
)
>
1
def
test_convert_emotions
(
self
):
self
.
senpy
.
activate_all
()
self
.
senpy
.
activate_all
(
sync
=
True
)
plugin
=
Plugin
({
'id'
:
'imaginary'
,
'onyx:usesEmotionModel'
:
'emoml:fsre-dimensions'
...
...
@@ -205,3 +205,14 @@ class ExtensionsTest(TestCase):
[
plugin
,
],
params
)
assert
len
(
r3
.
entries
[
0
].
emotions
)
==
1
# def test_async_plugin(self):
# """ We should accept multiprocessing plugins with async=False"""
# thread1 = self.senpy.activate_plugin("Async", sync=False)
# thread1.join(timeout=1)
# assert len(self.senpy.plugins['Async'].value) == 4
# resp = self.senpy.analyse(input='nothing', algorithm='Async')
# assert len(resp.entries[0].async_values) == 2
# self.senpy.activate_plugin("Async", sync=True)
tests/test_models.py
View file @
1a9dd07f
...
...
@@ -109,13 +109,15 @@ class ModelsTest(TestCase):
}
}})
c
=
p
.
jsonld
()
assert
"info"
not
in
c
assert
"repo"
not
in
c
assert
"extra_params"
in
c
logging
.
debug
(
"Framed:"
)
assert
'@type'
in
c
assert
c
[
'@type'
]
==
'plugin'
assert
'info'
not
in
c
assert
'repo'
not
in
c
assert
'extra_params'
in
c
logging
.
debug
(
'Framed:'
)
logging
.
debug
(
c
)
p
.
validate
()
assert
"
es
"
in
c
[
'extra_params'
][
'none'
][
'options'
]
assert
'
es
'
in
c
[
'extra_params'
][
'none'
][
'options'
]
assert
isinstance
(
c
[
'extra_params'
][
'none'
][
'options'
],
list
)
def
test_str
(
self
):
...
...
@@ -158,14 +160,20 @@ class ModelsTest(TestCase):
g
=
rdflib
.
Graph
().
parse
(
data
=
t
,
format
=
'turtle'
)
assert
len
(
g
)
==
len
(
triples
)
def
test_plugin_list
(
self
):
"""The plugin list should be of type
\"
plugins
\"
"""
plugs
=
Plugins
()
c
=
plugs
.
jsonld
()
assert
'@type'
in
c
assert
c
[
'@type'
]
==
'plugins'
def
test_single_plugin
(
self
):
"""A response with a single plugin should still return a list"""
plugs
=
Plugins
()
for
i
in
range
(
10
):
p
=
Plugin
({
'id'
:
str
(
i
),
'version'
:
0
,
'description'
:
'dummy'
})
plugs
.
plugins
.
append
(
p
)
p
=
Plugin
({
'id'
:
str
(
1
),
'version'
:
0
,
'description'
:
'dummy'
})
plugs
.
plugins
.
append
(
p
)
assert
isinstance
(
plugs
.
plugins
,
list
)
js
=
plugs
.
jsonld
()
assert
isinstance
(
js
[
'plugins'
],
list
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment