Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
senpy
senpy
Commits
778746c5
Commit
778746c5
authored
Nov 22, 2017
by
J. Fernando Sánchez
Browse files
Added data folder configuration
Closes
#46
parent
19278d0a
Changes
8
Hide whitespace changes
Inline
Side-by-side
README.rst
View file @
778746c5
.. image:: img/header.png
:
height: 6em
:
width: 100%
:target: http://demos.gsi.dit.upm.es/senpy
.. image:: https://travis-ci.org/gsi-upm/senpy.svg?branch=master
...
...
senpy/__main__.py
View file @
778746c5
...
...
@@ -75,6 +75,12 @@ def main():
action
=
'store_true'
,
default
=
False
,
help
=
'Do not run a server, only install plugin dependencies'
)
parser
.
add_argument
(
'--data-folder'
,
'--data'
,
type
=
str
,
default
=
None
,
help
=
'Where to look for data. It be set with the SENPY_DATA environment variable as well.'
)
parser
.
add_argument
(
'--threaded'
,
action
=
'store_false'
,
...
...
@@ -96,7 +102,9 @@ def main():
rl
.
setLevel
(
getattr
(
logging
,
args
.
level
))
app
=
Flask
(
__name__
)
app
.
debug
=
args
.
debug
sp
=
Senpy
(
app
,
args
.
plugins_folder
,
default_plugins
=
args
.
default_plugins
)
sp
=
Senpy
(
app
,
args
.
plugins_folder
,
default_plugins
=
args
.
default_plugins
,
data_folder
=
args
.
data_folder
)
sp
.
install_deps
()
if
args
.
only_install
:
return
...
...
senpy/extensions.py
View file @
778746c5
...
...
@@ -15,6 +15,7 @@ from functools import partial
import
os
import
copy
import
errno
import
logging
import
traceback
...
...
@@ -27,6 +28,7 @@ class Senpy(object):
def
__init__
(
self
,
app
=
None
,
plugin_folder
=
"."
,
data_folder
=
None
,
default_plugins
=
False
):
self
.
app
=
app
self
.
_search_folders
=
set
()
...
...
@@ -42,6 +44,17 @@ class Senpy(object):
self
.
add_folder
(
os
.
path
.
join
(
'plugins'
,
'conversion'
),
from_root
=
True
)
self
.
data_folder
=
data_folder
or
os
.
environ
.
get
(
'SENPY_DATA'
,
os
.
path
.
join
(
os
.
getcwd
(),
'senpy_data'
))
try
:
os
.
makedirs
(
self
.
data_folder
)
except
OSError
as
e
:
if
e
.
errno
==
errno
.
EEXIST
:
print
(
'Directory not created.'
)
else
:
raise
if
app
is
not
None
:
self
.
init_app
(
app
)
...
...
@@ -312,7 +325,8 @@ class Senpy(object):
def
plugins
(
self
):
""" Return the plugins registered for a given application. """
if
self
.
_outdated
:
self
.
_plugin_list
=
plugins
.
load_plugins
(
self
.
_search_folders
)
self
.
_plugin_list
=
plugins
.
load_plugins
(
self
.
_search_folders
,
data_folder
=
self
.
data_folder
)
self
.
_outdated
=
False
return
self
.
_plugin_list
...
...
senpy/plugins/__init__.py
View file @
778746c5
...
...
@@ -5,7 +5,6 @@ import os.path
import
os
import
pickle
import
logging
import
tempfile
import
copy
import
fnmatch
...
...
@@ -16,6 +15,8 @@ import importlib
import
yaml
import
threading
from
contextlib
import
contextmanager
from
..
import
models
,
utils
from
..api
import
API_PARAMS
...
...
@@ -23,7 +24,7 @@ logger = logging.getLogger(__name__)
class
Plugin
(
models
.
Plugin
):
def
__init__
(
self
,
info
=
None
):
def
__init__
(
self
,
info
=
None
,
data_folder
=
None
):
"""
Provides a canonical name for plugins and serves as base for other
kinds of plugins.
...
...
@@ -36,6 +37,7 @@ class Plugin(models.Plugin):
super
(
Plugin
,
self
).
__init__
(
id
=
id
,
**
info
)
self
.
is_activated
=
False
self
.
_lock
=
threading
.
Lock
()
self
.
data_folder
=
data_folder
or
os
.
getcwd
()
def
get_folder
(
self
):
return
os
.
path
.
dirname
(
inspect
.
getfile
(
self
.
__class__
))
...
...
@@ -61,6 +63,13 @@ class Plugin(models.Plugin):
for
r
in
res
:
r
.
validate
()
@
contextmanager
def
open
(
self
,
fpath
,
*
args
,
**
kwargs
):
if
not
os
.
path
.
isabs
(
fpath
):
fpath
=
os
.
path
.
join
(
self
.
data_folder
,
fpath
)
with
open
(
fpath
,
*
args
,
**
kwargs
)
as
f
:
yield
f
SenpyPlugin
=
Plugin
...
...
@@ -121,7 +130,8 @@ class ShelfMixin(object):
self
.
__dict__
[
'_sh'
]
=
{}
if
os
.
path
.
isfile
(
self
.
shelf_file
):
try
:
self
.
__dict__
[
'_sh'
]
=
pickle
.
load
(
open
(
self
.
shelf_file
,
'rb'
))
with
self
.
open
(
self
.
shelf_file
,
'rb'
)
as
p
:
self
.
__dict__
[
'_sh'
]
=
pickle
.
load
(
p
)
except
(
IndexError
,
EOFError
,
pickle
.
UnpicklingError
):
logger
.
warning
(
'{} has a corrupted shelf file!'
.
format
(
self
.
id
))
if
not
self
.
get
(
'force_shelf'
,
False
):
...
...
@@ -138,14 +148,13 @@ class ShelfMixin(object):
@
property
def
shelf_file
(
self
):
if
'shelf_file'
not
in
self
or
not
self
[
'shelf_file'
]:
sd
=
os
.
environ
.
get
(
'SENPY_DATA'
,
tempfile
.
gettempdir
())
self
.
shelf_file
=
os
.
path
.
join
(
sd
,
self
.
name
+
'.p'
)
self
.
shelf_file
=
os
.
path
.
join
(
self
.
data_folder
,
self
.
name
+
'.p'
)
return
self
[
'shelf_file'
]
def
save
(
self
):
logger
.
debug
(
'saving pickle'
)
if
hasattr
(
self
,
'_sh'
)
and
self
.
_sh
is
not
None
:
with
open
(
self
.
shelf_file
,
'wb'
)
as
f
:
with
self
.
open
(
self
.
shelf_file
,
'wb'
)
as
f
:
pickle
.
dump
(
self
.
_sh
,
f
)
...
...
@@ -207,12 +216,11 @@ def log_subprocess_output(process):
def
install_deps
(
*
plugins
):
installed
=
False
for
info
in
plugins
:
requirements
=
info
.
get
(
'requirements'
,
[])
if
requirements
:
pip_args
=
[
sys
.
executable
,
'-m'
,
'pip'
]
pip_args
.
append
(
'install'
)
pip_args
.
append
(
'--use-wheel'
)
pip_args
=
[
sys
.
executable
,
'-m'
,
'pip'
,
'install'
,
'--use-wheel'
]
for
req
in
requirements
:
pip_args
.
append
(
req
)
logger
.
info
(
'Installing requirements: '
+
str
(
requirements
))
...
...
@@ -221,11 +229,13 @@ def install_deps(*plugins):
stderr
=
subprocess
.
PIPE
)
log_subprocess_output
(
process
)
exitcode
=
process
.
wait
()
installed
=
True
if
exitcode
!=
0
:
raise
models
.
Error
(
"Dependencies not properly installed"
)
return
installed
def
load_plugin_from_info
(
info
,
root
=
None
,
validator
=
validate_info
,
install
=
True
):
def
load_plugin_from_info
(
info
,
root
=
None
,
validator
=
validate_info
,
install
=
True
,
*
args
,
**
kwargs
):
if
not
root
and
'_path'
in
info
:
root
=
os
.
path
.
dirname
(
info
[
'_path'
])
if
not
validator
(
info
):
...
...
@@ -249,7 +259,7 @@ def load_plugin_from_info(info, root=None, validator=validate_info, install=True
if
not
candidate
:
logger
.
debug
(
"No valid plugin for: {}"
.
format
(
module
))
return
module
=
candidate
(
info
=
info
)
module
=
candidate
(
info
=
info
,
*
args
,
**
kwargs
)
return
module
...
...
@@ -262,14 +272,14 @@ def parse_plugin_info(fpath):
return
name
,
info
def
load_plugin
(
fpath
):
def
load_plugin
(
fpath
,
*
args
,
**
kwargs
):
name
,
info
=
parse_plugin_info
(
fpath
)
logger
.
debug
(
"Info: {}"
.
format
(
info
))
plugin
=
load_plugin_from_info
(
info
)
plugin
=
load_plugin_from_info
(
info
,
*
args
,
**
kwargs
)
return
name
,
plugin
def
load_plugins
(
folders
,
loader
=
load_plugin
):
def
load_plugins
(
folders
,
loader
=
load_plugin
,
*
args
,
**
kwargs
):
plugins
=
{}
for
search_folder
in
folders
:
for
root
,
dirnames
,
filenames
in
os
.
walk
(
search_folder
):
...
...
@@ -277,7 +287,7 @@ def load_plugins(folders, loader=load_plugin):
dirnames
[:]
=
[
d
for
d
in
dirnames
if
d
[
0
]
not
in
[
'.'
,
'_'
]]
for
filename
in
fnmatch
.
filter
(
filenames
,
'*.senpy'
):
fpath
=
os
.
path
.
join
(
root
,
filename
)
name
,
plugin
=
loader
(
fpath
)
name
,
plugin
=
loader
(
fpath
,
*
args
,
**
kwargs
)
if
plugin
and
name
:
plugins
[
name
]
=
plugin
return
plugins
senpy/plugins/conversion/emotion/centroids.py
View file @
778746c5
...
...
@@ -6,7 +6,7 @@ logger = logging.getLogger(__name__)
class
CentroidConversion
(
EmotionConversionPlugin
):
def
__init__
(
self
,
info
):
def
__init__
(
self
,
info
,
*
args
,
**
kwargs
):
if
'centroids'
not
in
info
:
raise
Error
(
'Centroid conversion plugins should provide '
'the centroids in their senpy file'
)
...
...
@@ -33,7 +33,7 @@ class CentroidConversion(EmotionConversionPlugin):
ncentroids
[
aliases
.
get
(
k1
,
k1
)]
=
nv1
info
[
'centroids'
]
=
ncentroids
super
(
CentroidConversion
,
self
).
__init__
(
info
)
super
(
CentroidConversion
,
self
).
__init__
(
info
,
*
args
,
**
kwargs
)
self
.
dimensions
=
set
()
for
c
in
self
.
centroids
.
values
():
...
...
tests/plugins/noop/noop_plugin.py
0 → 100644
View file @
778746c5
from
senpy.plugins
import
SentimentPlugin
class
DummyPlugin
(
SentimentPlugin
):
import
noop
tests/test_extensions.py
View file @
778746c5
...
...
@@ -49,14 +49,13 @@ class ExtensionsTest(TestCase):
""" Installing a plugin """
info
=
{
'name'
:
'TestPip'
,
'module'
:
'
dummy
'
,
'module'
:
'
noop_plugin
'
,
'description'
:
None
,
'requirements'
:
[
'noop'
],
'version'
:
0
}
root
=
os
.
path
.
join
(
self
.
dir
,
'plugins'
,
'dummy_plugin'
)
module
=
plugins
.
load_plugin_from_info
(
info
,
root
=
root
)
plugins
.
install_deps
(
info
)
root
=
os
.
path
.
join
(
self
.
dir
,
'plugins'
,
'noop'
)
module
=
plugins
.
load_plugin_from_info
(
info
,
root
=
root
,
install
=
True
)
assert
module
.
name
==
'TestPip'
assert
module
import
noop
...
...
tests/test_plugins.py
View file @
778746c5
...
...
@@ -222,8 +222,9 @@ def make_mini_test(plugin_info):
def
_add_tests
():
root
=
os
.
path
.
dirname
(
__file__
)
plugs
=
plugins
.
load_plugins
(
os
.
path
.
join
(
root
,
".."
),
loader
=
plugins
.
parse_plugin_info
)
root
=
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
'..'
)
print
(
root
)
plugs
=
plugins
.
load_plugins
([
root
,
],
loader
=
plugins
.
parse_plugin_info
)
for
k
,
v
in
plugs
.
items
():
pass
t_method
=
make_mini_test
(
v
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment