Commit 7fd69cc6 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

YAPFed

parent b543a461
...@@ -7,6 +7,10 @@ VERSION=$(shell cat $(NAME)/VERSION) ...@@ -7,6 +7,10 @@ VERSION=$(shell cat $(NAME)/VERSION)
all: build run all: build run
yapf:
yapf -i -r senpy
yapf -i -r tests
dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS))
ln -s Dockerfile-$(PYMAIN) Dockerfile ln -s Dockerfile-$(PYMAIN) Dockerfile
...@@ -71,4 +75,4 @@ pip_test: $(addprefix pip_test-,$(PYVERSIONS)) ...@@ -71,4 +75,4 @@ pip_test: $(addprefix pip_test-,$(PYVERSIONS))
run: build run: build
docker run --rm -p 5000:5000 -ti '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' docker run --rm -p 5000:5000 -ti '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)'
.PHONY: test test-% build-% build test test_pip run .PHONY: test test-% build-% build test test_pip run yapf
...@@ -22,5 +22,4 @@ import os ...@@ -22,5 +22,4 @@ import os
from .version import __version__ from .version import __version__
__all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins'] __all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins']
...@@ -34,42 +34,51 @@ patch_all(thread=False) ...@@ -34,42 +34,51 @@ patch_all(thread=False)
SERVER_PORT = os.environ.get("PORT", 5000) SERVER_PORT = os.environ.get("PORT", 5000)
def main(): def main():
parser = argparse.ArgumentParser(description='Run a Senpy server') parser = argparse.ArgumentParser(description='Run a Senpy server')
parser.add_argument('--level', parser.add_argument(
'-l', '--level',
metavar='logging_level', '-l',
type=str, metavar='logging_level',
default="INFO", type=str,
help='Logging level') default="INFO",
parser.add_argument('--debug', help='Logging level')
'-d', parser.add_argument(
action='store_true', '--debug',
default=False, '-d',
help='Run the application in debug mode') action='store_true',
parser.add_argument('--default-plugins', default=False,
action='store_true', help='Run the application in debug mode')
default=False, parser.add_argument(
help='Load the default plugins') '--default-plugins',
parser.add_argument('--host', action='store_true',
type=str, default=False,
default="127.0.0.1", help='Load the default plugins')
help='Use 0.0.0.0 to accept requests from any host.') parser.add_argument(
parser.add_argument('--port', '--host',
'-p', type=str,
type=int, default="127.0.0.1",
default=SERVER_PORT, help='Use 0.0.0.0 to accept requests from any host.')
help='Port to listen on.') parser.add_argument(
parser.add_argument('--plugins-folder', '--port',
'-f', '-p',
type=str, type=int,
default='plugins', default=SERVER_PORT,
help='Where to look for plugins.') help='Port to listen on.')
parser.add_argument('--only-install', parser.add_argument(
'-i', '--plugins-folder',
action='store_true', '-f',
default=False, type=str,
help='Do not run a server, only install the dependencies of the plugins.') default='plugins',
help='Where to look for plugins.')
parser.add_argument(
'--only-install',
'-i',
action='store_true',
default=False,
help='Do not run a server, only install the dependencies of the plugins.'
)
args = parser.parse_args() args = parser.parse_args()
logging.basicConfig() logging.basicConfig()
rl = logging.getLogger() rl = logging.getLogger()
...@@ -92,5 +101,6 @@ def main(): ...@@ -92,5 +101,6 @@ def main():
http_server.stop() http_server.stop()
sp.deactivate_all() sp.deactivate_all()
if __name__ == '__main__': if __name__ == '__main__':
main() main()
...@@ -25,7 +25,7 @@ CLI_PARAMS = { ...@@ -25,7 +25,7 @@ CLI_PARAMS = {
"required": True, "required": True,
"default": "." "default": "."
}, },
} }
NIF_PARAMS = { NIF_PARAMS = {
"input": { "input": {
...@@ -96,10 +96,11 @@ def parse_params(indict, spec=NIF_PARAMS): ...@@ -96,10 +96,11 @@ def parse_params(indict, spec=NIF_PARAMS):
outdict[param] not in spec[param]["options"]: outdict[param] not in spec[param]["options"]:
wrong_params[param] = spec[param] wrong_params[param] = spec[param]
if wrong_params: if wrong_params:
message = Error(status=404, message = Error(
message="Missing or invalid parameters", status=404,
parameters=outdict, message="Missing or invalid parameters",
errors={param: error for param, error in parameters=outdict,
iteritems(wrong_params)}) errors={param: error
for param, error in iteritems(wrong_params)})
raise message raise message
return outdict return outdict
...@@ -30,6 +30,7 @@ logger = logging.getLogger(__name__) ...@@ -30,6 +30,7 @@ logger = logging.getLogger(__name__)
api_blueprint = Blueprint("api", __name__) api_blueprint = Blueprint("api", __name__)
demo_blueprint = Blueprint("demo", __name__) demo_blueprint = Blueprint("demo", __name__)
def get_params(req): def get_params(req):
if req.method == 'POST': if req.method == 'POST':
indict = req.form.to_dict(flat=True) indict = req.form.to_dict(flat=True)
...@@ -44,17 +45,20 @@ def get_params(req): ...@@ -44,17 +45,20 @@ def get_params(req):
def index(): def index():
return render_template("index.html") return render_template("index.html")
@api_blueprint.route('/contexts/<entity>.jsonld') @api_blueprint.route('/contexts/<entity>.jsonld')
def context(entity="context"): def context(entity="context"):
return jsonify({"@context": Response.context}) return jsonify({"@context": Response.context})
@api_blueprint.route('/schemas/<schema>') @api_blueprint.route('/schemas/<schema>')
def schema(schema="definitions"): def schema(schema="definitions"):
try: try:
return jsonify(read_schema(schema)) return jsonify(read_schema(schema))
except Exception: # Should be FileNotFoundError, but it's missing from py2 except Exception: # Should be FileNotFoundError, but it's missing from py2
return Error(message="Schema not found", status=404).flask() return Error(message="Schema not found", status=404).flask()
def basic_api(f): def basic_api(f):
@wraps(f) @wraps(f)
def decorated_function(*args, **kwargs): def decorated_function(*args, **kwargs):
...@@ -73,12 +77,15 @@ def basic_api(f): ...@@ -73,12 +77,15 @@ def basic_api(f):
response = ex response = ex
in_headers = web_params["inHeaders"] != "0" in_headers = web_params["inHeaders"] != "0"
headers = {'X-ORIGINAL-PARAMS': raw_params} headers = {'X-ORIGINAL-PARAMS': raw_params}
return response.flask(in_headers=in_headers, return response.flask(
headers=headers, in_headers=in_headers,
context_uri=url_for('api.context', entity=type(response).__name__, headers=headers,
_external=True)) context_uri=url_for(
'api.context', entity=type(response).__name__, _external=True))
return decorated_function return decorated_function
@api_blueprint.route('/', methods=['POST', 'GET']) @api_blueprint.route('/', methods=['POST', 'GET'])
@basic_api @basic_api
def api(): def api():
...@@ -92,7 +99,8 @@ def plugins(): ...@@ -92,7 +99,8 @@ def plugins():
sp = current_app.senpy sp = current_app.senpy
dic = Plugins(plugins=list(sp.plugins.values())) dic = Plugins(plugins=list(sp.plugins.values()))
return dic return dic
@api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET']) @api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET']) @api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
@basic_api @basic_api
...@@ -110,12 +118,13 @@ def plugin(plugin=None, action="list"): ...@@ -110,12 +118,13 @@ def plugin(plugin=None, action="list"):
if action == "list": if action == "list":
return response return response
method = "{}_plugin".format(action) method = "{}_plugin".format(action)
if(hasattr(sp, method)): if (hasattr(sp, method)):
getattr(sp, method)(plugin) getattr(sp, method)(plugin)
return Response(message="Ok") return Response(message="Ok")
else: else:
return Error(message="action '{}' not allowed".format(action)) return Error(message="action '{}' not allowed".format(action))
if __name__ == '__main__': if __name__ == '__main__':
import config import config
......
...@@ -3,6 +3,7 @@ from .models import Error ...@@ -3,6 +3,7 @@ from .models import Error
from .api import parse_params, CLI_PARAMS from .api import parse_params, CLI_PARAMS
from .extensions import Senpy from .extensions import Senpy
def argv_to_dict(argv): def argv_to_dict(argv):
'''Turns parameters in the form of '--key value' into a dict {'key': 'value'} '''Turns parameters in the form of '--key value' into a dict {'key': 'value'}
''' '''
...@@ -11,13 +12,14 @@ def argv_to_dict(argv): ...@@ -11,13 +12,14 @@ def argv_to_dict(argv):
for i in range(len(argv)): for i in range(len(argv)):
if argv[i][0] == '-': if argv[i][0] == '-':
key = argv[i].strip('-') key = argv[i].strip('-')
value = argv[i+1] if len(argv)>i+1 else None value = argv[i + 1] if len(argv) > i + 1 else None
if value and value[0] == '-': if value and value[0] == '-':
cli_dict[key] = "" cli_dict[key] = ""
else: else:
cli_dict[key] = value cli_dict[key] = value
return cli_dict return cli_dict
def parse_cli(argv): def parse_cli(argv):
cli_dict = argv_to_dict(argv) cli_dict = argv_to_dict(argv)
cli_params = parse_params(cli_dict, spec=CLI_PARAMS) cli_params = parse_params(cli_dict, spec=CLI_PARAMS)
...@@ -34,6 +36,7 @@ def main_function(argv): ...@@ -34,6 +36,7 @@ def main_function(argv):
res = sp.analyse(**cli_dict) res = sp.analyse(**cli_dict)
return res return res
def main(): def main():
'''This method is the entrypoint for the CLI (as configured un setup.py) '''This method is the entrypoint for the CLI (as configured un setup.py)
''' '''
...@@ -43,7 +46,7 @@ def main(): ...@@ -43,7 +46,7 @@ def main():
except Error as err: except Error as err:
print(err.to_JSON()) print(err.to_JSON())
sys.exit(2) sys.exit(2)
if __name__ == '__main__': if __name__ == '__main__':
main() main()
...@@ -29,10 +29,12 @@ logger = logging.getLogger(__name__) ...@@ -29,10 +29,12 @@ logger = logging.getLogger(__name__)
class Senpy(object): class Senpy(object):
""" Default Senpy extension for Flask """ """ Default Senpy extension for Flask """
def __init__(self, app=None, plugin_folder="plugins", default_plugins=False): def __init__(self,
app=None,
plugin_folder="plugins",
default_plugins=False):
self.app = app self.app = app
self._search_folders = set() self._search_folders = set()
...@@ -80,22 +82,24 @@ class Senpy(object): ...@@ -80,22 +82,24 @@ class Senpy(object):
elif self.plugins: elif self.plugins:
algo = self.default_plugin and self.default_plugin.name algo = self.default_plugin and self.default_plugin.name
if not algo: if not algo:
raise Error(status=404, raise Error(
message=("No plugins found." status=404,
" Please install one.").format(algo)) message=("No plugins found."
" Please install one.").format(algo))
if algo not in self.plugins: if algo not in self.plugins:
logger.debug(("The algorithm '{}' is not valid\n" logger.debug(("The algorithm '{}' is not valid\n"
"Valid algorithms: {}").format(algo, "Valid algorithms: {}").format(algo,
self.plugins.keys())) self.plugins.keys()))
raise Error(status=404, raise Error(
message="The algorithm '{}' is not valid" status=404,
.format(algo)) message="The algorithm '{}' is not valid".format(algo))
if not self.plugins[algo].is_activated: if not self.plugins[algo].is_activated:
logger.debug("Plugin not activated: {}".format(algo)) logger.debug("Plugin not activated: {}".format(algo))
raise Error(status=400, raise Error(
message=("The algorithm '{}'" status=400,
" is not activated yet").format(algo)) message=("The algorithm '{}'"
" is not activated yet").format(algo))
plug = self.plugins[algo] plug = self.plugins[algo]
nif_params = parse_params(params, spec=NIF_PARAMS) nif_params = parse_params(params, spec=NIF_PARAMS)
extra_params = plug.get('extra_params', {}) extra_params = plug.get('extra_params', {})
...@@ -120,9 +124,8 @@ class Senpy(object): ...@@ -120,9 +124,8 @@ class Senpy(object):
return None return None
def parameters(self, algo): def parameters(self, algo):
return getattr(self.plugins.get(algo) or self.default_plugin, return getattr(
"extra_params", self.plugins.get(algo) or self.default_plugin, "extra_params", {})
{})
def activate_all(self, sync=False): def activate_all(self, sync=False):
ps = [] ps = []
...@@ -146,18 +149,20 @@ class Senpy(object): ...@@ -146,18 +149,20 @@ class Senpy(object):
try: try:
plugin = self.plugins[plugin_name] plugin = self.plugins[plugin_name]
except KeyError: except KeyError:
raise Error(message="Plugin not found: {}".format(plugin_name), raise Error(
status=404) message="Plugin not found: {}".format(plugin_name), status=404)
logger.info("Activating plugin: {}".format(plugin.name)) logger.info("Activating plugin: {}".format(plugin.name))
def act(): def act():
try: try:
plugin.activate() plugin.activate()
logger.info("Plugin activated: {}".format(plugin.name)) logger.info("Plugin activated: {}".format(plugin.name))
except Exception as ex: except Exception as ex:
logger.error("Error activating plugin {}: {}".format(plugin.name, logger.error("Error activating plugin {}: {}".format(
ex)) plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc())) logger.error("Trace: {}".format(traceback.format_exc()))
th = gevent.spawn(act) th = gevent.spawn(act)
th.link_value(partial(self._set_active_plugin, plugin_name, True)) th.link_value(partial(self._set_active_plugin, plugin_name, True))
if sync: if sync:
...@@ -169,16 +174,16 @@ class Senpy(object): ...@@ -169,16 +174,16 @@ class Senpy(object):
try: try:
plugin = self.plugins[plugin_name] plugin = self.plugins[plugin_name]
except KeyError: except KeyError:
raise Error(message="Plugin not found: {}".format(plugin_name), raise Error(
status=404) message="Plugin not found: {}".format(plugin_name), status=404)
def deact(): def deact():
try: try:
plugin.deactivate() plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name)) logger.info("Plugin deactivated: {}".format(plugin.name))
except Exception as ex: except Exception as ex:
logger.error("Error deactivating plugin {}: {}".format(plugin.name, logger.error("Error deactivating plugin {}: {}".format(
ex)) plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc())) logger.error("Trace: {}".format(traceback.format_exc()))
th = gevent.spawn(deact) th = gevent.spawn(deact)
...@@ -199,7 +204,6 @@ class Senpy(object): ...@@ -199,7 +204,6 @@ class Senpy(object):
logger.error('Error reloading {}: {}'.format(name, ex)) logger.error('Error reloading {}: {}'.format(name, ex))
self.plugins[name] = plugin self.plugins[name] = plugin
@classmethod @classmethod
def validate_info(cls, info): def validate_info(cls, info):
return all(x in info for x in ('name', 'module', 'version')) return all(x in info for x in ('name', 'module', 'version'))
...@@ -215,15 +219,15 @@ class Senpy(object): ...@@ -215,15 +219,15 @@ class Senpy(object):
pip_args = [] pip_args = []
pip_args.append('install') pip_args.append('install')
for req in requirements: for req in requirements:
pip_args.append( req ) pip_args.append(req)
logger.info('Installing requirements: ' + str(requirements)) logger.info('Installing requirements: ' + str(requirements))
pip.main(pip_args) pip.main(pip_args)
@classmethod @classmethod
def _load_plugin_from_info(cls, info, root): def _load_plugin_from_info(cls, info, root):
if not cls.validate_info(info): if not cls.validate_info(info):
logger.warn('The module info is not valid.\n\t{}'.format(info)) logger.warn('The module info is not valid.\n\t{}'.format(info))
return None, None return None, None
module = info["module"] module = info["module"]
name = info["name"] name = info["name"]
requirements = info.get("requirements", []) requirements = info.get("requirements", [])
...@@ -237,8 +241,8 @@ class Senpy(object): ...@@ -237,8 +241,8 @@ class Senpy(object):
for _, obj in inspect.getmembers(tmp): for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp: if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug(("Found plugin class:" logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj)) " {}@{}").format(obj, inspect.getmodule(
) obj)))
candidate = obj candidate = obj
break break
if not candidate: if not candidate:
...@@ -248,7 +252,8 @@ class Senpy(object): ...@@ -248,7 +252,8 @@ class Senpy(object):
repo_path = root repo_path = root
module._repo = Repo(repo_path) module._repo = Repo(repo_path)
except InvalidGitRepositoryError: except InvalidGitRepositoryError:
logger.debug("The plugin {} is not in a Git repository".format(module)) logger.debug("The plugin {} is not in a Git repository".format(
module))
module._repo = None module._repo = None
except Exception as ex: except Exception as ex:
logger.error("Exception importing {}: {}".format(module, ex)) logger.error("Exception importing {}: {}".format(module, ex))
...@@ -265,7 +270,6 @@ class Senpy(object): ...@@ -265,7 +270,6 @@ class Senpy(object):
logger.debug("Info: {}".format(info)) logger.debug("Info: {}".format(info))
return cls._load_plugin_from_info(info, root) return cls._load_plugin_from_info(info, root)
def _load_plugins(self): def _load_plugins(self):
plugins = {} plugins = {}
for search_folder in self._search_folders: for search_folder in self._search_folders:
...@@ -293,8 +297,7 @@ class Senpy(object): ...@@ -293,8 +297,7 @@ class Senpy(object):
def matches(plug): def matches(plug):
res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items()) res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
logger.debug("matching {} with {}: {}".format(plug.name, logger.debug("matching {} with {}: {}".format(plug.name, kwargs,
kwargs,
res)) res))
return res return res
...@@ -305,5 +308,8 @@ class Senpy(object): ...@@ -305,5 +308,8 @@ class Senpy(object):
def sentiment_plugins(self): def sentiment_plugins(self):
""" Return only the sentiment plugins """ """ Return only the sentiment plugins """
return {p: plugin for p, plugin in self.plugins.items() if return {
isinstance(plugin, SentimentPlugin)} p: plugin
for p, plugin in self.plugins.items()
if isinstance(plugin, SentimentPlugin)
}
...@@ -18,15 +18,18 @@ import jsonschema ...@@ -18,15 +18,18 @@ import jsonschema
from flask import Response as FlaskResponse from flask import Response as FlaskResponse
DEFINITIONS_FILE = 'definitions.json' DEFINITIONS_FILE = 'definitions.json'
CONTEXT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld') CONTEXT_PATH = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
def get_schema_path(schema_file, absolute=False): def get_schema_path(schema_file, absolute=False):
if absolute: if absolute: