Skip to content

GitLab

Commit 7fd69cc6 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

YAPFed

parent b543a461
Hide whitespace changes
Inline Side-by-side
Showing with 283 additions and 227 deletions
Makefile
View file @ 7fd69cc6
......@@ -7,6 +7,10 @@ VERSION=$(shell cat $(NAME)/VERSION)
all: build run
yapf:
yapf -i -r senpy
yapf -i -r tests
dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS))
ln -s Dockerfile-$(PYMAIN) Dockerfile
......@@ -71,4 +75,4 @@ pip_test: $(addprefix pip_test-,$(PYVERSIONS))
run: build
docker run --rm -p 5000:5000 -ti '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)'
.PHONY: test test-% build-% build test test_pip run
.PHONY: test test-% build-% build test test_pip run yapf
senpy/__init__.py
View file @ 7fd69cc6
......@@ -22,5 +22,4 @@ import os
from .version import __version__
__all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins']
senpy/__main__.py
View file @ 7fd69cc6
......@@ -34,42 +34,51 @@ patch_all(thread=False)
SERVER_PORT = os.environ.get("PORT", 5000)
def main():
parser = argparse.ArgumentParser(description='Run a Senpy server')
parser.add_argument('--level',
'-l',
metavar='logging_level',
type=str,
default="INFO",
help='Logging level')
parser.add_argument('--debug',
'-d',
action='store_true',
default=False,
help='Run the application in debug mode')
parser.add_argument('--default-plugins',
action='store_true',
default=False,
help='Load the default plugins')
parser.add_argument('--host',
type=str,
default="127.0.0.1",
help='Use 0.0.0.0 to accept requests from any host.')
parser.add_argument('--port',
'-p',
type=int,
default=SERVER_PORT,
help='Port to listen on.')
parser.add_argument('--plugins-folder',
'-f',
type=str,
default='plugins',
help='Where to look for plugins.')
parser.add_argument('--only-install',
'-i',
action='store_true',
default=False,
help='Do not run a server, only install the dependencies of the plugins.')
parser.add_argument(
'--level',
'-l',
metavar='logging_level',
type=str,
default="INFO",
help='Logging level')
parser.add_argument(
'--debug',
'-d',
action='store_true',
default=False,
help='Run the application in debug mode')
parser.add_argument(
'--default-plugins',
action='store_true',
default=False,
help='Load the default plugins')
parser.add_argument(
'--host',
type=str,
default="127.0.0.1",
help='Use 0.0.0.0 to accept requests from any host.')
parser.add_argument(
'--port',
'-p',
type=int,
default=SERVER_PORT,
help='Port to listen on.')
parser.add_argument(
'--plugins-folder',
'-f',
type=str,
default='plugins',
help='Where to look for plugins.')
parser.add_argument(
'--only-install',
'-i',
action='store_true',
default=False,
help='Do not run a server, only install the dependencies of the plugins.'
)
args = parser.parse_args()
logging.basicConfig()
rl = logging.getLogger()
......@@ -92,5 +101,6 @@ def main():
http_server.stop()
sp.deactivate_all()
if __name__ == '__main__':
main()
senpy/api.py
View file @ 7fd69cc6
......@@ -25,7 +25,7 @@ CLI_PARAMS = {
"required": True,
"default": "."
},
}
}
NIF_PARAMS = {
"input": {
......@@ -96,10 +96,11 @@ def parse_params(indict, spec=NIF_PARAMS):
outdict[param] not in spec[param]["options"]:
wrong_params[param] = spec[param]
if wrong_params:
message = Error(status=404,
message="Missing or invalid parameters",
parameters=outdict,
errors={param: error for param, error in
iteritems(wrong_params)})
message = Error(
status=404,
message="Missing or invalid parameters",
parameters=outdict,
errors={param: error
for param, error in iteritems(wrong_params)})
raise message
return outdict
senpy/blueprints.py
View file @ 7fd69cc6
......@@ -30,6 +30,7 @@ logger = logging.getLogger(__name__)
api_blueprint = Blueprint("api", __name__)
demo_blueprint = Blueprint("demo", __name__)
def get_params(req):
if req.method == 'POST':
indict = req.form.to_dict(flat=True)
......@@ -44,17 +45,20 @@ def get_params(req):
def index():
return render_template("index.html")
@api_blueprint.route('/contexts/<entity>.jsonld')
def context(entity="context"):
return jsonify({"@context": Response.context})
@api_blueprint.route('/schemas/<schema>')
def schema(schema="definitions"):
try:
return jsonify(read_schema(schema))
except Exception: # Should be FileNotFoundError, but it's missing from py2
except Exception: # Should be FileNotFoundError, but it's missing from py2
return Error(message="Schema not found", status=404).flask()
def basic_api(f):
@wraps(f)
def decorated_function(*args, **kwargs):
......@@ -73,12 +77,15 @@ def basic_api(f):
response = ex
in_headers = web_params["inHeaders"] != "0"
headers = {'X-ORIGINAL-PARAMS': raw_params}
return response.flask(in_headers=in_headers,
headers=headers,
context_uri=url_for('api.context', entity=type(response).__name__,
_external=True))
return response.flask(
in_headers=in_headers,
headers=headers,
context_uri=url_for(
'api.context', entity=type(response).__name__, _external=True))
return decorated_function
@api_blueprint.route('/', methods=['POST', 'GET'])
@basic_api
def api():
......@@ -92,7 +99,8 @@ def plugins():
sp = current_app.senpy
dic = Plugins(plugins=list(sp.plugins.values()))
return dic
@api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
@basic_api
......@@ -110,12 +118,13 @@ def plugin(plugin=None, action="list"):
if action == "list":
return response
method = "{}_plugin".format(action)
if(hasattr(sp, method)):
if (hasattr(sp, method)):
getattr(sp, method)(plugin)
return Response(message="Ok")
else:
return Error(message="action '{}' not allowed".format(action))
if __name__ == '__main__':
import config
......
senpy/cli.py
View file @ 7fd69cc6
......@@ -3,6 +3,7 @@ from .models import Error
from .api import parse_params, CLI_PARAMS
from .extensions import Senpy
def argv_to_dict(argv):
'''Turns parameters in the form of '--key value' into a dict {'key': 'value'}
'''
......@@ -11,13 +12,14 @@ def argv_to_dict(argv):
for i in range(len(argv)):
if argv[i][0] == '-':
key = argv[i].strip('-')
value = argv[i+1] if len(argv)>i+1 else None
value = argv[i + 1] if len(argv) > i + 1 else None
if value and value[0] == '-':
cli_dict[key] = ""
else:
cli_dict[key] = value
return cli_dict
def parse_cli(argv):
cli_dict = argv_to_dict(argv)
cli_params = parse_params(cli_dict, spec=CLI_PARAMS)
......@@ -34,6 +36,7 @@ def main_function(argv):
res = sp.analyse(**cli_dict)
return res
def main():
'''This method is the entrypoint for the CLI (as configured un setup.py)
'''
......@@ -43,7 +46,7 @@ def main():
except Error as err:
print(err.to_JSON())
sys.exit(2)
if __name__ == '__main__':
main()
senpy/extensions.py
View file @ 7fd69cc6
......@@ -29,10 +29,12 @@ logger = logging.getLogger(__name__)
class Senpy(object):
""" Default Senpy extension for Flask """
def __init__(self, app=None, plugin_folder="plugins", default_plugins=False):
def __init__(self,
app=None,
plugin_folder="plugins",
default_plugins=False):
self.app = app
self._search_folders = set()
......@@ -80,22 +82,24 @@ class Senpy(object):
elif self.plugins:
algo = self.default_plugin and self.default_plugin.name
if not algo:
raise Error(status=404,
message=("No plugins found."
" Please install one.").format(algo))
raise Error(
status=404,
message=("No plugins found."
" Please install one.").format(algo))
if algo not in self.plugins:
logger.debug(("The algorithm '{}' is not valid\n"
"Valid algorithms: {}").format(algo,
self.plugins.keys()))
raise Error(status=404,
message="The algorithm '{}' is not valid"
.format(algo))
raise Error(
status=404,
message="The algorithm '{}' is not valid".format(algo))
if not self.plugins[algo].is_activated:
logger.debug("Plugin not activated: {}".format(algo))
raise Error(status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
raise Error(
status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
plug = self.plugins[algo]
nif_params = parse_params(params, spec=NIF_PARAMS)
extra_params = plug.get('extra_params', {})
......@@ -120,9 +124,8 @@ class Senpy(object):
return None
def parameters(self, algo):
return getattr(self.plugins.get(algo) or self.default_plugin,
"extra_params",
{})
return getattr(
self.plugins.get(algo) or self.default_plugin, "extra_params", {})
def activate_all(self, sync=False):
ps = []
......@@ -146,18 +149,20 @@ class Senpy(object):
try:
plugin = self.plugins[plugin_name]
except KeyError:
raise Error(message="Plugin not found: {}".format(plugin_name),
status=404)
raise Error(
message="Plugin not found: {}".format(plugin_name), status=404)
logger.info("Activating plugin: {}".format(plugin.name))
def act():
try:
plugin.activate()
logger.info("Plugin activated: {}".format(plugin.name))
except Exception as ex:
logger.error("Error activating plugin {}: {}".format(plugin.name,
ex))
logger.error("Error activating plugin {}: {}".format(
plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc()))
th = gevent.spawn(act)
th.link_value(partial(self._set_active_plugin, plugin_name, True))
if sync:
......@@ -169,16 +174,16 @@ class Senpy(object):
try:
plugin = self.plugins[plugin_name]
except KeyError:
raise Error(message="Plugin not found: {}".format(plugin_name),
status=404)
raise Error(
message="Plugin not found: {}".format(plugin_name), status=404)
def deact():
try:
plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name))
except Exception as ex:
logger.error("Error deactivating plugin {}: {}".format(plugin.name,
ex))
logger.error("Error deactivating plugin {}: {}".format(
plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc()))
th = gevent.spawn(deact)
......@@ -199,7 +204,6 @@ class Senpy(object):
logger.error('Error reloading {}: {}'.format(name, ex))
self.plugins[name] = plugin
@classmethod
def validate_info(cls, info):
return all(x in info for x in ('name', 'module', 'version'))
......@@ -215,15 +219,15 @@ class Senpy(object):
pip_args = []
pip_args.append('install')
for req in requirements:
pip_args.append( req )
pip_args.append(req)
logger.info('Installing requirements: ' + str(requirements))
pip.main(pip_args)
pip.main(pip_args)
@classmethod
def _load_plugin_from_info(cls, info, root):
if not cls.validate_info(info):
logger.warn('The module info is not valid.\n\t{}'.format(info))
return None, None
return None, None
module = info["module"]
name = info["name"]
requirements = info.get("requirements", [])
......@@ -237,8 +241,8 @@ class Senpy(object):
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj))
)
" {}@{}").format(obj, inspect.getmodule(
obj)))
candidate = obj
break
if not candidate:
......@@ -248,7 +252,8 @@ class Senpy(object):
repo_path = root
module._repo = Repo(repo_path)
except InvalidGitRepositoryError:
logger.debug("The plugin {} is not in a Git repository".format(module))
logger.debug("The plugin {} is not in a Git repository".format(
module))
module._repo = None
except Exception as ex:
logger.error("Exception importing {}: {}".format(module, ex))
......@@ -265,7 +270,6 @@ class Senpy(object):
logger.debug("Info: {}".format(info))
return cls._load_plugin_from_info(info, root)
def _load_plugins(self):
plugins = {}
for search_folder in self._search_folders:
......@@ -293,8 +297,7 @@ class Senpy(object):
def matches(plug):
res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
logger.debug("matching {} with {}: {}".format(plug.name,
kwargs,
logger.debug("matching {} with {}: {}".format(plug.name, kwargs,
res))
return res
......@@ -305,5 +308,8 @@ class Senpy(object):
def sentiment_plugins(self):
""" Return only the sentiment plugins """
return {p: plugin for p, plugin in self.plugins.items() if
isinstance(plugin, SentimentPlugin)}
return {
p: plugin
for p, plugin in self.plugins.items()
if isinstance(plugin, SentimentPlugin)
}
senpy/models.py
View file @ 7fd69cc6
......@@ -18,15 +18,18 @@ import jsonschema
from flask import Response as FlaskResponse
DEFINITIONS_FILE = 'definitions.json'
CONTEXT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
CONTEXT_PATH = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
def get_schema_path(schema_file, absolute=False):
if absolute:
return os.path.realpath(schema_file)
else:
return os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', schema_file)
return os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'schemas',
schema_file)
def read_schema(schema_file, absolute=False):
......@@ -34,13 +37,13 @@ def read_schema(schema_file, absolute=False):
schema_uri = 'file://{}'.format(schema_path)
with open(schema_path) as f:
return jsonref.load(f, base_uri=schema_uri)
base_schema = read_schema(DEFINITIONS_FILE)
logging.debug(base_schema)
class Context(dict):
class Context(dict):
@staticmethod
def load(context):
logging.debug('Loading context: {}'.format(context))
......@@ -60,17 +63,16 @@ class Context(dict):
except IOError:
return context
else:
raise AttributeError('Please, provide a valid context')
raise AttributeError('Please, provide a valid context')
base_context = Context.load(CONTEXT_PATH)
class SenpyMixin(object):
context = base_context["@context"]
def flask(self,
in_headers=False,
headers=None,
**kwargs):
def flask(self, in_headers=False, headers=None, **kwargs):
"""
Return the values and error to be used in flask.
So far, it returns a fixed context. We should store/generate different
......@@ -87,33 +89,34 @@ class SenpyMixin(object):
'rel="http://www.w3.org/ns/json-ld#context";'
' type="application/ld+json"' % url)
})
return FlaskResponse(json.dumps(js, indent=2, sort_keys=True),
status=getattr(self, "status", 200),
headers=headers,
mimetype="application/json")
return FlaskResponse(
json.dumps(
js, indent=2, sort_keys=True),
status=getattr(self, "status", 200),
headers=headers,
mimetype="application/json")
def serializable(self):
def ser_or_down(item):
if hasattr(item, 'serializable'):
return item.serializable()
elif isinstance(item, dict):
temp = dict()
for kp in item:
vp = item[kp]
temp[kp] = ser_or_down(vp)
return temp
elif isinstance(item, list):
return list(ser_or_down(i) for i in item)
else:
return item
return ser_or_down(self._plain_dict())
if hasattr(item, 'serializable'):
return item.serializable()
elif isinstance(item, dict):
temp = dict()
for kp in item:
vp = item[kp]
temp[kp] = ser_or_down(vp)
return temp
elif isinstance(item, list):
return list(ser_or_down(i) for i in item)
else:
return item
return ser_or_down(self._plain_dict())
def jsonld(self, with_context=True, context_uri=None):
ser = self.serializable()
if with_context:
if with_context:
context = []
if context_uri:
context = context_uri
......@@ -133,10 +136,8 @@ class SenpyMixin(object):
ser["@context"] = context
return ser
def to_JSON(self, *args, **kwargs):
js = json.dumps(self.jsonld(*args, **kwargs), indent=4,
sort_keys=True)
js = json.dumps(self.jsonld(*args, **kwargs), indent=4, sort_keys=True)
return js
def validate(self, obj=None):
......@@ -145,18 +146,19 @@ class SenpyMixin(object):
if hasattr(obj, "jsonld"):
obj = obj.jsonld()
jsonschema.validate(obj, self.schema)
class SenpyModel(SenpyMixin, dict):
schema = base_schema
def __init__(self, *args, **kwargs):
self.id = kwargs.pop('id', '{}_{}'.format(type(self).__name__,
time.time()))
self.id = kwargs.pop('id', '{}_{}'.format(
type(self).__name__, time.time()))
temp = dict(*args, **kwargs)
for obj in [self.schema,]+self.schema.get('allOf', []):
for obj in [self.schema, ] + self.schema.get('allOf', []):
for k, v in obj.get('properties', {}).items():
if 'default' in v:
temp[k] = copy.deepcopy(v['default'])
......@@ -172,7 +174,6 @@ class SenpyModel(SenpyMixin, dict):
self.__dict__['context'] = Context.load(context)
super(SenpyModel, self).__init__(temp)
def _get_key(self, key):
key = key.replace("__", ":", 1)
return key
......@@ -180,7 +181,6 @@ class SenpyModel(SenpyMixin, dict):
def __setitem__(self, key, value):
dict.__setitem__(self, key, value)
def __delitem__(self, key):
dict.__delitem__(self, key)
......@@ -195,62 +195,80 @@ class SenpyModel(SenpyMixin, dict):
def __delattr__(self, key):
self.__delitem__(self._get_key(key))
def _plain_dict(self):
d = { k: v for (k,v) in self.items() if k[0] != "_"}
d = {k: v for (k, v) in self.items() if k[0] != "_"}
d["@id"] = d.pop('id')
return d
class Response(SenpyModel):
schema = read_schema('response.json')