Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
sefarad
dashboard-gsicrawler
Commits
b50585e1
Commit
b50585e1
authored
Sep 06, 2017
by
J. Fernando Sánchez
Browse files
k8s deployment
parent
bebdaa05
Changes
20
Hide whitespace changes
Inline
Side-by-side
.gitignore
View file @
b50585e1
...
...
@@ -2,3 +2,4 @@
.*
*~
bower_components
elasticsearch
.gitmodules
View file @
b50585e1
[submodule "
bower_components/
comment-chart"]
path =
bower_compon
ents/comment-chart
[submodule "comment-chart"]
path =
elem
ents/comment-chart
url = ../comment-chart
Dockerfile
View file @
b50585e1
from
node:7.10.0
# Install gettext to get envsubst
RUN
apt-get update
&&
apt-get
install
-y
gettext
ENV
NODE_PATH=/tmp/node_modules APP_NAME=dashboard-reddit
# Install dependencies first to use cache
RUN
npm
install
-g
http-server bower
ADD
bower.json /usr/src/bower.json
RUN
cd
/usr/src
&&
\
bower
install
--allow-root
WORKDIR
/usr/src/app/
ADD
elements elements
ADD
bower.json /usr/src/app/bower.json
RUN
bower
link
--allow-root
RUN
bower
install
--allow-root
&&
mv
bower_components ..
ADD
. /usr/src/app
WORKDIR
/usr/src/app/
CMD
["/usr/src/app/init.sh"]
Makefile
0 → 100644
View file @
b50585e1
NAME
=
dashboard-reddit
DEVPORT
=
8080
# Deployment with Kubernetes
# KUBE_CA_PEM_FILE is the path of a certificate file. It automatically set by GitLab
# if you enable Kubernetes integration in a project.
#
# As of this writing, Kubernetes integration can not be set on a group level, so it has to
# be manually set in every project.
# Alternatively, we use a custom KUBE_CA_BUNDLE environment variable, which can be set at
# the group level. In this case, the variable contains the whole content of the certificate,
# which we dump to a temporary file
#
# Check if the KUBE_CA_PEM_FILE exists. Otherwise, create it from KUBE_CA_BUNDLE
KUBE_CA_TEMP
=
false
ifeq
($(wildcard $(KUBE_CA_PEM_FILE)),)
KUBE_CA_PEM_FILE
:=
"
$$
PWD/.ca.crt"
CREATED
:=
$(
shell
echo
-e
"
$$
KUBE_CA_BUNDLE"
>
$(KUBE_CA_PEM_FILE)
)
endif
KUBE_URL
=
""
KUBE_TOKEN
=
""
KUBE_NAMESPACE
=
$(NAME)
KUBECTL
=
docker run
--rm
-v
$(KUBE_CA_PEM_FILE)
:/tmp/ca.pem
-i
lachlanevenson/k8s-kubectl
--server
=
"
$(KUBE_URL)
"
--token
=
"
$(KUBE_TOKEN)
"
--certificate-authority
=
"/tmp/ca.pem"
-n
$(KUBE_NAMESPACE)
CI_PROJECT_NAME
=
$(NAME)
CI_REGISTRY
=
docker.io
CI_REGISTRY_USER
=
gitlab
CI_COMMIT_REF_NAME
=
master
help
:
##
Show this help.
@
fgrep
-h
"##"
$(MAKEFILE_LIST)
| fgrep
-v
fgrep |
sed
-e
's/\\$$//'
|
sed
-e
's/\(.*:\)\s*##\s*\(.*\)/\1\t\t\2\n/'
config
:
##
Load config from the environment. You should run it once in every session before other tasks. Run: eval $(make config)
@
echo
". ../.env || true;"
@
awk
'{ print "export " $$0}'
.env
@
echo
"# Please, run: "
@
echo
"# eval
\$
$
(make config)"
# If you need to run a command on the key/value pairs, use this:
# @awk '{ split($$0, a, "="); "echo " a[2] " | base64 -w 0" |& getline b64; print "export " a[1] "=" a[2]; print "export " a[1] "_BASE64=" b64}' .env
deploy
:
##
Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN
@
cat
k8s/
*
| envsubst |
$(KUBECTL)
apply
-f
-
deploy-check
:
##
Get the deployed configuration.
@
$(KUBECTL)
get deploy,pods,svc,ingress
login
:
##
Log in to the registry. It will only be used in the server
,
or when running a CI task locally (if CI_BUILD_TOKEN is set).
ifeq
($(CI_BUILD_TOKEN),)
@
echo
"Not logging in to the docker registry"
"
$(CI_REGISTRY)
"
else
docker login
-u
gitlab-ci-token
-p
$(CI_BUILD_TOKEN)
$(CI_REGISTRY)
endif
info
:
##
Print variables. Useful for debugging.
@
echo
"#KUBERNETES"
@
echo
KUBE_URL
=
$(KUBE_URL)
@
echo
KUBE_CA_PEM_FILE
=
$(KUBE_CA_PEM_FILE)
@
echo
KUBE_CA_BUNDLE
=
$$
KUBE_CA_BUNDLE
@
echo
KUBE_TOKEN
=
$(KUBE_TOKEN)
@
echo
KUBE_NAMESPACE
=
$(KUBE_NAMESPACE)
@
echo
KUBECTL
=
$(KUBECTL)
@echo
"#CI"
@echo
CI_PROJECT_NAME
=
$(CI_PROJECT_NAME)
@echo
CI_REGISTRY
=
$(CI_REGISTRY)
@echo
CI_REGISTRY_USER
=
$(CI_REGISTRY_USER)
@echo
CI_COMMIT_REF_NAME
=
$(CI_COMMIT_REF_NAME)
#
# For local development
#
run
:
docker-compose up
-d
stop
:
docker-compose stop
build
:
##
Build all docker images
docker-compose build
push
:
##
Push docker all built docker images to the registry
docker-compose push
build-%
:
##
Build a specific image. For example
,
to build the 'web' image: make build-web
docker-compose build
$*
push-%
:
##
Push a specific image to the repository. For example
,
to push the 'web' image: make push-web
docker-compose build
$*
ci
:
##
Run a task locally like GitLab will run it in the server. For example: make -e action=build ci
gitlab-runner
exec
shell
${action}
.PHONY
:
deploy info build push
help
deploy-check push-% build-% ci
bower.json
View file @
b50585e1
...
...
@@ -18,7 +18,7 @@
"reviews-table"
:
"reviews-table#*"
,
"number-chart"
:
"number-chart#^1.1.1"
,
"google-apis"
:
"GoogleWebComponents/google-apis#^1.0.0"
,
"google-chart-elasticsearch"
:
"google-chart-elasticsearch#^1.1.
1
"
,
"google-chart-elasticsearch"
:
"google-chart-elasticsearch#^1.1.
3
"
,
"iron-icons"
:
"PolymerElements/iron-icons#^1.0.0"
,
"paper-icon-button"
:
"PolymerElements/paper-icon-button#^1.0.0"
,
"polymer"
:
"Polymer/polymer#^1.1.0"
,
...
...
@@ -28,7 +28,8 @@
"elastic-client"
:
"DigElements/elastic-client#~1.0.1"
,
"webcomponentsjs"
:
"< 1.0.0"
,
"spider-chart"
:
"spider-chart#*"
,
"tweet-chart"
:
"tweet-chart#*"
"tweet-chart"
:
"tweet-chart#*"
,
"comment-chart"
:
"./elements/comment-chart"
},
"resolutions"
:
{
"webcomponentsjs"
:
"0.7.24"
,
...
...
dashboard-reddit.html
View file @
b50585e1
...
...
@@ -21,16 +21,6 @@
url=
"queries.json"
handle-as=
"json"
last-response=
"{{queries}}"
></iron-ajax>
<iron-ajax
auto
url=
"/endpoint.json"
handle-as=
"json"
last-response=
"{{endpoint}}"
></iron-ajax>
<!-- <template is="dom-repeat" items="{{ids}}" as="id">
<iron-ajax auto
url="{{getName(id)}}"
handle-as="json"
on-response="addPlace"></iron-ajax>
</template> -->
<paper-tabs
selected=
"{{selected}}"
>
<paper-tab>
Dashboard
</paper-tab>
...
...
@@ -84,12 +74,12 @@
icon=
'icons:list'
param=
'{{param}}'
options=
'{"title": "Comments Subreddits"}'
cols=
'[{"label": "subreddit
Name
", "type": "string"},{"label": "Count", "type": "number"}]'
cols=
'[{"label": "subreddit", "type": "string"},{"label": "Count", "type": "number"}]'
</
google-chart
>
</div>
<div
class=
"chart_container"
style=
"left: 0; float: left"
>
<google-chart
field=
"subreddit
Name
"
field=
"subreddit"
data=
"{{data}}"
id=
'pie-chart3'
extra-id=
'pie-chart3'
...
...
@@ -239,14 +229,6 @@
}
},
aggs
:
{
subredditName
:
{
terms
:
{
field
:
"
subredditName
"
,
order
:
{
_count
:
"
desc
"
}
}
},
type
:
{
terms
:
{
field
:
"
_type
"
,
...
...
@@ -271,16 +253,15 @@
}
}
},
// ,
//emotion: {
// terms: {
// field: "emotion",
// order: {
// _count: "desc"
// }
// }
// }
}
emotion
:
{
terms
:
{
field
:
"
emotion
"
,
order
:
{
_count
:
"
desc
"
}
}
}
}
}
}).
then
(
function
(
resp
)
{
var
myids
=
[]
...
...
@@ -288,7 +269,7 @@
that
.
ids
=
myids
;
//console.log(that.ids)
that
.
data
=
resp
;
//
console.log(that.data);
console
.
log
(
that
.
data
);
});
}
...
...
demo/imports.html
View file @
b50585e1
<link
rel=
"import"
href=
"
../
bower_components/polymer/polymer.html"
>
<link
rel=
"import"
href=
"bower_components/polymer/polymer.html"
>
<link
rel=
"import"
href=
"
../
bower_components/elastic-client/elastic-client.html"
>
<link
rel=
"import"
href=
"
../
bower_components/dashboard-reddit/dashboard-reddit.html"
>
<link
rel=
"import"
href=
"bower_components/elastic-client/elastic-client.html"
>
<link
rel=
"import"
href=
"bower_components/dashboard-reddit/dashboard-reddit.html"
>
demo/index.env.html
0 → 100644
View file @
b50585e1
<html>
<head>
<script
src=
"bower_components/webcomponentsjs/webcomponents-lite.js"
></script>
<link
rel=
"import"
href=
"imports.html"
></link>
<style>
.info-box-icon.number-chart
iron-icon
.number-chart
{
top
:
26%
;
}
</style>
</head>
<body>
<template
is=
"dom-bind"
>
<elastic-client
config=
'{"host": "${ES_ENDPOINT_EXTERNAL}"}'
client=
"{{client}}"
cluster-status=
"{{myStatus}}"
>
</elastic-client>
<dashboard-reddit
client=
"{{client}}"
></dashboard-reddit>
<!-- <button id="databutton" onclick="changedata()">Click to change data</button> -->
<script>
var
datas
=
[
{
"
hits
"
:
{
"
total
"
:
20000
},
"
aggregations
"
:
{
"
category
"
:
{
"
buckets
"
:
[
{
"
key
"
:
"
myObject
"
,
"
doc_count
"
:
3000
},
{
"
key
"
:
"
otherObject
"
,
"
doc_count
"
:
1000
}
]
}
}
},
{
"
hits
"
:
{
"
total
"
:
30000
},
"
aggregations
"
:
{
"
category
"
:
{
"
buckets
"
:
[
{
"
key
"
:
"
myObject
"
,
"
doc_count
"
:
1000
},
{
"
key
"
:
"
otherObject
"
,
"
doc_count
"
:
4000
}
]
}
}
}];
var
numdata
=
0
;
<!--
var
nc1
=
document
.
getElementById
(
'
demo-chart1
'
);
-->
<!--
var
nc2
=
document
.
getElementById
(
'
demo-chart2
'
);
-->
<!--
nc1
.
data
=
nc2
.
data
=
datas
[
0
];
-->
<!--
function
changedata
(){
-->
<!--
numdata
+=
1
;
-->
<!--
nc1
.
data
=
nc2
.
data
=
datas
[
numdata
%
2
];
-->
<!--
}
-->
</script>
</template>
</body>
</html>
demo/index.html
View file @
b50585e1
<html>
<head>
<script
src=
"
../
bower_components/webcomponentsjs/webcomponents-lite.js"
></script>
<script
src=
"bower_components/webcomponentsjs/webcomponents-lite.js"
></script>
<link
rel=
"import"
href=
"imports.html"
></link>
<style>
.info-box-icon.number-chart
iron-icon
.number-chart
{
...
...
@@ -14,7 +14,7 @@
<template
is=
"dom-bind"
>
<elastic-client
config=
'{"host": "
http://
localhost:9200"}'
config=
'{"host": "localhost:9200"}'
client=
"{{client}}"
cluster-status=
"{{myStatus}}"
>
</elastic-client>
...
...
docker-compose.yml
View file @
b50585e1
...
...
@@ -3,33 +3,28 @@ version: '2'
services
:
sefarad
:
build
:
.
image
:
registry.cluster.gsi.dit.upm.es/sefarad/dashboard-reddit
ports
:
-
"
8080:8080"
volumes
:
-
.:/usr/src/app
networks
:
-
sefarad-network
depends_on
:
-
elasticsearch
elasticsearch
:
image
:
elasticsearch
ports
:
-
"
9200:9200"
-
"
9300:9300"
image
:
"
docker.elastic.co/elasticsearch/elasticsearch:5.5.2"
ulimits
:
memlock
:
soft
:
-1
hard
:
-1
environment
:
-
cluster.name=docker-cluster
-
bootstrap.memory_lock=false
-
"
ES_JAVA_OPTS=-Xms512m
-Xmx512m"
-
"
xpack.security.enabled=false"
-
"
http.cors.enabled=true"
-
'
http.cors.allow-origin=*'
volumes
:
-
./elasticsearch/nodes:/usr/share/elasticsearch/data/nodes
-
./elasticsearch/config:/usr/share/elasticsearch/config
networks
:
-
sefarad-network
luigi
:
build
:
context
:
luigi/
volumes
:
-
./luigi:/usr/src/app
networks
:
-
sefarad-network
-
esdata:/usr/share/elasticsearch/data/
networks
:
sefarad-network
:
driver
:
bridge
\ No newline at end of file
ports
:
-
9200:9200
volumes
:
esdata
:
elasticsearch/config/elasticsearch.yml
deleted
100644 → 0
View file @
bebdaa05
# ======================== Elasticsearch Configuration =========================
#
# NOTE: Elasticsearch comes with reasonable defaults for most settings.
# Before you set out to tweak and tune the configuration, make sure you
# understand what are you trying to accomplish and the consequences.
#
# The primary way of configuring a node is via this file. This template lists
# the most important settings you may want to configure for a production cluster.
#
# Please see the documentation for further information on configuration options:
# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/settings.html>
#
# ---------------------------------- Cluster -----------------------------------
#
# Use a descriptive name for your cluster:
#
cluster.name
:
sefarad
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
#node.name: node-1
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#
#path.data: /path/to/data
#
# Path to log files:
#
#path.logs: /path/to/logs
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
#
#network.host: 192.168.0.1
#
# Set a custom port for HTTP:
#
#http.port: 9200
#
# For more information, see the documentation at:
# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/modules-network.html>
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when new node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
#
#discovery.zen.ping.unicast.hosts: ["host1", "host2"]
#
# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
#
#discovery.zen.minimum_master_nodes: 3
#
# For more information, see the documentation at:
# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/modules-discovery-zen.html>
#
# ---------------------------------- Gateway -----------------------------------
#
# Block initial recovery after a full cluster restart until N nodes are started:
#
#gateway.recover_after_nodes: 3
#
# For more information, see the documentation at:
# <https://www.elastic.co/guide/en/elasticsearch/reference/5.0/modules-gateway.html>
#
# ---------------------------------- Various -----------------------------------
#
# Require explicit names when deleting indices:
#
#action.destructive_requires_name: true
http.host
:
0.0.0.0
http.cors.enabled
:
true
http.cors.allow-origin
:
"
*"
http.cors.allow-methods
:
OPTIONS, HEAD, GET, POST, PUT, DELETE
http.cors.allow-headers
:
X-Requested-With, X-Auth-Token, Content-Type, Content-Length
elasticsearch/config/jvm.options
deleted
100644 → 0
View file @
bebdaa05
## JVM configuration
################################################################
## IMPORTANT: JVM heap size
################################################################
##
## You should always set the min and max JVM heap
## size to the same value. For example, to set
## the heap to 4 GB, set:
##
## -Xms4g
## -Xmx4g
##
## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
## for more information
##
################################################################
# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space
-Xms2g
-Xmx2g
################################################################
## Expert settings
################################################################
##
## All settings below this section are considered
## expert settings. Don't tamper with them unless
## you understand what you are doing
##
################################################################
## GC configuration
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
## optimizations
# disable calls to System#gc
-XX:+DisableExplicitGC
# pre-touch memory pages used by the JVM during initialization
-XX:+AlwaysPreTouch
## basic
# force the server VM
-server
# set to headless, just in case
-Djava.awt.headless=true
# ensure UTF-8 encoding by default (e.g. filenames)
-Dfile.encoding=UTF-8
# use our provided JNA always versus the system one
-Djna.nosys=true
# flags to keep Netty from being unsafe
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
# log4j 2
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
## heap dumps
# generate a heap dump when an allocation from the Java heap fails
# heap dumps are created in the working directory of the JVM
-XX:+HeapDumpOnOutOfMemoryError
# specify an alternative path for heap dumps
# ensure the directory exists and has sufficient space
#-XX:HeapDumpPath=${heap.dump.path}
## GC logging
#-XX:+PrintGCDetails
#-XX:+PrintGCTimeStamps
#-XX:+PrintGCDateStamps
#-XX:+PrintClassHistogram
#-XX:+PrintTenuringDistribution
#-XX:+PrintGCApplicationStoppedTime
# log GC status to a file with time stamps
# ensure the directory exists
#-Xloggc:${loggc}
# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
# If documents were already indexed with unquoted fields in a previous version
# of Elasticsearch, some operations may throw errors.
#
# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
# only for migration purposes.
#-Delasticsearch.json.allow_unquoted_field_names=true
elasticsearch/config/log4j2.properties
deleted
100644 → 0
View file @
bebdaa05
status
=
error
# log action execution errors for easier debugging
logger.action.name
=
org.elasticsearch.action
logger.action.level
=
debug
appender.console.type
=
Console
appender.console.name
=
console
appender.console.layout.type
=
PatternLayout
appender.console.layout.pattern
=
[%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
appender.rolling.type
=
RollingFile
appender.rolling.name
=
rolling
appender.rolling.fileName
=
${sys:es.logs}.log
appender.rolling.layout.type
=
PatternLayout
appender.rolling.layout.pattern
=
[%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n
appender.rolling.filePattern
=
${sys:es.logs}-%d{yyyy-MM-dd}.log
appender.rolling.policies.type
=
Policies
appender.rolling.policies.time.type
=
TimeBasedTriggeringPolicy
appender.rolling.policies.time.interval
=
1
appender.rolling.policies.time.modulate
=
true
rootLogger.level
=
info
rootLogger.appenderRef.console.ref
=
console
rootLogger.appenderRef.rolling.ref
=
rolling
appender.deprecation_rolling.type
=
RollingFile
appender.deprecation_rolling.name
=
deprecation_rolling
appender.deprecation_rolling.fileName
=
${sys:es.logs}_deprecation.log
appender.deprecation_rolling.layout.type
=
PatternLayout
appender.deprecation_rolling.layout.pattern
=
[%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n
appender.deprecation_rolling.filePattern
=
${sys:es.logs}_deprecation-%i.log.gz
appender.deprecation_rolling.policies.type
=
Policies
appender.deprecation_rolling.policies.size.type
=
SizeBasedTriggeringPolicy
appender.deprecation_rolling.policies.size.size
=
1GB
appender.deprecation_rolling.strategy.type
=
DefaultRolloverStrategy
appender.deprecation_rolling.strategy.max
=
4
logger.deprecation.name
=
org.elasticsearch.deprecation
logger.deprecation.level
=
warn
logger.deprecation.appenderRef.deprecation_rolling.ref
=
deprecation_rolling
logger.deprecation.additivity
=
false
appender.index_search_slowlog_rolling.type
=
RollingFile
appender.index_search_slowlog_rolling.name
=
index_search_slowlog_rolling
appender.index_search_slowlog_rolling.fileName
=
${sys:es.logs}_index_search_slowlog.log
appender.index_search_slowlog_rolling.layout.type
=
PatternLayout
appender.index_search_slowlog_rolling.layout.pattern
=
[%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n
appender.index_search_slowlog_rolling.filePattern
=
${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
appender.index_search_slowlog_rolling.policies.type
=
Policies
appender.index_search_slowlog_rolling.policies.time.type
=
TimeBasedTriggeringPolicy
appender.index_search_slowlog_rolling.policies.time.interval
=
1
appender.index_search_slowlog_rolling.policies.time.modulate
=