Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
sefarad
dashboard-gsicrawler
Commits
dc288e64
Commit
dc288e64
authored
Oct 09, 2017
by
Alberto Pascual
Browse files
this repo only has visualization now, name updated to dashboard-gsicrawler
parent
502e7167
Pipeline
#342
failed with stages
in 49 seconds
Changes
17
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Dockerfile
View file @
dc288e64
...
...
@@ -3,7 +3,7 @@ from node:7.10.0
# Install gettext to get envsubst
RUN
apt-get update
&&
apt-get
install
-y
gettext
ENV
NODE_PATH=/tmp/node_modules APP_NAME=dashboard-
reddit
ENV
NODE_PATH=/tmp/node_modules APP_NAME=dashboard-
gsicrawler
# Install dependencies first to use cache
RUN
npm
install
-g
http-server bower
...
...
Makefile
View file @
dc288e64
NAME
=
dashboard-
reddit
NAME
=
dashboard-
gsicrawler
DEVPORT
=
8080
...
...
README.md
View file @
dc288e64
`Sefarad dashboard for t
witter and reddit data
`
`Sefarad dashboard for t
rivalent crawler demo
`
## Usage
...
...
@@ -6,13 +6,13 @@ This web component accepts the following parameters:
```
html
<dashboard-
reddit
<dashboard-
gsicrawler
client=
"<!-- elasticsearch client -->"
>
</dashboard-
reddit
>
</dashboard-
gsicrawler
>
```
See the
`dashboard-
reddit
.html`
and
`demo/index.html`
for more information.
See the
`dashboard-
gsicrawler
.html`
and
`demo/index.html`
for more information.
## Installation
...
...
@@ -20,7 +20,7 @@ This web component is available in bower.
```
bash
$
bower
install
dashboard-
reddit
$
bower
install
dashboard-
gsicrawler
```
...
...
borrar-datos.sh
0 → 100755
View file @
dc288e64
git filter-branch
--force
--index-filter
\
"git rm -r --cached --ignore-unmatch
$@
"
\
--prune-empty
--tag-name-filter
cat
--
--all
bower.json
View file @
dc288e64
{
"name"
:
"dashboard-
reddit
"
,
"name"
:
"dashboard-
gsicrawler
"
,
"authors"
:
[
"gsi-upm"
],
"description"
:
"Dashboard to visualize data from
reddit API
"
,
"main"
:
"dashboard-
reddit
.html"
,
"description"
:
"Dashboard to visualize data from
GSICrawler
"
,
"main"
:
"dashboard-
gsicrawler
.html"
,
"keywords"
:
[
"dashboard"
,
"web-component"
,
...
...
@@ -44,5 +44,5 @@
"polymer"
:
"1.7.0"
},
"license"
:
"Apache-2.0"
,
"homepage"
:
"https://lab.cluster.gsi.dit.upm.es/sefarad/dashboard-
reddit
"
"homepage"
:
"https://lab.cluster.gsi.dit.upm.es/sefarad/dashboard-
gsicrawler
"
}
dasboard-reddit.html
deleted
100644 → 0
View file @
502e7167
<link
rel=
"import"
href=
"/bower_components/polymer/polymer.html"
>
<link
rel=
"import"
href=
"/bower_components/material-search/material-search.html"
>
<link
rel=
"import"
href=
"/bower_components/paper-material/paper-material.html"
>
<link
rel=
"import"
href=
"/bower_components/google-chart-elasticsearch/google-chart.html"
>
<link
rel=
"import"
href=
"/bower_components/paper-tabs/paper-tabs.html"
>
<link
rel=
"import"
href=
"/bower_components/paper-tabs/paper-tab.html"
>
<link
rel=
"import"
href=
"/bower_components/number-chart/number-chart.html"
>
<link
rel=
"import"
href=
"/bower_components/comment-chart/comment-chart.html"
>
<link
rel=
"import"
href=
"/bower_components/paper-tabs/paper-tabs.html"
>
<link
rel=
"import"
href=
"/bower_components/yasgui-polymer/yasgui.html"
>
<link
rel=
"import"
href=
"/bower_components/iron-pages/iron-pages.html"
>
<link
rel=
"import"
href=
"/bower_components/paper-item/paper-item.html"
>
<link
rel=
"import"
href=
"/bower_components/paper-menu/paper-menu.html"
>
<link
rel=
"import"
href=
"/elements/news-chart/news-chart.html"
>
<dom-module
id=
"dashboard-reddit"
>
<link
rel=
"import"
type=
"css"
href=
"dashboard-tourpedia.css"
type=
"css"
>
<link
rel=
"import"
href=
"/styles/app-theme.html"
type=
"css"
>
<template>
<iron-ajax
auto
url=
"queries.json"
handle-as=
"json"
last-response=
"{{queries}}"
></iron-ajax>
<paper-tabs
selected=
"{{selected}}"
>
<paper-tab>
Dashboard
</paper-tab>
<paper-tab>
Sparql Editor
</paper-tab>
</paper-tabs>
<iron-pages
selected=
"{{selected}}"
>
<div>
<material-search
search-value=
"{{query}}"
></material-search>
<br>
<div
style=
"width: 100%; display: inline-block"
>
<div
style=
"width: 32%; margin-right: 2%; float: left"
>
<number-chart
icon=
"info"
stylebg=
"bg-red"
data=
"{{data}}"
>
</number-chart>
</div>
<div
style=
"width: 32%; margin-right: 2%; float: left"
>
<number-chart
data=
"{{data}}"
object=
"cnn"
aggKey=
"author"
title=
"CNN"
icon=
"icons:receipt"
stylebg=
"bg-yellow"
>
</number-chart>
</div>
<div
style=
"width: 32%; float: left"
>
<number-chart
data=
"{{data}}"
object=
"The New York Times"
aggKey=
"author"
title=
"The New York Times"
icon=
"icons:receipt"
aggkey=
"location"
stylebg=
"bg-green"
>
</number-chart>
</div>
</div>
<div
style=
"width: 100%; display: inline-block"
>
<div
class=
"chart_container"
style=
"right: 0; float: right"
>
<google-chart
data=
"{{data}}"
field=
"about"
id=
'pie-chart4'
extra-id=
'pie-chart4'
type=
'pie'
filters=
"{{filters}}"
icon=
'icons:list'
param=
'{{param}}'
options=
'{"title": "Article Sections"}'
cols=
'[{"label": "articlesection", "type": "string"},{"label": "Count", "type": "number"}]'
</
google-chart
>
</div>
<div
class=
"chart_container"
style=
"left: 0; float: left"
>
<google-chart
field=
"author"
data=
"{{data}}"
id=
'pie-chart3'
extra-id=
'pie-chart3'
type=
'pie'
filters=
"{{filters}}"
icon=
'icons:reorder'
options=
'{"title": "Newspapers"}'
cols=
'[{"label": "type", "type": "string"},{"label": "Count", "type": "number"}]'
</
google-chart
>
</div>
</div>
<div
style=
"width: 100%"
>
<div
class=
"chart_container"
style=
"left: 0; float: left"
>
<google-chart
field=
"sentiment"
data=
"{{data}}"
id=
'pie-chart5'
extra-id=
'pie-chart5'
type=
'pie'
filters=
"{{filters}}"
icon=
'social:mood'
options=
'{"title": "Sentiments"}'
cols=
'[{"label": "Sentiment", "type": "string"},{"label": "Count", "type": "number"}]'
</
google-chart
>
</div>
<div
class=
"chart_container"
style=
"left: 0; float: right"
>
<google-chart
field=
"emotion"
data=
"{{data}}"
id=
'pie-chart5'
extra-id=
'pie-chart5'
type=
'pie'
filters=
"{{filters}}"
icon=
'social:sentiment-very-dissatisfied'
options=
'{"title": "Emotions"}'
cols=
'[{"label": "Sentiment", "type": "string"},{"label": "Count", "type": "number"}]'
</
google-chart
>
</div>
</div>
<div
class=
"chart_container"
style=
"left: 0; float: left; width: 100%"
>
<news-chart
datos =
"{{data}}"
title=
"List of news"
filters=
"{{filters}}"
icon=
'icons:receipt'
>
</news-chart>
</div>
</div>
</div>
<div>
<yasgui-ui
endpoint=
"http:///gsicrawler/query"
queries=
"{{queries}}"
>
</yasgui-ui>
</div>
</iron-pages>
</template>
<script>
var
ready
=
false
;
Polymer
({
is
:
'
dashboard-reddit
'
,
properties
:
{
selected
:
{
type
:
Number
,
value
:
0
},
ids
:{
type
:
Array
},
data
:{
type
:
Object
},
client
:
{
type
:
Object
,
notify
:
true
,
observer
:
'
_clientChanged
'
},
fields
:
{
type
:
Array
,
value
:
function
()
{
return
[];
}
},
filters
:
{
type
:
Array
,
notify
:
true
,
value
:
function
()
{
return
[];
}
}
},
observers
:
[
'
_filtersChange(filters.*)
'
],
ready
:
function
(){
console
.
log
(
"
ready
"
);
},
_clientChanged
:
function
()
{
console
.
log
(
"
ClientChanged
"
);
ready
=
true
;
this
.
_query
();
},
_filtersChange
:
function
()
{
this
.
_query
();
},
_query
:
function
()
{
console
.
log
(
"
_filtersChangedash
"
)
var
that
=
this
;
console
.
log
(
"
Ready?:
"
,
ready
);
if
(
ready
){
this
.
client
.
search
({
// undocumented params are appended to the query string
index
:
"
gsicrawler
"
,
body
:
{
size
:
10
,
query
:
{
bool
:
{
must
:
this
.
filters
,
}
},
aggs
:
{
type
:
{
terms
:
{
field
:
"
@type.keyword
"
,
order
:
{
_count
:
"
desc
"
}
}
},
author
:
{
terms
:
{
field
:
"
author.keyword
"
,
order
:
{
_count
:
"
desc
"
}
}
},
about
:
{
terms
:
{
field
:
"
about.keyword
"
,
order
:
{
_count
:
"
desc
"
}
}
},
sentiment
:
{
terms
:
{
field
:
"
sentiments.marl:hasPolarity.keyword
"
,
order
:
{
_count
:
"
desc
"
}
}
},
emotion
:
{
terms
:
{
field
:
"
emotions.onyx:hasEmotion.onyx:hasEmotionCategory.keyword
"
,
order
:
{
_count
:
"
desc
"
}
}
}
}
}
}).
then
(
function
(
resp
)
{
var
myids
=
[]
resp
.
hits
.
hits
.
forEach
(
function
(
entry
){
myids
.
push
(
entry
.
_id
)})
that
.
ids
=
myids
;
//console.log(that.ids)
that
.
data
=
resp
;
console
.
log
(
that
.
data
);
});
}
}
});
</script>
</dom-module>
dashboard-
tourpedia
.css
→
dashboard-
gsicrawler
.css
View file @
dc288e64
File moved
dashboard-
reddit
.html
→
dashboard-
gsicrawler.env
.html
View file @
dc288e64
...
...
@@ -17,10 +17,10 @@
<link
rel=
"import"
href=
"/elements/tweet-chart/tweet-chart.html"
>
<link
rel=
"import"
href=
"/elements/leaflet-maps/leaflet-map.html"
>
<dom-module
id=
"dashboard-
reddit
"
>
<dom-module
id=
"dashboard-
gsicrawler
"
>
<link
rel=
"import"
type=
"css"
href=
"dashboard-
tourpedia
.css"
type=
"css"
>
<link
rel=
"import"
type=
"css"
href=
"dashboard-
gsicrawler
.css"
type=
"css"
>
<link
rel=
"import"
href=
"/styles/app-theme.html"
type=
"css"
>
<template>
...
...
@@ -196,29 +196,13 @@
<span>
{{place.name}}
</span>
</leaflet-marker>
</template>
<!-- <leaflet-marker latitude="40.4520074" longitude="-3.7258213" title="Demo place">
<b>Name:</b>
<span>Demo<span>
</leaflet-marker>
<leaflet-marker class="style-scope dashboard-tourpedia" latitude="41.90059" longitude="12.474896" title="Margherita">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="52.547617" longitude="13.353173" title="Hotel Arcos Appartement House">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="52.357974741256" longitude="4.9006604030728" title="ibis Styles Amsterdam City">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="52.352212775572" longitude="4.8916602507234" title="easyHotel Amsterdam">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="52.532396478848" longitude="13.380556404591" title="baxpax Mitte Hostel">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="41.908899" longitude="12.479041" title="Edy">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="48.822176" longitude="2.361568" title="Long Hoa">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="48.827534" longitude="2.370932" title="Boulangerie de Tolbiac">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="41.386084311622" longitude="2.1677023172379" title="St Christopher's Inn">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="41.38985943707" longitude="2.1471921354532" title="Sunotel Junior">
</leaflet-marker>-->
</leaflet-map>
</div>
</div>
</div>
<div>
<yasgui-ui
endpoint=
"http:///krusti.gsi:13030/gsicrawler/query"
queries=
"{{queries}}"
>
endpoint=
"http:///${FUSEKI_ENDPOINT_DASHBOARD}/gsicrawler/query"
</
yasgui-ui
>
</div>
</iron-pages>
...
...
@@ -233,7 +217,7 @@
<script>
var
ready
=
false
;
Polymer
({
is
:
'
dashboard-
reddit
'
,
is
:
'
dashboard-
gsicrawler
'
,
properties
:
{
selected
:
{
type
:
Number
,
...
...
dashboard-
reddit.env
.html
→
dashboard-
gsicrawler
.html
View file @
dc288e64
...
...
@@ -17,10 +17,10 @@
<link
rel=
"import"
href=
"/elements/tweet-chart/tweet-chart.html"
>
<link
rel=
"import"
href=
"/elements/leaflet-maps/leaflet-map.html"
>
<dom-module
id=
"dashboard-
reddit
"
>
<dom-module
id=
"dashboard-
gsicrawler
"
>
<link
rel=
"import"
type=
"css"
href=
"dashboard-
tourpedia
.css"
type=
"css"
>
<link
rel=
"import"
type=
"css"
href=
"dashboard-
gsicrawler
.css"
type=
"css"
>
<link
rel=
"import"
href=
"/styles/app-theme.html"
type=
"css"
>
<template>
...
...
@@ -52,11 +52,7 @@
<number-chart
icon=
"/images/news.ico"
stylebg=
"bg-yellow"
<<<<<<<
HEAD
title=
"Selected elements"
=======
title=
"News selected"
>
>>>>>> 9575f09c4cfbf975c275bacc6dc603bc856e7992
data=
"{{data}}"
>
</number-chart>
</div>
...
...
@@ -200,33 +196,13 @@
<span>
{{place.name}}
</span>
</leaflet-marker>
</template>
<!-- <leaflet-marker latitude="40.4520074" longitude="-3.7258213" title="Demo place">
<b>Name:</b>
<span>Demo<span>
</leaflet-marker>
<leaflet-marker class="style-scope dashboard-tourpedia" latitude="41.90059" longitude="12.474896" title="Margherita">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="52.547617" longitude="13.353173" title="Hotel Arcos Appartement House">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="52.357974741256" longitude="4.9006604030728" title="ibis Styles Amsterdam City">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="52.352212775572" longitude="4.8916602507234" title="easyHotel Amsterdam">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="52.532396478848" longitude="13.380556404591" title="baxpax Mitte Hostel">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="41.908899" longitude="12.479041" title="Edy">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="48.822176" longitude="2.361568" title="Long Hoa">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="48.827534" longitude="2.370932" title="Boulangerie de Tolbiac">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="41.386084311622" longitude="2.1677023172379" title="St Christopher's Inn">
</leaflet-marker><leaflet-marker class="style-scope dashboard-tourpedia" latitude="41.38985943707" longitude="2.1471921354532" title="Sunotel Junior">
</leaflet-marker>-->
</leaflet-map>
</div>
</div>
</div>
<div>
<yasgui-ui
<<<<<<<
HEAD
endpoint=
"http:///${FUSEKI_ENDPOINT_DASHBOARD}/gsicrawler/query"
=======
endpoint=
"http:///krusti.gsi:13030/gsicrawler/query"
>
>>>>>> 9575f09c4cfbf975c275bacc6dc603bc856e7992
queries="{{queries}}">
endpoint=
"http:///sefarad-fuseki.cluster.gsi.dit.upm.es/gsicrawler/query"
</
yasgui-ui
>
</div>
</iron-pages>
...
...
@@ -241,7 +217,7 @@
<script>
var
ready
=
false
;
Polymer
({
is
:
'
dashboard-
reddit
'
,
is
:
'
dashboard-
gsicrawler
'
,
properties
:
{
selected
:
{
type
:
Number
,
...
...
demo/imports.html
View file @
dc288e64
<link
rel=
"import"
href=
"/bower_components/polymer/polymer.html"
>
<link
rel=
"import"
href=
"/bower_components/elastic-client/elastic-client.html"
>
<link
rel=
"import"
href=
"/bower_components/dashboard-
reddit
/dashboard-
reddit
.html"
>
<link
rel=
"import"
href=
"/bower_components/dashboard-
gsicrawler
/dashboard-
gsicrawler
.html"
>
demo/index.env.html
View file @
dc288e64
...
...
@@ -23,8 +23,8 @@
cluster-status=
"{{myStatus}}"
>
</elastic-client>
<dashboard-
reddit
client=
"{{client}}"
></dashboard-
reddit
>
<dashboard-
gsicrawler
client=
"{{client}}"
></dashboard-
gsicrawler
>
<!-- <button id="databutton" onclick="changedata()">Click to change data</button> -->
<script>
var
datas
=
[
...
...
demo/index.html
View file @
dc288e64
...
...
@@ -18,13 +18,13 @@
<template
is=
"dom-bind"
>
<elastic-client
config=
'{"host": "
kr
ust
i
.gsi.dit.upm.es
:19200
"}'
config=
'{"host": "
sefarad-elasticsearch.cl
ust
er
.gsi.dit.upm.es"}'
client=
"{{client}}"
cluster-status=
"{{myStatus}}"
>
</elastic-client>
<dashboard-
reddit
client=
"{{client}}"
></dashboard-
reddit
>
<dashboard-
gsicrawler
client=
"{{client}}"
></dashboard-
gsicrawler
>
<!-- <button id="databutton" onclick="changedata()">Click to change data</button> -->
<script>
var
datas
=
[
...
...
docker-compose.yml
View file @
dc288e64
...
...
@@ -3,7 +3,7 @@ version: '2'
services
:
sefarad
:
build
:
.
image
:
registry.cluster.gsi.dit.upm.es/sefarad/dashboard-
reddit
image
:
registry.cluster.gsi.dit.upm.es/sefarad/dashboard-
gsicrawler
ports
:
-
"
8080:8080"
volumes
:
...
...
@@ -14,62 +14,6 @@ services:
networks
:
-
sefarad-network
elasticsearch
:
image
:
"
docker.elastic.co/elasticsearch/elasticsearch:5.5.2"
ulimits
:
memlock
:
soft
:
-1
hard
:
-1
environment
:
-
cluster.name=docker-cluster
-
bootstrap.memory_lock=false
-
"
ES_JAVA_OPTS=-Xms512m
-Xmx512m"
-
"
xpack.security.enabled=false"
-
"
http.cors.enabled=true"
-
'
http.cors.allow-origin=*'
volumes
:
-
esdata:/usr/share/elasticsearch/data/
ports
:
-
19200:9200
-
19300:9300
networks
:
-
sefarad-network
fuseki
:
image
:
stain/jena-fuseki
ports
:
-
"
13030:3030"
volumes
:
-
./fuseki:/fuseki
environment
:
-
ADMIN_PASSWORD=${FUSEKI_PASSWORD}
networks
:
-
sefarad-network
luigi
:
build
:
./luigi/
image
:
registry.cluster.gsi.dit.upm.es/sefarad/gsicrawler/luigi
volumes
:
-
./luigi:/usr/src/app
ports
:
-
"
8082:8082"
networks
:
-
sefarad-network
environment
:
-
ES_ENDPOINT=${ES_ENDPOINT}
-
ES_PORT=${ES_PORT}
-
FUSEKI_ENDPOINT_EXTERNAL=${FUSEKI_ENDPOINT_EXTERNAL}
-
API_KEY_MEANING_CLOUD=${API_KEY_MEANING_CLOUD}
-
TWITTER_CONSUMER_KEY=${TWITTER_CONSUMER_KEY}
-
TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET}
-
TWITTER_ACCESS_TOKEN=${TWITTER_ACCESS_TOKEN}
-
TWITTER_ACCESS_TOKEN_SECRET=${TWITTER_ACCESS_TOKEN_SECRET}
depends_on
:
-
elasticsearch
networks
:
sefarad-network
:
driver
:
bridge
...
...
init.sh
View file @
dc288e64
#!/bin/sh
# Copy bower dependencies when using -v $PWD/:/usr/src/app
envsubst < /usr/src/app/dashboard-gsicrawler.env.html
>
/usr/src/app/dashboard-gsicrawler.html
||
exit
1
;
if
[
-f
/.dockerenv
]
;
then
for
i
in
"/usr/src/app"
"/usr/src/app/demo"
;
do
if
[
!
-L
"
$i
/bower_components"
]
&&
[
-d
"
$i
/bower_components"
]
;
then
...
...
@@ -16,5 +18,4 @@ cd demo
bower
link
$APP_NAME
--allow-root
envsubst < /usr/src/app/demo/index.env.html
>
/usr/src/app/demo/index.html
||
exit
1
;
envsubst < /usr/src/app/dashboard-reddit.env.html
>
/usr/src/app/dashboard-reddit.html
||
exit
1
;
http-server
.
k8s/deployment.yaml
deleted
100644 → 0
View file @
502e7167
---
apiVersion
:
v1
kind
:
ConfigMap
metadata
:
name
:
${NAME}-config
data
:
ES_ENDPOINT_EXTERNAL
:
"
$ES_ENDPOINT_EXTERNAL"
ES_ENDPOINT
:
$ES_ENDPOINT
ES_PORT
:
"
$ES_PORT"
---
apiVersion
:
extensions/v1beta1
kind
:
Deployment
metadata
:
name
:
${NAME}-deploy
spec
:
replicas
:
1
template
:
metadata
:
labels
:
role
:
${NAME}-web
app
:
${NAME}
spec
:
containers
:
-
name
:
${NAME}
image
:
registry.cluster.gsi.dit.upm.es/sefarad/dashboard-reddit:latest
imagePullPolicy
:
Always
resources
:
limits
:
memory
:
"
512Mi"
cpu
:
"
500m"
ports
:
-
name
:
web
containerPort
:
8080
envFrom
:
-
configMapRef
:
name
:
${NAME}-config
k8s/gsicrawler-ingress.yaml
deleted
100644 → 0
View file @
502e7167
---