Commit 9318436e authored by Alberto Pascual's avatar Alberto Pascual
Browse files

tutorials changed

parent 9292a6bb
......@@ -14,4 +14,6 @@ ADD run-cron.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/run-cron.sh
CMD ["sh","/usr/local/bin/run-cron.sh"]
\ No newline at end of file
ENTRYPOINT ["run-cron.sh"]
CMD ["10", "isis"]
\ No newline at end of file
......@@ -5,21 +5,31 @@ import sys
s = sched.scheduler(time.time, time.sleep)
def runevery():
if sys.argv[1] == "tutorial2":
def main(args):
print("Calling main with: ", args)
if len(args) > 0 and args[0] == "tutorial2":
print("Tutorial 2 STARTED!")
identifier = time.time()
command = 'python -m luigi --module tutorialtask CrawlerTask --url "{url}" --id "{id}" --num 10'.format(url="isis",id=identifier)
command = 'python -m luigi --module tutorialtask CrawlerTask --url "{url}" --id "{id}"'.format(url="isis",id=identifier)
subprocess.call(command.split(), shell= False)
elif sys.argv[1] == "tutorial3":
elif len(args) > 0 and args[0] == "tutorial3":
print("Tutorial 3 STARTED!")
identifier = time.time()
command = 'python -m luigi --module tutorialtask PipelineTask --index tutorial --doc-type news --url "{url}" --id "{id}" --num 10'.format(url="isis",id=identifier)
command = 'python -m luigi --module tutorialtask PipelineTask --index tutorial --doc-type news --url "{url}" --id "{id}"'.format(url="isis",id=identifier)
subprocess.call(command.split(), shell= False)
elif len(args) > 0 and args[0] == "cron":
print(args[1:])
s.enter(2, 1, cron, [args[1:]])
s.run()
else:
identifier = time.time()
command = 'python -m luigi --module analysistask PipelineTask --index gsicrawler --doc-type news --url "{url}" --id "{id}" --analysisType "sentiments,emotions" --num {num}'.format(url="isis",id=identifier, num=int(sys.argv[1]))
command = 'python -m luigi --module analysistask PipelineTask --index gsicrawler --doc-type news --url "{url}" --id "{id}" --analysisType "sentiments,emotions" --num {num}'.format(url=str(args[1]),id=identifier, num=int(args[0]))
subprocess.call(command.split(), shell= False)
s.enter(86400, 1, runevery) #Change here your cron time
s.enter(2, 1, runevery)
s.run()
\ No newline at end of file
def cron(arg):
main(arg)
s.enter(86400, 1, cron, arg) #Change here your cron time
main(sys.argv[1:])
from node:7.10.0
# Install gettext to get envsubst
RUN apt-get update && apt-get install -y gettext
ENV NODE_PATH=/tmp/node_modules APP_NAME=dashboard-gsicrawler
# Install dependencies first to use cache
RUN npm install -g http-server bower
WORKDIR /usr/src/app/
RUN mkdir -p /etc/ssl/certs
COPY lab.cluster.gsi.dit.upm.es.crt /usr/local/share/ca-certificates/
RUN update-ca-certificates
ADD bower.json /usr/src/app/bower.json
RUN bower link --allow-root
RUN bower install --allow-root && mv bower_components ..
ADD . /usr/src/app
CMD ["/usr/src/app/init.sh"]
{
"name": "dashboard-gsicrawler",
"authors": [
"gsi-upm"
],
"description": "Dashboard to visualize data from GSICrawler",
"main": "dashboard-gsicrawler.html",
"keywords": [
"dashboard",
"web-component",
"elasticsearch",
"polymer"
],
"dependencies": {
"material-search": "material-search#*",
"leaflet-maps": "leaflet-maps#2.1.3",
"paper-tabs": "paper-tabs#1.1.0",
"number-chart": "number-chart#^1.1.1",
"google-apis": "GoogleWebComponents/google-apis#^1.0.0",
"google-chart-elasticsearch": "google-chart-elasticsearch#^1.1.3",
"iron-icons": "PolymerElements/iron-icons#^1.0.0",
"paper-icon-button": "PolymerElements/paper-icon-button#^1.0.0",
"polymer": "Polymer/polymer#1.7.0",
"jquery": "jquery#^2.1.4",
"elasticsearchjs-import": "DigElements/elasticsearchjs-import#~1.0.0",
"elasticjs-import": "DigElements/elasticjs-import#~1.0.1",
"elastic-client": "DigElements/elastic-client#~1.0.1",
"webcomponentsjs": "< 1.0.0",
"tweet-chart": "tweet-chart#*",
"comment-chart": "comment-chart#*",
"paper-material": "paper-material#*",
"yasgui-polymer": "yasgui-polymer#*",
"paper-dialog":"paper-dialog#1.1.0",
"iron-pages": "PolymerElements/iron-pages#1.0.2",
"paper-item": "paper-item#1.0.2",
"paper-menu": "paper-menu#1.0.1",
"iron-a11y-keys-behavior": "iron-a11y-keys-behavior#1.1.9",
"entities-chart": "entities-chart#*",
"iron-collapse": "iron-collapse#1.0.2"
},
"resolutions": {
"webcomponentsjs": "0.7.24",
"elastic.js": "^1.2.0",
"polymer":"1.7.0"
},
"license": "Apache-2.0",
"homepage": "https://lab.cluster.gsi.dit.upm.es/sefarad/dashboard-gsicrawler"
}
/usr/src/bower_components
\ No newline at end of file
:root {
--paper-tab-ink: var(--accent-color);
--paper-tabs-selection-bar-color: var(--accent-color);
}
paper-material{
background: white;
}
google-chart {
margin: 0 auto;
}
.lists{
padding: 0px;
}
.chart_container {
width: 49%;
margin-bottom: 20px;
}
/* Tablet+ */
@media (min-width: 601px) {
paper-tabs, paper-toolbar {
}
}
paper-tabs, paper-toolbar {
background-color: #449DD5;
color: #fff;
box-shadow: 0px 3px 6px rgba(0, 0, 0, 0.2);
border-radius: 5px 5px 0 0;
}
paper-tabs[noink][no-bar] paper-tab.iron-selected {
color: #04419B;
}
#content, .announcement {
max-width: 790px;
background-color: #FFF;
padding: 1em;
margin: 0 auto;
box-shadow: 0 0 50px rgba(0, 0, 0, 0.2);
}
.announcement {
margin: 1em auto;
border: 2px solid #0A0;
}
h2 {
font-size: 1.8em;
margin: 1.5em 0 0.5em -2px;
font-weight: bold;
padding: .2em .5em;
background: #edeeef;
border-right: 1px solid #ccc;
border-bottom: 1px solid #ccc;
color: #fff;
background-color: #b0de5c;
background-image: -moz-linear-gradient(top, #b0de5c, #82cb00);
background-image: -o-linear-gradient(top, #b0de5c, #82cb00);
background-image: -webkit-gradient(linear, left top, left bottom, from(#b0de5c), to(#82cb00));
background-image: -webkit-linear-gradient(top, #b0de5c, #82cb00);
background-image: linear-gradient(top, #b0de5c, #82cb00);
filter: progid:DXImageTransform.Microsoft.gradient(startColorStr='#b0de5c', EndColorStr='#82cb00');
border-radius: 5px;
text-shadow: 0 -1px 1px rgba(0,0,0,0.35);
box-shadow: 2px 2px 3px rgba(0, 0, 0, 0.2);
}
.nav {
color: #000;
text-decoration: none;
border-bottom: 1px dashed #000;
display: inline-block;
padding: 3px;
}
.nav:hover {
background-color: #DDF;
}
leaflet-map,
geolocation-sample {
max-width: 100%;
height: 30em;
border: 1px solid #aaa;
}
geolocation-sample {
height: 21em;
}
pre {
font-size: large;
padding: 0.5em 1em;
background-color: #DDD;
clear: all;
}
.i {
color: #00D
}
.a {
color: #008;
}
.v {
color: #800;
}
.c {
color: #080;
}
iron-pages {
padding: 48px 62px 10px;
}
#barsentiment > paper-material{
height: 261px;
}
.top-bar{
width: 100%;
background-color: #449DD5 !important;
color: white;
text-align: center;
height: 35px;
line-height: 35px;
border-radius: 5px 5px 0 0;
}
@media screen and (min-width: 480px) and (max-width: 959px){
iron-pages {
padding: 0px;
}
}
<link rel="import" href="/bower_components/polymer/polymer.html">
<link rel="import" href="/bower_components/material-search/material-search.html">
<link rel="import" href="/bower_components/paper-material/paper-material.html">
<link rel="import" href="/elements/number-chart/number-chart.html">
<link rel="import" href="/elements/google-chart-elasticsearch/google-chart.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tab.html">
<link rel="import" href="/bower_components/comment-chart/comment-chart.html">
<link rel="import" href="/bower_components/yasgui-polymer/yasgui.html">
<link rel="import" href="/bower_components/iron-pages/iron-pages.html">
<link rel="import" href="/bower_components/paper-item/paper-item.html">
<link rel="import" href="/bower_components/paper-menu/paper-menu.html">
<link rel="import" href="/bower_components/leaflet-maps/leaflet-map.html">
<dom-module id="dashboard-gsicrawler">
<link rel="import" type="css" href="dashboard-gsicrawler.css" type="css">
<link rel="import" href="/styles/app-theme.html" type="css">
<template>
<iron-ajax auto
url="queries.json"
handle-as="json"
last-response="{{queries}}"></iron-ajax>
<div style="width:10%; padding: 1em; display:inline-block">
<img style="width: 100%;"src="/images/trivalent.jpg">
</div>
<div style="width:80%; padding-bottom: 12px; text-align:center; display:inline-block">
<img style="width: 30%; margin:0 auto;"src="/images/logo-gsi-crawler.png">
</div>
<paper-tabs selected="{{selected}}">
<paper-tab>Dashboard</paper-tab>
</paper-tabs>
<iron-pages selected="{{selected}}">
<div>
<div class="row">
<div class="col-md-12">
<google-chart
field="schema:datePublished"
data="{{data}}"
id='line-chart'
extra-id='line-chart'
type='area'
filters="{{filters}}"
icon='icons:trending-up'
options='{"title": "Temporal Trends"}'
cols='[{"label": "Date", "type": "string"},{"label": "Count", "type": "number"}]'
</google-chart>
</div>
</div>
<br>
<div class="row">
<div class="col-md-12">
<!-- YOUR NEW COMPONENTS GOES HERE -->
</div>
</div>
</div>
</iron-pages>
<div style="background-color:#ccc; padding: 1%;">
<div style="width:32%; margin: 0 auto">
<img height="60px" src="/images/gsi.png"> © 2017 Grupo de Sistemas Inteligentes GSI-UPM
</div>
</div>
</template>
<script>
var ready = false;
Polymer({
is: 'dashboard-gsicrawler',
properties: {
selected: {
type: Number,
value: 0
},
selectedpage: {
type: Number,
value: 0
},
query: {
type: String,
},
ids:{
type: Array
},
data:{
type: Object
},
client: {
type: Object,
notify: true,
observer: '_clientChanged'
},
fields: {
type: Array,
value: function() { return []; }
},
filters: {
type: Array,
notify: true,
value: function() { return []; }
}
},
observers: [
'_filtersChange(filters.*)'
],
behaviors: [
Polymer.IronA11yKeysBehavior
],
keyBindings: {
'enter': '_search'
},
ready: function(){
//console.log("ready");
},
_clientChanged: function() {
//console.log("ClientChanged");
ready = true;
this._query();
},
_filtersChange: function() {
this._query();
},
_query: function() {
//console.log("_query")
var that = this;
//console.log("Ready?: ", ready);
if(ready){
this.client.search({
// undocumented params are appended to the query string
index: "tutorial",
body: {
size: 200,
query: {
bool: {
must: this.filters,
}
},
sort:{'schema:datePublished':{order: "desc"}},
aggs: {
'schema:datePublished': {
date_histogram : {
field : "schema:datePublished",
format: "dd-MM-yyyy",
interval : "day"
}
}
}
}
}).then(function (resp) {
that.data = resp;
console.log(that.data);
});
}
}
});
</script>
</dom-module>
/usr/src/bower_components
\ No newline at end of file
`<google-chart>` is a web component designed to visualize data with Google Charts but obtaining data from an elasticSearch index.
### Usage
This web component accepts the following parameters:
```html
<google-chart
data="<!-- json with the ElasticSearch results -->"
field="<! -- field to visualize -->"
type='<!-- google chart type: column, pie, column, etc -->'
filters="{{filters}}"
icon='maps:my-location'
param='{{param}}'
options='{"title": "Places at different cities"}'
cols='[{"label": "Place", "type": "string"},{"label": "Count", "type": "number"}]' '<!-- visalized data in case of connection error -->'
</google-chart>
```
data object is obtained from elasticsearch. The query is made in the dashboard.
For more information visit: https://developers.google.com/chart/
### Installation
This web component is available in bower.
```bash
$ bower install google-chart-elasticsearch
```
This command will install it inside `bower_components` folder
Remember to edit your `elements.html` with this component.
\ No newline at end of file
{
"name": "google-chart-elasticsearch",
"authors": [
"gsi-upm"
],
"version": "1.1.2",
"description": "Modified Google chart web component to visualize data from elasticSearch",
"main": "google-chart.html",
"keywords": [
"chart",
"web-component",
"elasticsearch",
"polymer"
],
"dependencies": {
"iron-ajax": "PolymerElements/iron-ajax#^1.0.0",
"google-apis": "GoogleWebComponents/google-apis#^1.0.0",
"iron-icons": "PolymerElements/iron-icons#^1.0.0",
"paper-icon-button": "PolymerElements/paper-icon-button#^1.0.0",
"polymer": "Polymer/polymer#^1.1.0"
},
"license": "MIT",
"homepage": "https://lab.cluster.gsi.dit.upm.es/sefarad/google-chart-elasticsearch"
}
:host {
margin: 0;
padding: 0;
width: 400px;
height: 300px;
}
:host([type="gauge"]) {
width: 300px;
height: 300px;
}
#chartdiv {
width: 100%;
}
#nodata {
position: absolute;
width: 100%;
line-height: 35px;
text-align: center;
}
.top-bar{
width: 100%;
background-color: #449DD5;
border-radius: 5px 5px 0 0;
color: white;
text-align: center;
height: 35px;
line-height: 35px;
}
.top-bar span {
vertical-align: middle;
}
iron-icon {
margin-right: 8px;
width: 21px;
height: 21px;
}
paper-icon-button.green {
color: var(--paper-green-500);
}
paper-icon-button.green:hover {
background: var(--paper-green-50);
border-radius: 50%;
}
paper-material {
background-color: #ffffff
}
\ No newline at end of file
<link rel="import" href="/bower_components/polymer/polymer.html">
<link rel="import" href="/bower_components/iron-ajax/iron-ajax.html">
<link rel="import" href="/bower_components/google-apis/google-legacy-loader.html">
<link rel="import" href="/bower_components/iron-icons/iron-icons.html">
<link rel="import" href="/bower_components/iron-icons/maps-icons.html">
<link rel="import" href="/bower_components/paper-icon-button/paper-icon-button.html">
<!--
`google-chart` encapsulates Google Charts as a web component, allowing you to easily visualize
data. From simple line charts to complex hierarchical tree maps, the chart element provides a
number of ready-to-use chart types.
<google-chart
type='pie'
options='{"title": "Distribution of days in 2001Q1"}'
cols='[{"label":"Month", "type":"string"}, {"label":"Days", "type":"number"}]'
rows='[["Jan", 31],["Feb", 28],["Mar", 31]]'>
</google-chart>
Height and width are specified as style attributes:
google-chart {
height: 300px;
width: 50em;
}
Data can be provided in one of three ways:
- Via the `cols` and `rows` attributes:
cols='[{"label":"Mth", "type":"string"}, {"label":"Days", "type":"number"}]'
rows='[["Jan", 31],["Feb", 28],["Mar", 31]]'
- Via the `data` attribute, passing in the data directly:
data='[["Month", "Days"], ["Jan", 31], ["Feb", 28], ["Mar", 31]]'
- Via the `data` attribute, passing in the URL to a resource containing the
data, in JSON format:
data='http://example.com/chart-data.json'
@demo
-->
<dom-module id="google-chart">
<style is="custom-style">
@import url("../../styles/app-theme.html");
</style>
<link rel="import" type="css" href="google-chart.css">
<template>
<paper-material elevation="1">
<div class="top-bar">
<iron-icon icon="{{icon}}"></iron-icon>
<span>{{options.title}}</span>
</div>
<div id="nodata">No entries match the current filters</div>
<iron-ajax id="ajax" handle-as="json" url="{{data}}"
on-response="_externalDataLoaded"></iron-ajax>
<div id="chartdiv"></div>
<google-legacy-loader on-api-load="_readyForAction"></google-legacy-loader>