Commit 1fcee3db authored by Alberto Pascual's avatar Alberto Pascual
Browse files

timeline with more data, twitter scrapper fixes

parent 91515425
Pipeline #440 passed with stages
in 41 seconds
......@@ -154,7 +154,7 @@
type='area'
filters="{{filters}}"
icon='icons:trending-up'
optionsbi='{"colors": ["#00C0EF", "#DD4B39", "#9B11F1", "#F27311"], "explorer": { "actions": ["dragToZoom", "rightClickToReset"], "axis": "horizontal", "keepInBounds": true, "maxZoomIn": 4.0}}'
optionsbi='{"colors": ["#00C0EF", "#DD4B39", "#9B11F1", "#F27311"], "explorer": { "actions": ["dragToZoom", "rightClickToReset"], "axis": "horizontal", "keepInBounds": true, "maxZoomIn": 10.0}}'
options='{"title": "Temporal Trends"}'
cols='[{"label": "Date", "type": "date"},{"label": "PP", "type": "number"}, {"label": "PSOE", "type": "number"}, {"label": "Podemos", "type": "number"}, {"label": "Ciudadanos", "type": "number"}]'
</google-chart>
......@@ -355,7 +355,7 @@
// undocumented params are appended to the query string
index: "somedi",
body: {
size: 200,
size: 500,
query: {
multi_match:{
query: this.query,
......@@ -435,7 +435,7 @@
'schema:datePublished': {
date_histogram : {
field : "schema:datePublished",
format: "MM-yyyy",
format: "dd-MM-yyyy",
interval : "month"
}
}
......@@ -461,7 +461,7 @@
// undocumented params are appended to the query string
index: "somedi",
body: {
size: 200,
size: 500,
query: {
bool: {
must: this.filters,
......@@ -540,7 +540,7 @@
'schema:datePublished': {
date_histogram : {
field : "schema:datePublished",
format: "MM-yyyy",
format: "dd-MM-yyyy",
interval : "month"
}
}
......
......@@ -469,7 +469,8 @@ Data can be provided in one of three ways:
this.rows.forEach(function(entry,index){
//console.log(entry[0].split('-')[0], entry[0].split('-')[1])
//console.log(new Date(entry[0].split('-')[0], entry[0].split('-')[1]-1), entry[1])
rows.push([new Date(entry[0].split('-')[0], entry[0].split('-')[1]-1), entry[1], entry[2], entry[3], entry[4]])
//console.log(entry[0])
rows.push([new Date(entry[0].split('-')[0], entry[0].split('-')[1]-1, entry[0].split('-')[2]), entry[1], entry[2], entry[3], entry[4]])
});
//console.log(rows)
this.rows = rows
......@@ -512,8 +513,11 @@ Data can be provided in one of three ways:
var datos = {}
//console.log(this.data)
this.data.hits.hits.forEach(function(entry) {
//console.log(entry._source)
var fecha = entry._source['schema:datePublished'].split('-')
var fechacompleta = fecha[0]+'-'+fecha[1]
//console.log(fecha)
var fechacompleta = fecha[0]+'-'+fecha[1]+'-'+fecha[2].split('T')[0]
//console.log(fechacompleta)
if(datos.hasOwnProperty(fechacompleta)){
if(entry._source['schema:search'] == 'pp'){
if (datos[fechacompleta].hasOwnProperty('pp')){
......@@ -574,7 +578,7 @@ Data can be provided in one of three ways:
return [key, pp, psoe, podemos, ciudadanos];
});
console.log(data)
//console.log(data)
}
else {
......
......@@ -94,7 +94,7 @@
'_selectedChanged(selected.splices)'
],
ready: function() {
console.log('LOG happymap-element: happymap ready()');
//console.log('LOG happymap-element: happymap ready()');
var that = this;
// Create Leaflet Map
var map = L.map(that.$.happymap).setView([40.46157664398332, -3.7380981445312504], 1);
......@@ -270,13 +270,13 @@
if (!that.isReady) {
return;
}
console.log('LOG happymap-element: switchMapLayer');
//console.log('LOG happymap-element: switchMapLayer');
if($.inArray(newMapMode, ['HAPPY_MODE', 'INTENSE_MODE', 'POWER_MODE', 'EMOJI_MODE'])>=0){
that.mapMode = newMapMode;
that.syncMapLayers();
}
else {
console.log('switchMapLayer: ERR!');
//console.log('switchMapLayer: ERR!');
}
},
syncMapLayers: function() {
......@@ -284,7 +284,7 @@
if (!that.isReady) {
return;
}
console.log('LOG happymap-element: syncMapLayers');
//console.log('LOG happymap-element: syncMapLayers');
if(that.mapMode == 'EMOJI_MODE'){
that.map.removeLayer(that.markerMapLayer);
that.map.removeLayer(that.happyMapLayer);
......@@ -330,7 +330,7 @@
that.powerMapLeyend.addTo(that.map);
}
else {
console.log('syncMapLayers: ERR!');
//console.log('syncMapLayers: ERR!');
}
that.infoTag.update(that.mapMode)
},
......@@ -355,7 +355,7 @@
if (!that.isReady) {
return;
}
console.log('LOG happymap-element: getMarkerInfo');
//console.log('LOG happymap-element: getMarkerInfo');
return "@"+entry.username+": "+entry.text + " \n[Classified as: " + entry.emotion+"]";
},
refreshMapPoints: function(){
......
......@@ -51,7 +51,7 @@ def retrieve_tweets(query, filePath, count=200):
mytweet["schema:headline"] = tweet["text"]
mytweet["schema:creator"] = tweet['user']['screen_name']
mytweet["schema:author"] = 'twitter'
mytweet["schema_datePublished"] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.strptime(tweet['created_at'],'%a %b %d %H:%M:%S +0000 %Y'))
mytweet["schema:datePublished"] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.strptime(tweet['created_at'],'%a %b %d %H:%M:%S +0000 %Y'))
mytweet["schema:parentItem"] = 'https://twitter.com/{screen_name}/status/{id}'.format(screen_name=tweet["in_reply_to_screen_name"], id=tweet["in_reply_to_status_id"])
json.dump(mytweet, output)
output.write('\n')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment