dashboard-gsicrawler.html 16.3 KB
Newer Older
militarpancho's avatar
militarpancho committed
1
2
3
<link rel="import" href="/bower_components/polymer/polymer.html">
<link rel="import" href="/bower_components/material-search/material-search.html">
<link rel="import" href="/bower_components/paper-material/paper-material.html">
4
<link rel="import" href="/elements/google-chart-elasticsearch/google-chart.html">
militarpancho's avatar
militarpancho committed
5
6
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tab.html">
7
<link rel="import" href="/elements/number-chart/number-chart.html">
militarpancho's avatar
militarpancho committed
8
<link rel="import" href="/bower_components/comment-chart/comment-chart.html">
militarpancho's avatar
militarpancho committed
9
10
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/yasgui-polymer/yasgui.html">
11
12
13
<link rel="import" href="/bower_components/iron-pages/iron-pages.html">
<link rel="import" href="/bower_components/paper-item/paper-item.html">
<link rel="import" href="/bower_components/paper-menu/paper-menu.html">
14
<link rel="import" href="/elements/entities-chart/entities-chart.html">
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
15
<link rel="import" href="/elements/people-chart/people-chart.html">
16
<link rel="import" href="/elements/news-chart/news-chart.html">
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
17
<link rel="import" href="/elements/tweet-chart/tweet-chart.html">
Alberto Pascual's avatar
Alberto Pascual committed
18
<link rel="import" href="/bower_components/leaflet-maps/leaflet-map.html">
19

20
<dom-module id="dashboard-gsicrawler">
21

militarpancho's avatar
militarpancho committed
22

23
  <link rel="import" type="css" href="dashboard-gsicrawler.css" type="css">
militarpancho's avatar
militarpancho committed
24
  <link rel="import" href="/styles/app-theme.html" type="css">
25
26
27
28
29
30
31

  <template>
    <iron-ajax auto
        url="queries.json"
        handle-as="json"
        last-response="{{queries}}"></iron-ajax>

Alberto Pascual's avatar
new viz    
Alberto Pascual committed
32
33
34
35
36
37
38
    <div style="width:10%; padding: 1em; display:inline-block">
      <img style="width: 100%;"src="/images/trivalent.jpg">
    </div>
    <div style="width:80%;     padding-bottom: 12px; text-align:center; display:inline-block">
      <img style="width: 30%;  margin:0 auto;"src="/images/logo-gsi-crawler.png">
    </div>

39
40
41
    <paper-tabs selected="{{selected}}">
      <paper-tab>Dashboard</paper-tab>
      <paper-tab>Sparql Editor</paper-tab>
Alberto Pascual's avatar
Alberto Pascual committed
42
      <paper-tab>About</paper-tab>
43
44
45
46
    </paper-tabs>

    <iron-pages selected="{{selected}}">
      <div>
47
        <div class="row">
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
48
        <material-search active="true" search-value="{{query}}"></material-search>
49
50
        </div>
        <br>  
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
51
52
        <div class="row">
          <div class="col-md-3">
53
            <number-chart 
54
              icon="/images/news.ico"
55
              stylebg="bg-yellow"
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
56
              title="Selected elements"
57
58
59
              data="{{data}}">
            </number-chart>
          </div>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
60
          <div class="col-md-3">
61
62
            <number-chart 
              data="{{data}}"
63
              object="cnn"
Alberto Pascual's avatar
Alberto Pascual committed
64
              aggKey="schema:author"
65
              title="CNN"
66
67
              icon="/images/cnn1.png"
              stylebg="bg-red">
68
69
            </number-chart>
          </div>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
70
          <div class="col-md-3">
militarpancho's avatar
militarpancho committed
71
72
             <number-chart  
               data="{{data}}" 
73
               object="The New York Times"
Alberto Pascual's avatar
Alberto Pascual committed
74
               aggKey="schema:author"
75
               title="The New York Times" 
76
               icon="/images/times1.png"
militarpancho's avatar
militarpancho committed
77
78
79
80
               aggkey="location"
               stylebg="bg-green"> 
             </number-chart>
           </div>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
81
82
83
84
85
86
87
88
89
90
91
92
           <div class="col-md-3">
             <number-chart  
               data="{{data}}" 
               object="twitter"
               aggKey="schema:author"
               title="Twitter" 
               icon="/images/twitter.ico"
               subtitle="Total"
               aggkey="location"
               stylebg="bg-aqua"> 
             </number-chart>
           </div>
militarpancho's avatar
militarpancho committed
93
         </div>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
94
95
96
         <br> 
        <div class="row">
          <div class="col-md-6">
97
            <google-chart
Alberto Pascual's avatar
Alberto Pascual committed
98
              field="schema:author"
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
99
100
101
102
              data="{{data}}"             
              id='pie-chart3'
              extra-id='pie-chart3'
              type='pie'
103
              filters="{{filters}}"
militarpancho's avatar
militarpancho committed
104
              icon='icons:reorder'
105
              options='{"title": "Source"}'
106
              cols='[{"label": "type", "type": "string"},{"label": "Count", "type": "number"}]'
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
107
108
109
110
111
112
113
114
115
116
117
118
            </google-chart>            
          </div>
          <div class="col-md-6">
            <entities-chart
                field="topics.rdfs:subClassOf"
                data="{{data}}"
                title="Topics"
                icon="icons:list"
                param="{{param}}"
                id="entitieschart"
                filters="{{filters}}">
            </entities-chart>
119
120
          </div>
        </div>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
121
122
123
        <br> 
     <div class="row"> 
        <div class="col-md-6">
militarpancho's avatar
militarpancho committed
124
125
126
            <google-chart
              field="sentiment"
              data="{{data}}"             
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
127
128
              id='barsentiment'
              extra-id='1'
129
              type='column'
militarpancho's avatar
militarpancho committed
130
              filters="{{filters}}"
militarpancho's avatar
militarpancho committed
131
132
133
              icon='social:mood'
              options='{"title": "Sentiments"}'
              cols='[{"label": "Sentiment", "type": "string"},{"label": "Count", "type": "number"}]'
militarpancho's avatar
militarpancho committed
134
            </google-chart>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
135
136
137
138
139
140
141
142
143
144
         </div> 
          <div class="col-md-6">
            <people-chart
                data="[[getPeople(data)]]"
                title="People"
                icon="maps:person-pin"
                param="{{param}}"
                id="entitieschart"
                filters="{{filters}}">
            </people-chart>
145
          </div>
146
      </div>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
147
148
149
      <br> 
      <div class="row">
        <div class="col-md-12">
150
        <google-chart
Alberto Pascual's avatar
Alberto Pascual committed
151
          field="schema:datePublished"
152
153
154
155
156
157
          data="{{data}}"             
          id='line-chart'
          extra-id='line-chart'
          type='area'
          filters="{{filters}}"
          icon='icons:trending-up'
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
158
          options='{"title": "Temporal Trends"}'
159
160
          cols='[{"label": "Date", "type": "string"},{"label": "Count", "type": "number"}]'
        </google-chart>
161
      </div>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
162
163
      </div>
      <br> 
164
      <div class="row">
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
165
      <div class="col-md-6">
166
167
        <paper-tabs selected="{{selectedpage}}">
          <paper-tab><iron-icon icon="icons:receipt"></iron-icon> List of news</paper-tab>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
168
          <paper-tab><iron-icon icon="icons:speaker-notes"></iron-icon> Social Media</paper-tab>
169
170
171
172
173
174
175
176
177
178
        </paper-tabs>
        <iron-pages class="lists" selected="{{selectedpage}}">
        <div>
          <news-chart 
            datos = "{{data}}"
            title="List of news"
            filters="{{filters}}"
            icon='icons:receipt'>
          </news-chart>
        </div>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
179
180
181
182
183
184
185
        <div>
          <tweet-chart 
            datos = "{{data}}"
            title="List of tweets"
            filters="{{filters}}"
            icon='communication:comment'>
          </tweet-chart></div>
186
187
        </iron-pages>
       </div>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
188
       <div class="col-md-6">
189
190
191
192
193
        <div class="top-bar">
        <iron-icon icon="maps:my-location"></iron-icon>
          <span>Geolocated News and Social Media</span>
        </div>
        <leaflet-map fit-to-markers>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
194
          <template is="dom-repeat" items="[[getPlaces(data)]]" as="place">
195
                <leaflet-marker latitude="{{place.lat}}" longitude="{{place.lng}}" title="{{place.name}}"> 
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
196
                    <b>Headline:</b>
197
198
                    <span>{{place.name}}</span>
                </leaflet-marker>
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
199
              </template>
200
        </leaflet-map>
201
       </div>
202
      </div>
203
204
205
    </div>
    <div>
        <yasgui-ui
206
       endpoint="http:///sefarad-fuseki.cluster.gsi.dit.upm.es/gsicrawler/query"
207
       queries="{{queries}}"
208
209
        </yasgui-ui>
    </div>
Alberto Pascual's avatar
Alberto Pascual committed
210
211
212
213
214
215
216
217
218
219
220
221
222
    <div>
<br>        
This demo shows GSI-Crawler functionalities in the context of the project Trivalent. The system is aimed at crawling from a variety of data sources including news web sites such as CNN and New York Times, as well as social media platforms, like Twitter. The presented demo is the result of the ingest of the crawled online data and posterior analysis and semantic augmentation of said data. The resulting information is stored in both ElastiSearch and Fuseki databases, which allows the user to consult the data is a variety of forms, as showed.
<br><br>
For a quick overview of the funtionalities of the demo, please watch the following video:
<br><br>

<br><br>
To discover more, please consult the documentation at <a href="http://gsicrawler.readthedocs.io" target="_blank">Read The Docs</a>
<br><br>
H2020 RIA Action Grant No. 740934 under the call SEC-06-FCT-2016 — Developing a comprehensive approach to violent radicalization in the EU from early understanding to improving protection.
<br><br> 
    </div>
223
    </iron-pages>
224
225
         <div style="background-color:#ccc; padding: 1%;">
        <div style="width:32%; margin: 0 auto">
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
226
       <img height="60px" src="/images/gsi.png">   © 2017 Grupo de Sistemas Inteligentes GSI-UPM 
227
228
        </div>
       </div>
229
  </template>
230
   
231
232

  <script>
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
233
  var ready = false;
234
    Polymer({
235
      is: 'dashboard-gsicrawler',
236
237
238
239
240
      properties: {
        selected: {
          type: Number,
          value: 0
        }, 
241
242
243
244
        selectedpage: {
          type: Number,
          value: 0
        }, 
245
246
247
        query: {
          type: String,
        },
248
        ids:{
249
250
          type: Array
           
251
252
253
254
        },
        data:{
          type: Object
        },
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
255
256
257
258
        client: {
          type: Object,
          notify: true,
          observer: '_clientChanged'              
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
        },
        fields: {
          type: Array,
          value: function() { return []; }
        },

        filters: {
          type: Array,
          notify: true,
          value: function() { return []; }
        }
      },
    
      observers: [
      '_filtersChange(filters.*)'
      ],
275
276
277
278
279
280
      behaviors: [
        Polymer.IronA11yKeysBehavior
      ],
      keyBindings: {
        'enter': '_search'
      },
281
282

      ready: function(){
283
        //console.log("ready");
284
285
      },

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
286
      _clientChanged: function() {
287
        //console.log("ClientChanged");
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
288
289
        ready = true;
        this._query();
290
      },
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
      getPeople: function(data){
        var people = []
        //console.log(data)
        data.hits.hits.forEach( function (entry){
          //console.log(entry)
          entry._source.entities.forEach(function(entity){
            //console.log(entity)
            if ('dbo:thumbnail' in entity) {
              people.push({'name': entity['nif:anchorOf'], 'image': entity['dbo:thumbnail']})
            }
          })
        })
        people = this.removeDuplicates(people, 'name')
        console.log(people)
        return people.slice(0,8)
      },
      removeDuplicates: function(originalArray, prop){
        var newArray = [];
        var lookupObject  = {};

         for(var i in originalArray) {
            lookupObject[originalArray[i][prop]] = originalArray[i];
         }

         for(i in lookupObject) {
             newArray.push(lookupObject[i]);
         }
          return newArray;
      },
      getPlaces: function(data){
        var places = []
        data.hits.hits.forEach( function (entry){
          entry._source.entities.forEach(function(entity){
            if ('latitude' in entity) {
              places.push({'lat': entity.latitude, 'lng': entity.longitude, 'name': entry._source['schema:headline']})
            }
          })
        })
        return places
      },
331
      _search: function(){
332
333
        //console.log("search fired")
        //console.log(this.query.length)
334
        if (this.query.length == 0){
335
          //console.log("default search fired")
336
337
338
339
          this._query()
        }
        else this._ESsearch()
      },
340
      _filtersChange: function() {
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
341
        this._query();
342
343
       },
      _ESsearch: function() {
344
        //console.log("_filtersChangedash")
345
        var that = this;
346
        //console.log("Ready?: ", ready);
347
348
349
350
351
        if(ready){
          this.client.search({
          // undocumented params are appended to the query string
          index: "gsicrawler",
          body: {
352
            size: 200,
353
354
355
            query: {
              multi_match:{
                query: this.query,
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
356
                fields: ['schema:author', 'schema:headline', 'sentiments', 'schema:about', 'entities.nif:anchorOf']
357
358
              }
            },
359
            sort:{'schema:datePublished':{order: "desc"}},
360
361
362
363
364
365
366
367
368
            aggs: {
             type: {
               terms: {
                 field: "@type.keyword",
                 order: {
                   _count: "desc"
                 }
               }
             },
Alberto Pascual's avatar
Alberto Pascual committed
369
             'schema:author': {
370
               terms: {
Alberto Pascual's avatar
Alberto Pascual committed
371
                 field: "schema:author.keyword",
372
373
374
375
376
                 order: {
                   _count: "desc"
                 }
               }
             },
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
377
378
379
380
381
382
383
384
385
             'entities.rdfs:subClassOf': {
               terms: {
                 field: "entities.rdfs:subClassOf.keyword",
                 order: {
                   _count: "desc"
                 } 
               }
             },
             'topics.rdfs:subClassOf': {
386
               terms: {
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
387
388
                 field: "topics.rdfs:subClassOf.keyword",
                 size: 20,
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
                 order: {
                   _count: "desc"
                 } 
               }
             },
           sentiment: {
             terms: {
             field: "sentiments.marl:hasPolarity.keyword",
             order: {
               _count: "desc"
               }
             }
             },
          emotion: {
              terms: {
              field: "emotions.onyx:hasEmotion.onyx:hasEmotionCategory.keyword",
             order: {
               _count: "desc"
                     }      
                   }
409
      },
Alberto Pascual's avatar
Alberto Pascual committed
410
      'schema:datePublished': {
411
              date_histogram : {
Alberto Pascual's avatar
Alberto Pascual committed
412
                field : "schema:datePublished",
413
414
415
                format: "MM-yyyy",
                interval : "month"
            }
416
       }
417
        }
418
419
420
421
422
423
424
      } 
          }).then(function (resp) {
            var myids = []
            resp.hits.hits.forEach(function(entry){myids.push(entry._id)})
            that.ids = myids;
            //console.log(that.ids)
            that.data = resp;
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
425
            console.log(that.data);
426
427
428
            
            });
        }
Alberto Pascual's avatar
Alberto Pascual committed
429
      },
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
430
       _query: function() {
431
        //console.log("_query")
432
        var that = this;
433
        //console.log("Ready?: ", ready);
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
434
435
        if(ready){
          this.client.search({
436
          // undocumented params are appended to the query string
437
          index: "gsicrawler",
438
          body: {
439
            size: 200,
440
441
442
443
444
            query: {
              bool: {
                must: this.filters,
              }
            },
445
            sort:{'schema:datePublished':{order: "desc"}},
446
            aggs: {
447
             type: {
militarpancho's avatar
militarpancho committed
448
               terms: {
449
450
451
452
453
454
                 field: "@type.keyword",
                 order: {
                   _count: "desc"
                 }
               }
             },
Alberto Pascual's avatar
Alberto Pascual committed
455
             'schema:author': {
456
               terms: {
Alberto Pascual's avatar
Alberto Pascual committed
457
                 field: "schema:author.keyword",
militarpancho's avatar
militarpancho committed
458
459
                 order: {
                   _count: "desc"
460
                 }
militarpancho's avatar
militarpancho committed
461
               }
militarpancho's avatar
militarpancho committed
462
             },
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
463
464
465
466
467
468
469
470
471
             'entities.rdfs:subClassOf': {
               terms: {
                 field: "entities.rdfs:subClassOf.keyword",
                 order: {
                   _count: "desc"
                 } 
               }
             },
             'topics.rdfs:subClassOf': {
472
               terms: {
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
473
474
                 field: "topics.rdfs:subClassOf.keyword",
                 size: 20,
475
476
477
                 order: {
                   _count: "desc"
                 } 
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
478
               }
479
             },
480
481
482
483
484
485
486
487
488
489
490
491
492
           sentiment: {
             terms: {
             field: "sentiments.marl:hasPolarity.keyword",
             order: {
               _count: "desc"
               }
             }
             },
          emotion: {
              terms: {
              field: "emotions.onyx:hasEmotion.onyx:hasEmotionCategory.keyword",
             order: {
               _count: "desc"
493
494
                     }      
                   }
495
      },
Alberto Pascual's avatar
Alberto Pascual committed
496
      'schema:datePublished': {
497
              date_histogram : {
Alberto Pascual's avatar
Alberto Pascual committed
498
                field : "schema:datePublished",
499
500
501
502
                format: "MM-yyyy",
                interval : "month"
            }
       }
503
504
       }
      } 
505
          }).then(function (resp) {
506
507
508
509
            var myids = []
            resp.hits.hits.forEach(function(entry){myids.push(entry._id)})
            that.ids = myids;
            //console.log(that.ids)
510
            that.data = resp;
Alberto Pascual's avatar
new viz    
Alberto Pascual committed
511
            console.log(that.data);
512
            
513
514
515
516
517
518
519
            });
        }
      }
    });
  </script>

</dom-module>