dashboard-gsicrawler.html 16.9 KB
Newer Older
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
<link rel="import" href="/bower_components/polymer/polymer.html">
<link rel="import" href="/bower_components/material-search/material-search.html">
<link rel="import" href="/bower_components/paper-material/paper-material.html">
<link rel="import" href="/elements/google-chart-elasticsearch/google-chart.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tab.html">
<link rel="import" href="/elements/number-chart/number-chart.html">
<link rel="import" href="/bower_components/comment-chart/comment-chart.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/yasgui-polymer/yasgui.html">
<link rel="import" href="/bower_components/iron-pages/iron-pages.html">
<link rel="import" href="/bower_components/paper-item/paper-item.html">
<link rel="import" href="/bower_components/paper-menu/paper-menu.html">
<link rel="import" href="/bower_components/paper-autocomplete/paper-autocomplete.html">
<link rel="import" href="/elements/entities-chart/entities-chart.html">
<link rel="import" href="/elements/people-chart/people-chart.html">
<link rel="import" href="/elements/news-chart/news-chart.html">
<link rel="import" href="/elements/propaganda-chart/propaganda-chart.html">
<link rel="import" href="/elements/tweet-chart/tweet-chart.html">
<link rel="import" href="/elements/annotate-view/annotate-view.html">
<link rel="import" href="/elements/happymap/happymap-element.html">

Daniel Suarez Souto's avatar
Styled    
Daniel Suarez Souto committed
23
24
<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.2.0/js/bootstrap.min.js"></script>

25

Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
26
27
<!--<script src="https://hypothes.is/embed.js" async></script>-->

28
29


Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
30
31
<dom-module id="dashboard-gsicrawler">

32

Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
33
34
35
36
  <link rel="import" type="css" href="dashboard-gsicrawler.css" type="css">
  <link rel="import" href="/styles/app-theme.html" type="css">

  <template>
37
    <!-- <iron-ajax auto
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
38
39
        url="queries.json"
        handle-as="json"
40
        last-response="{{queries}}"></iron-ajax> -->
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
41
42
43
44
45
    <!--
    <div style="width:10%; padding: 1em; display:inline-block">
      <img style="width: 100%;"src="/images/trivalent.jpg">
    </div>
    -->
Daniel Suarez Souto's avatar
Styled    
Daniel Suarez Souto committed
46
    <div style="width:100%; padding-bottom: 20px 0; text-align:center; display:inline-block">
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
47
48
49
      <img style="width: 20%;  margin:0 auto;"src="/images/trivalent.jpg">
    </div>

Daniel Suarez Souto's avatar
Styled    
Daniel Suarez Souto committed
50
    <paper-tabs selected="{{selected}}" style="margin: 0 60px;">
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
51
52
53
54
55
      <paper-tab>Dashboard</paper-tab>
      <paper-tab>Sparql Editor</paper-tab>
      <paper-tab>About</paper-tab>
    </paper-tabs>

56
   <iron-pages selected="{{selected}}">
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
57
58
      <div>
        <div class="row">
Daniel Suarez Souto's avatar
Styled    
Daniel Suarez Souto committed
59
        <material-search active="true" search-value="{{query}}" style="margin: 0 15px;"></material-search>
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
60
        <br>
61
      
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

        </div>
        <br>  
        <div class="row">
          <div class="col-md-4">
            <number-chart 
              icon="/images/news.ico"
              stylebg="bg-white"
              title="Selected elements"
              data="{{data}}">
            </number-chart>
          </div>
          <div class="col-md-4">
            <number-chart 
              data="{{data}}"
              object="cnn"
              aggKey="schema:author"
              title="CNN"
80
              icon="/images/cnn_blanco.svg"
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
              stylebg="bg-red">
            </number-chart>
          </div>
          <div class="col-md-4">
             <number-chart  
               data="{{data}}" 
               object="The New York Times"
               aggKey="schema:author"
               title="The New York Times" 
               icon="/images/times1.png"
               stylebg="bg-green"> 
             </number-chart>
           </div>
           </div>
           <div class="row">
           <div class="col-md-4">
             <number-chart  
               data="{{data}}" 
               object="Al Jazeera"
               aggKey="schema:author"
               title="Al Jazeera" 
102
               icon="/images/Aljazeera_blanco.svg"
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
103
104
105
106
107
108
109
110
111
112
               subtitle="Total"
               stylebg="bg-yellow"> 
             </number-chart>
           </div>
            <div class="col-md-4">
             <number-chart  
               data="{{data}}" 
               object="twitter"
               aggKey="schema:author"
               title="Twitter" 
113
               icon="/images/twitter.svg"
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
114
115
116
117
118
119
120
121
122
123
               subtitle="Total"
               stylebg="bg-aqua"> 
             </number-chart>
           </div>
           <div class="col-md-4">
             <number-chart  
               data="{{data}}" 
               aggKey="magazines"
               object=""
               title="Magazines" 
124
               icon="/images/magazine.svg"
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
125
               subtitle="Total"
126
               stylebg="bg-purple"> 
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
             </number-chart>
           </div>
         </div>
        
         <br> 
        <div class="row">
          <div class="col-md-6">
            <google-chart
              field="schema:author"
              data="{{data}}"             
              id='pie-chart3'
              extra-id='pie-chart3'
              type='pie'
              filters="{{filters}}"
              icon='icons:reorder'
              options='{"title": "Source"}'
143
              cols='[{"label": "type", "type": "string"},{"label": "Count", "type": "number"}]'>
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
144
145
146
147
            </google-chart>            
          </div>
          <div class="col-md-6">
            <entities-chart
148
                field="entities.name"
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
149
150
151
152
153
154
155
156
157
158
159
                data="{{data}}"
                title="Organizations"
                icon="icons:list"
                param="{{param}}"
                id="entitieschart"
                filters="{{filters}}">
            </entities-chart>
          </div>
        </div>
        <br> 
     <div class="row"> 
160
161
         <div class="col-md-6">
           <!-- <google-chart
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
162
163
164
165
166
167
168
169
              field="sentiment"
              data="{{data}}"             
              id='barsentiment'
              extra-id='1'
              type='column'
              filters="{{filters}}"
              icon='social:mood'
              options='{"title": "Sentiments"}'
170
171
172
173
174
175
176
177
178
179
180
181
182
183
              cols='[{"label": "Sentiment", "type": "string"},{"label": "Count", "type": "number"}]'>
            </google-chart>-->
         <google-chart
          field="schema:datePublished"
          data="{{data}}"             
          id='line-chart'
          extra-id='line-chart'
          type='area'
          filters="{{filters}}"
          icon='icons:trending-up'
          options='{"title": "Temporal Trends"}'
          cols='[{"label": "Date", "type": "string"},{"label": "Count", "type": "number"}]'>
        </google-chart>   
         </div>  
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
184
185
          <div class="col-md-6">
            <people-chart
186
                data="{{data}}"
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
187
188
189
190
191
192
193
194
                title="People"
                icon="maps:person-pin"
                param="{{param}}"
                id="entitieschart"
                filters="{{filters}}">
            </people-chart>
          </div>
      </div>
195
      
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
196
197
198
      <br> 
      <div class="row">
      <div class="col-md-6">
199
        <paper-tabs selected="{{selectedpage}}" style="height: 38px">
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
          <paper-tab><iron-icon icon="icons:receipt"></iron-icon> List of news</paper-tab>
          <paper-tab><iron-icon icon="icons:speaker-notes"></iron-icon> Social Media</paper-tab>
          <paper-tab><iron-icon icon="icons:speaker-notes"></iron-icon> Magazines</paper-tab>
        </paper-tabs>
        <iron-pages class="lists" selected="{{selectedpage}}">
        <div>
          <news-chart 
            datos = "{{data}}"
            title="List of news"
            filters="{{filters}}"
            icon='icons:receipt'>
          </news-chart>
        </div>
        <div>
          <tweet-chart 
            datos = "{{data}}"
            title="List of tweets"
            filters="{{filters}}"
            icon='communication:comment'>
219
220
221
          </tweet-chart>
        </div>
        <div>
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
222
223
224
225
226
227
          
          <propaganda-chart 
            datos = "{{data}}"
            title="List of propaganda"
            filters="{{filters}}"
            icon='communication:comment'>
228
229
          </propaganda-chart>
        </div>
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
          
        </iron-pages>
       </div>
       <div class="col-md-6">
        <div class="top-bar">
        <iron-icon icon="maps:my-location"></iron-icon>
          <span>Geolocated News and Social Media</span>
        </div>
        <!--
        <leaflet-map fit-to-markers>
          <template is="dom-repeat" items="[[getPlaces(data)]]" as="place">
                <leaflet-marker latitude="{{place.lat}}" longitude="{{place.lng}}" title="{{place.name}}"> 
                    <b>Headline:</b>
                    <span>{{place.name}}</span>
                </leaflet-marker>
              </template>
        </leaflet-map>
        -->
248
        <happymap-element 
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
249
          data="{{data}}",
250
251
252
          '<!-- visalized data in case of connection error -->'>
            
          </happymap-element>
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
253
254
255
256
257
258
       </div>
      </div>
    </div>
    <div>
        <yasgui-ui
       endpoint="http:///localhost:13030/gsicrawler/query"
259
       queries="{{queries}}">
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
        </yasgui-ui>
    </div>
    <div>
    

This demo shows GSI-Crawler functionalities in the context of the project Trivalent. The system is aimed at crawling from a variety of data sources including news web sites such as CNN and New York Times, as well as social media platforms, like Twitter. The presented demo is the result of the ingest of the crawled online data and posterior analysis and semantic augmentation of said data. The resulting information is stored in both ElastiSearch and Fuseki databases, which allows the user to consult the data is a variety of forms, as showed.

<br><br>
For a quick overview of the funtionalities of the demo, please watch the following video:
<br><br>
<iframe width="560" height="315" src="https://www.youtube.com/embed/3s894sjevBQ" frameborder="0" allowfullscreen></iframe>
<br><br>
To discover more, please consult the documentation at <a href="http://gsicrawler.readthedocs.io" target="_blank">Read The Docs</a>
<br><br>
H2020 RIA Action Grant No. 740934 under the call SEC-06-FCT-2016 — Developing a comprehensive approach to violent radicalization in the EU from early understanding to improving protection.
<br><br> 
    </div>
      <annotate-view name="annotate-view"></annotate-view>
    </iron-pages>
Daniel Suarez Souto's avatar
Styled    
Daniel Suarez Souto committed
279
280
281
282
         <div style="background-color:#ccc; padding: 1%; margin: -16px;margin-top: 20px;">
        <div style="display: flex; flex-direction: column; justify-content: center; align-content: center; margin: 0 65px;">
          <img style="width: 100%; height: 50px; object-fit: contain; margin-bottom: 5px;" src="/images/gsi.png">   
          <div style="display: flex; flex-direction: column; justify-content: center; font-size: 12px; text-align: center;">© 2018 Grupo de Sistemas Inteligentes GSI-UPM </div>
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
        </div>
       </div>



  </template>
   

  <script>
  var ready = false;
    Polymer({
      is: 'dashboard-gsicrawler',
      properties: {
        selected: {
          type: Number,
          value: 0
        }, 
        selectedpage: {
          type: Number,
          value: 0
        }, 
        query: {
          type: String,
        },
        ids:{
          type: Array
           
        },
        data:{
          type: Object
        },
        client: {
          type: Object,
          notify: true,
          observer: '_clientChanged'              
        },
        fields: {
          type: Array,
          value: function() { return []; }
        },

        filters: {
          type: Array,
          notify: true,
          value: function() { return []; }
        }
      },
    
      observers: [
      '_filtersChange(filters.*)',
      'entityselection(selectedentity)'
      ],
      behaviors: [
        Polymer.IronA11yKeysBehavior
      ],
      keyBindings: {
        'enter': '_search'
      },

      ready: function(){
        //console.log("ready");
      },

      _clientChanged: function() {
        //console.log("ClientChanged");
        ready = true;
        this._query();
      },
      
      getEntities: function(datos){
353
            console.log("getEntities")
354
355
356
357
358
359
360
361
362
363
            if(this.entityselection){
              var entities = []
              var hits = this.data.aggregations['entities.name'].buckets;
              console.log(hits)
              hits.forEach(function(entry){
                entities.push(entry.key.split('/')[4]);
              });
              console.log(entities)
              return entities
            } 
364
          },
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
365
366
367
368
369
370
      
      entityselection: function(entity){
        console.log("entityselection")
        console.log(entity)
        if(entity == null){
          this.filters = [];
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
371
        } else this.push('filters', { terms: { 'entities.schema:name': entity.toLowerCase().split(" ")}})
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
372
373
374
375
      },
      
      getPlaces: function(data){
        var places = []
376
        console.log("getPlace")
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
377
378
379
380
381
382
383
384
385
386
        data.hits.hits.forEach( function (entry){
          entry._source.entities.forEach(function(entity){
            if ('latitude' in entity) {
              places.push({'lat': entity.latitude, 'lon': entity.longitude, 'name': entry._source['http://schema.org/headline'][0]['@value']})
            }
          })
        })
        return places
      },
      _search: function(){
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
387
        console.log("search fired")
388
        console.log(this.selectedentity)
389
390
391
392
393
394
395
396
397
398
        
        if (this.query.length == 0){
          //console.log("default search fired")
          this.filters = []
          this._query()
        }
        else {
          console.log("Filter")
          this.push('filters', {terms: {'schema:articleBody': [this.query]}});
          this._query()
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
399
        }
400
        
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
      },
      _filtersChange: function() {
        this._query();
       },
       _query: function() {
        //console.log("_query")
        var that = this;
        //console.log("Ready?: ", ready);
        if(ready){
          this.client.search({
          // undocumented params are appended to the query string
          index: "gsicrawler",
          body: {
            size: 200,
            query: {
              bool: {
                must: this.filters,
              }
            },
            aggs: {
             type: {
               terms: {
                 field: "@type.keyword",
                 order: {
                   _count: "desc"
                 }
               }
             },
             'schema:author': {
               terms: {
                 field: "schema:author.keyword",
                 order: {
                   _count: "desc"
                 }
               }
             },
             magazines: {
               terms: {
                 field: "schema:author.keyword",
                 include: ".*dbpedia.*",
                 order: {
                   _count: "desc"
                 }
               }
             },
446
             'entities.type': {
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
447
               terms: {
448
                 field: "entities.@type.keyword",
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
449
450
451
452
453
454
                 order: {
                   _count: "desc"
                 } 
               }
             },

455
             'entities.name': {
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
456
               terms: {
457
                 field: "entities.schema:name.keyword",
458
                 size : 16,
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
                 order: {
                   _count: "desc"
                 } 
               }
             },
             'topics.rdfs:subClassOf': {
               terms: {
                 field: "topics.rdfs:subClassOf.keyword",
                 size: 20,
                 order: {
                   _count: "desc"
                 } 
               }
             },
           sentiment: {
             terms: {
475
476
477
478
               field: "sentiments.marl:hasPolarity.keyword",
               order: {
                 _count: "desc"
                 }
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
479
             }
480
481
482
483
484
485
486
            },
            'schema:datePublished': {
                    date_histogram : {
                      field : "schema:datePublished",
                      format: "MM-yyyy",
                      interval : "month"
                  }
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
487
            }
488
489
490
491
492
493
494
          }
        } 
        }).then(function (resp) {
          var myids = []
          resp.hits.hits.forEach(function(entry){myids.push(entry._id)})
          that.ids = myids;
          //console.log(that.ids)
495
496
497
498
499
500
          for (var i=0; i<resp['aggregations']['schema:author']['buckets'].length;i++) {
            if(resp['aggregations']['schema:author']['buckets'][i]["key"]=="http://dbpedia.org/page/Dabiq_(magazine)")
                 resp['aggregations']['schema:author']['buckets'][i]["key"]="Dabiq" 
            if(resp['aggregations']['schema:author']['buckets'][i]["key"]=="http://dbpedia.org/page/Rumiyah_(magazine)")
                 resp['aggregations']['schema:author']['buckets'][i]["key"]="Rumiyah"          
          }
501
502
          //resp['aggregations']['schema:author']['buckets'][1]["key"]="Dabiq"
          //resp['aggregations']['schema:author']['buckets'][2]["key"]="Rumiyah"
503
          that.data = resp;
504
         
505
506
507
          //console.log(that.data);
          
          });
Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
508
509
510
        }
      }
    });
511
512
  

Daniel Suarez Souto's avatar
Daniel Suarez Souto committed
513
514
  </script>

515
</dom-module>