dashboard-gsicrawler.html

<link rel="import" href="/bower_components/polymer/polymer.html">
<link rel="import" href="/bower_components/material-search/material-search.html">
<link rel="import" href="/bower_components/paper-material/paper-material.html">
<link rel="import" href="/elements/google-chart-elasticsearch/google-chart.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tab.html">
<link rel="import" href="/elements/number-chart/number-chart.html">
<link rel="import" href="/bower_components/comment-chart/comment-chart.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/yasgui-polymer/yasgui.html">
<link rel="import" href="/bower_components/iron-pages/iron-pages.html">
<link rel="import" href="/bower_components/paper-item/paper-item.html">
<link rel="import" href="/bower_components/paper-menu/paper-menu.html">
<link rel="import" href="/elements/entities-chart/entities-chart.html">
<link rel="import" href="/elements/people-chart/people-chart.html">
<link rel="import" href="/elements/news-chart/news-chart.html">
<link rel="import" href="/elements/tweet-chart/tweet-chart.html">
<link rel="import" href="/bower_components/leaflet-maps/leaflet-map.html">

<dom-module id="dashboard-gsicrawler">


  <link rel="import" type="css" href="dashboard-gsicrawler.css" type="css">
  <link rel="import" href="/styles/app-theme.html" type="css">

  <template>
    <iron-ajax auto
        url="queries.json"
        handle-as="json"
        last-response="{{queries}}"></iron-ajax>

    <div style="width:10%; padding: 1em; display:inline-block">
      <img style="width: 100%;"src="/images/trivalent.jpg">
    </div>
    <div style="width:80%;     padding-bottom: 12px; text-align:center; display:inline-block">
      <img style="width: 30%;  margin:0 auto;"src="/images/logo-gsi-crawler.png">
    </div>

    <paper-tabs selected="{{selected}}">
      <paper-tab>Dashboard</paper-tab>
      <paper-tab>Sparql Editor</paper-tab>
      <paper-tab>About</paper-tab>
    </paper-tabs>

    <iron-pages selected="{{selected}}">
      <div>
        <div class="row">
        <material-search active="true" search-value="{{query}}"></material-search>
        </div>
        <br>  
        <div class="row">
          <div class="col-md-3">
            <number-chart 
              icon="/images/news.ico"
              stylebg="bg-yellow"
              title="Selected elements"
              data="{{data}}">
            </number-chart>
          </div>
          <div class="col-md-3">
            <number-chart 
              data="{{data}}"
              object="cnn"
              aggKey="schema:author"
              title="CNN"
              icon="/images/cnn1.png"
              stylebg="bg-red">
            </number-chart>
          </div>
          <div class="col-md-3">
             <number-chart  
               data="{{data}}" 
               object="The New York Times"
               aggKey="schema:author"
               title="The New York Times" 
               icon="/images/times1.png"
               aggkey="location"
               stylebg="bg-green"> 
             </number-chart>
           </div>
           <div class="col-md-3">
             <number-chart  
               data="{{data}}" 
               object="twitter"
               aggKey="schema:author"
               title="Twitter" 
               icon="/images/twitter.ico"
               subtitle="Total"
               aggkey="location"
               stylebg="bg-aqua"> 
             </number-chart>
           </div>
         </div>
         <br> 
        <div class="row">
          <div class="col-md-6">
            <google-chart
              field="schema:author"
              data="{{data}}"             
              id='pie-chart3'
              extra-id='pie-chart3'
              type='pie'
              filters="{{filters}}"
              icon='icons:reorder'
              options='{"title": "Source"}'
              cols='[{"label": "type", "type": "string"},{"label": "Count", "type": "number"}]'
            </google-chart>            
          </div>
          <div class="col-md-6">
            <entities-chart
                field="topics.rdfs:subClassOf"
                data="{{data}}"
                title="Topics"
                icon="icons:list"
                param="{{param}}"
                id="entitieschart"
                filters="{{filters}}">
            </entities-chart>
          </div>
        </div>
        <br> 
     <div class="row"> 
        <div class="col-md-6">
            <google-chart
              field="sentiment"
              data="{{data}}"             
              id='barsentiment'
              extra-id='1'
              type='column'
              filters="{{filters}}"
              icon='social:mood'
              options='{"title": "Sentiments"}'
              cols='[{"label": "Sentiment", "type": "string"},{"label": "Count", "type": "number"}]'
            </google-chart>
         </div> 
          <div class="col-md-6">
            <people-chart
                data="[[getPeople(data)]]"
                title="People"
                icon="maps:person-pin"
                param="{{param}}"
                id="entitieschart"
                filters="{{filters}}">
            </people-chart>
          </div>
      </div>
      <br> 
      <div class="row">
        <div class="col-md-12">
        <google-chart
          field="schema:datePublished"
          data="{{data}}"             
          id='line-chart'
          extra-id='line-chart'
          type='area'
          filters="{{filters}}"
          icon='icons:trending-up'
          options='{"title": "Temporal Trends"}'
          cols='[{"label": "Date", "type": "string"},{"label": "Count", "type": "number"}]'
        </google-chart>
      </div>
      </div>
      <br> 
      <div class="row">
      <div class="col-md-6">
        <paper-tabs selected="{{selectedpage}}">
          <paper-tab><iron-icon icon="icons:receipt"></iron-icon> List of news</paper-tab>
          <paper-tab><iron-icon icon="icons:speaker-notes"></iron-icon> Social Media</paper-tab>
        </paper-tabs>
        <iron-pages class="lists" selected="{{selectedpage}}">
        <div>
          <news-chart 
            datos = "{{data}}"
            title="List of news"
            filters="{{filters}}"
            icon='icons:receipt'>
          </news-chart>
        </div>
        <div>
          <tweet-chart 
            datos = "{{data}}"
            title="List of tweets"
            filters="{{filters}}"
            icon='communication:comment'>
          </tweet-chart></div>
        </iron-pages>
       </div>
       <div class="col-md-6">
        <div class="top-bar">
        <iron-icon icon="maps:my-location"></iron-icon>
          <span>Geolocated News and Social Media</span>
        </div>
        <leaflet-map fit-to-markers>
          <template is="dom-repeat" items="[[getPlaces(data)]]" as="place">
                <leaflet-marker latitude="{{place.lat}}" longitude="{{place.lng}}" title="{{place.name}}"> 
                    <b>Headline:</b>
                    <span>{{place.name}}</span>
                </leaflet-marker>
              </template>
        </leaflet-map>
       </div>
      </div>
    </div>
    <div>
        <yasgui-ui
       endpoint="http:///sefarad-fuseki.cluster.gsi.dit.upm.es/gsicrawler/query"
       queries="{{queries}}"
        </yasgui-ui>
    </div>
    <div>
<br>        
This demo shows GSI-Crawler functionalities in the context of the project Trivalent. The system is aimed at crawling from a variety of data sources including news web sites such as CNN and New York Times, as well as social media platforms, like Twitter. The presented demo is the result of the ingest of the crawled online data and posterior analysis and semantic augmentation of said data. The resulting information is stored in both ElastiSearch and Fuseki databases, which allows the user to consult the data is a variety of forms, as showed.
<br><br>
For a quick overview of the funtionalities of the demo, please watch the following video:
<br><br>

<br><br>
To discover more, please consult the documentation at <a href="http://gsicrawler.readthedocs.io" target="_blank">Read The Docs</a>
<br><br>
H2020 RIA Action Grant No. 740934 under the call SEC-06-FCT-2016 — Developing a comprehensive approach to violent radicalization in the EU from early understanding to improving protection.
<br><br> 
    </div>
    </iron-pages>
         <div style="background-color:#ccc; padding: 1%;">
        <div style="width:32%; margin: 0 auto">
       <img height="60px" src="/images/gsi.png">   © 2017 Grupo de Sistemas Inteligentes GSI-UPM 
        </div>
       </div>
  </template>
   

  <script>
  var ready = false;
    Polymer({
      is: 'dashboard-gsicrawler',
      properties: {
        selected: {
          type: Number,
          value: 0
        }, 
        selectedpage: {
          type: Number,
          value: 0
        }, 
        query: {
          type: String,
        },
        ids:{
          type: Array
           
        },
        data:{
          type: Object
        },
        client: {
          type: Object,
          notify: true,
          observer: '_clientChanged'              
        },
        fields: {
          type: Array,
          value: function() { return []; }
        },

        filters: {
          type: Array,
          notify: true,
          value: function() { return []; }
        }
      },
    
      observers: [
      '_filtersChange(filters.*)'
      ],
      behaviors: [
        Polymer.IronA11yKeysBehavior
      ],
      keyBindings: {
        'enter': '_search'
      },

      ready: function(){
        //console.log("ready");
      },

      _clientChanged: function() {
        //console.log("ClientChanged");
        ready = true;
        this._query();
      },
      getPeople: function(data){
        var people = []
        //console.log(data)
        data.hits.hits.forEach( function (entry){
          //console.log(entry)
          entry._source.entities.forEach(function(entity){
            //console.log(entity)
            if ('dbo:thumbnail' in entity) {
              people.push({'name': entity['nif:anchorOf'], 'image': entity['dbo:thumbnail']})
            }
          })
        })
        people = this.removeDuplicates(people, 'name')
        console.log(people)
        return people.slice(0,8)
      },
      removeDuplicates: function(originalArray, prop){
        var newArray = [];
        var lookupObject  = {};

         for(var i in originalArray) {
            lookupObject[originalArray[i][prop]] = originalArray[i];
         }

         for(i in lookupObject) {
             newArray.push(lookupObject[i]);
         }
          return newArray;
      },
      getPlaces: function(data){
        var places = []
        data.hits.hits.forEach( function (entry){
          entry._source.entities.forEach(function(entity){
            if ('latitude' in entity) {
              places.push({'lat': entity.latitude, 'lng': entity.longitude, 'name': entry._source['schema:headline']})
            }
          })
        })
        return places
      },
      _search: function(){
        //console.log("search fired")
        //console.log(this.query.length)
        if (this.query.length == 0){
          //console.log("default search fired")
          this._query()
        }
        else this._ESsearch()
      },
      _filtersChange: function() {
        this._query();
       },
      _ESsearch: function() {
        //console.log("_filtersChangedash")
        var that = this;
        //console.log("Ready?: ", ready);
        if(ready){
          this.client.search({
          // undocumented params are appended to the query string
          index: "gsicrawler",
          body: {
            size: 200,
            query: {
              multi_match:{
                query: this.query,
                fields: ['schema:author', 'schema:headline', 'sentiments', 'schema:about', 'entities.nif:anchorOf']
              }
            },
            sort:{'schema:datePublished':{order: "desc"}},
            aggs: {
             type: {
               terms: {
                 field: "@type.keyword",
                 order: {
                   _count: "desc"
                 }
               }
             },
             'schema:author': {
               terms: {
                 field: "schema:author.keyword",
                 order: {
                   _count: "desc"
                 }
               }
             },
             'entities.rdfs:subClassOf': {
               terms: {
                 field: "entities.rdfs:subClassOf.keyword",
                 order: {
                   _count: "desc"
                 } 
               }
             },
             'topics.rdfs:subClassOf': {
               terms: {
                 field: "topics.rdfs:subClassOf.keyword",
                 size: 20,
                 order: {
                   _count: "desc"
                 } 
               }
             },
           sentiment: {
             terms: {
             field: "sentiments.marl:hasPolarity.keyword",
             order: {
               _count: "desc"
               }
             }
             },
          emotion: {
              terms: {
              field: "emotions.onyx:hasEmotion.onyx:hasEmotionCategory.keyword",
             order: {
               _count: "desc"
                     }      
                   }
      },
      'schema:datePublished': {
              date_histogram : {
                field : "schema:datePublished",
                format: "MM-yyyy",
                interval : "month"
            }
       }
        }
      } 
          }).then(function (resp) {
            var myids = []
            resp.hits.hits.forEach(function(entry){myids.push(entry._id)})
            that.ids = myids;
            //console.log(that.ids)
            that.data = resp;
            console.log(that.data);
            
            });
        }
      },
       _query: function() {
        //console.log("_query")
        var that = this;
        //console.log("Ready?: ", ready);
        if(ready){
          this.client.search({
          // undocumented params are appended to the query string
          index: "gsicrawler",
          body: {
            size: 200,
            query: {
              bool: {
                must: this.filters,
              }
            },
            sort:{'schema:datePublished':{order: "desc"}},
            aggs: {
             type: {
               terms: {
                 field: "@type.keyword",
                 order: {
                   _count: "desc"
                 }
               }
             },
             'schema:author': {
               terms: {
                 field: "schema:author.keyword",
                 order: {
                   _count: "desc"
                 }
               }
             },
             'entities.rdfs:subClassOf': {
               terms: {
                 field: "entities.rdfs:subClassOf.keyword",
                 order: {
                   _count: "desc"
                 } 
               }
             },
             'topics.rdfs:subClassOf': {
               terms: {
                 field: "topics.rdfs:subClassOf.keyword",
                 size: 20,
                 order: {
                   _count: "desc"
                 } 
               }
             },
           sentiment: {
             terms: {
             field: "sentiments.marl:hasPolarity.keyword",
             order: {
               _count: "desc"
               }
             }
             },
          emotion: {
              terms: {
              field: "emotions.onyx:hasEmotion.onyx:hasEmotionCategory.keyword",
             order: {
               _count: "desc"
                     }      
                   }
      },
      'schema:datePublished': {
              date_histogram : {
                field : "schema:datePublished",
                format: "MM-yyyy",
                interval : "month"
            }
       }
       }
      } 
          }).then(function (resp) {
            var myids = []
            resp.hits.hits.forEach(function(entry){myids.push(entry._id)})
            that.ids = myids;
            //console.log(that.ids)
            that.data = resp;
            console.log(that.data);
            
            });
        }
      }
    });
  </script>

</dom-module>