dashboard-gsicrawler.env.html

<link rel="import" href="/bower_components/polymer/polymer.html">
<link rel="import" href="/bower_components/material-search/material-search.html">
<link rel="import" href="/bower_components/paper-material/paper-material.html">
<link rel="import" href="/elements/google-chart-elasticsearch/google-chart.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tab.html">
<link rel="import" href="/elements/number-chart/number-chart.html">
<link rel="import" href="/bower_components/comment-chart/comment-chart.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/yasgui-polymer/yasgui.html">
<link rel="import" href="/bower_components/iron-pages/iron-pages.html">
<link rel="import" href="/bower_components/paper-item/paper-item.html">
<link rel="import" href="/bower_components/paper-menu/paper-menu.html">
<link rel="import" href="/bower_components/paper-autocomplete/paper-autocomplete.html">
<link rel="import" href="/elements/entities-chart/entities-chart.html">
<link rel="import" href="/elements/people-chart/people-chart.html">
<link rel="import" href="/elements/news-chart/news-chart.html">
<link rel="import" href="/elements/propaganda-chart/propaganda-chart.html">
<link rel="import" href="/elements/tweet-chart/tweet-chart.html">
<link rel="import" href="/elements/annotate-view/annotate-view.html">
<link rel="import" href="/elements/happymap/happymap-element.html">

<!-- <link href="dist/css/fs-modal.min.css" rel="stylesheet"> -->
<!-- <script src="dist/js/fs-modal.min.js"></script> -->

<!-- Bootstrap -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.2.0/js/bootstrap.min.js"></script>
<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">

<!--<script src="https://hypothes.is/embed.js" async></script>-->

<dom-module id="dashboard-gsicrawler">

  <link rel="import" type="css" href="dashboard-gsicrawler.css">
  <link rel="import" type="text/html" href="/styles/app-theme.html">

  <template>
    <iron-ajax auto
        url="/queries.json"
        handle-as="json"
        last-response="{{queries}}"></iron-ajax>
    <iron-ajax auto
        url="/sources.json"
        handle-as="json"
        on-response="sourceResponse"></iron-ajax>

    <!--
    <div style="width:10%; padding: 1em; display:inline-block">
      <img style="width: 100%;"src="/images/trivalent.jpg">
    </div>
    -->
    <div class="container">
    <div class="header" style="width:100%; padding-bottom: 20px; text-align:center; display:inline-block">
      <img class="trivalent" src="/images/trivalent.jpg">
    </div>

    <paper-tabs selected="{{selected}}">
      <paper-tab>Dashboard</paper-tab>
      <paper-tab>Sparql Editor</paper-tab>
      <paper-tab>About</paper-tab>
    </paper-tabs>

   <iron-pages selected="{{selected}}">
      <div>
        <div class="row">
          <material-search active="true" search-value="{{query}}" style="margin: 0 15px;"></material-search>
        </div>
        <br>
        <div class="row">
          <div class="col-md-6 col-xs-12 col-sm-6">
            <number-chart
               data="{{data}}"
               object="schema:NewsArticle"
               aggKey="type"
               title="News"
               icon="/images/news.svg"
               subtitle="Total"
               stylebg="bg-yellow">
             </number-chart>
          </div>
          <div class="col-md-6 col-xs-12 col-sm-6">
             <number-chart
               data="{{data}}"
               aggKey="type"
               object="schema:Article"
               title="Magazines"
               icon="/images/magazine.svg"
               subtitle="Total"
               stylebg="bg-purple">
             </number-chart>
          </div>
        </div>
         <br>
     <div class="row">
         <div class="col-md-6 col-xs-12 margin-bottom-phone">
            <entities-chart
                field="entities.name"
                data="{{data}}"
                title="Entities"
                icon="icons:list"
                param="{{param}}"
                id="entitieschart"
                filters="{{filters}}"
                legend='[{"name":"Organization","type":"schema:Organization","color":"#dd4b39"},
                          {"name":"Place","type":"schema:Place","color":"#ff9900"},
                          {"name": "Person","type":"schema:Person","color":"#00c0ef"}]'>
            </entities-chart>
         </div>
          <div class="col-md-6 col-xs-12 margin-bottom-phone">
            <entities-chart
                field="categories.name"
                data="{{data}}"
                title="Categories"
                icon="icons:list"
                param="{{param}}"
                id="categorieschart"
                color = "#505050"
                filters="{{filters}}">
            </entities-chart>

          </div>
      </div>

      <br>
      <div class="row">
        <div class="col-md-6 col-xs-12 margin-bottom-phone">
          <google-chart
              field="schema:author"
              data="{{data}}"
              sources="{{sources}}"
              id='pie-chart3'
              extra-id='pie-chart3'
              type='pie'
              filters="{{filters}}"
              icon='icons:reorder'
              options='{"title": "Source"}'
              cols='[{"label": "type", "type": "string"},{"label": "Count", "type": "number"}]'>
            </google-chart>
        </div>
        <div class="col-md-6 col-xs-12 margin-bottom-phone">
          <people-chart
              data="{{data}}"
              title="People"
              icon="maps:person-pin"
              param="{{param}}"
              id="entitieschart"
              filters="{{filters}}">
          </people-chart>

        </div>
      </div>
      <br>
      <div class="row">
        <div class="col-md-12 col-xs-12 margin-bottom-phone">
          <google-chart
            field="schema:datePublished"
            data="{{data}}"
            id='line-chart'
            extra-id='line-chart'
            type='line'
            filters="{{filters}}"
            sources="{{sources}}"
            icon='icons:trending-up'
            options='{"title": "Temporal Trends"}'
            optionsbi='{"legend": { "position": "bottom"}}'>
          </google-chart>
        </div>
      </div>
      <br>
      <div class="row">
      <div class="col-md-6 col-xs-12 margin-bottom-phone">
        <paper-tabs selected="{{selectedpage}}" style="height: 38px">
          <paper-tab><iron-icon icon="icons:receipt"></iron-icon> News</paper-tab>
<!--           <paper-tab><iron-icon icon="icons:speaker-notes"></iron-icon> Social Media</paper-tab>
 -->          <paper-tab><iron-icon icon="icons:speaker-notes"></iron-icon> Magazines</paper-tab>
        </paper-tabs>
        <iron-pages class="lists" selected="{{selectedpage}}" style="padding-top: 0px !important;">
        <div>
          <news-chart
            datos = "{{data}}"
            title="List of news"
            filters="{{filters}}"
            icon='icons:receipt'>
          </news-chart>
        </div>
        <!-- <div>
          <tweet-chart
            datos = "{{data}}"
            title="List of tweets"
            filters="{{filters}}"
            icon='communication:comment'>
          </tweet-chart>
        </div> -->
        <div>
          <propaganda-chart
            datos = "{{data}}"
            title="List of propaganda"
            filters="{{filters}}"
            icon='communication:comment'>
          </propaganda-chart>
        </div>

        </iron-pages>
       </div>
       <div class="col-md-6 col-xs-12 margin-bottom-phone">
        <div class="top-bar">
        <iron-icon icon="maps:my-location"></iron-icon>
          <span>Geolocated News</span>
        </div>
        <!--
        <leaflet-map fit-to-markers>
          <template is="dom-repeat" items="[[getPlaces(data)]]" as="place">
                <leaflet-marker latitude="{{place.lat}}" longitude="{{place.lng}}" title="{{place.name}}">
                    <b>Headline:</b>
                    <span>{{place.name}}</span>
                </leaflet-marker>
              </template>
        </leaflet-map>
        -->
        <happymap-element
          data="{{data}}",
          '<!-- visalized data in case of connection error -->'>

        </happymap-element>
       </div>
      </div>

    </div>
    <div>
        <yasgui-ui
       endpoint="http://sefarad-fuseki.cluster.gsi.dit.upm.es/gsicrawler/query"
       queries="{{queries}}">
        </yasgui-ui>
    </div>
    <div class="about-info">


This demo shows GSI-Crawler functionalities in the context of the project Trivalent. The system is aimed at crawling from a variety of data sources including news web sites such as CNN and New York Times, as well as social media platforms, like Twitter. The presented demo is the result of the ingest of the crawled online data and posterior analysis and semantic augmentation of said data. The resulting information is stored in both ElastiSearch and Fuseki databases, which allows the user to consult the data is a variety of forms, as showed.

<br><br>
For a quick overview of the funtionalities of the demo, please watch the following video:
<br><br>
<iframe width="560" height="315" src="https://www.youtube.com/embed/3s894sjevBQ" frameborder="0" allowfullscreen></iframe>
<br><br>
To discover more, please consult the documentation at <a href="http://gsicrawler.readthedocs.io" target="_blank">Read The Docs</a>
<br><br>
H2020 RIA Action Grant No. 740934 under the call SEC-06-FCT-2016 — Developing a comprehensive approach to violent radicalization in the EU from early understanding to improving protection.
<br><br>
    </div>
      <annotate-view name="annotate-view"></annotate-view>
    </iron-pages>
    </div>
         <div style="background-color:#ccc; padding: 1%; margin: -16px;margin-top: 20px;">
        <div  class="footer-gsi">
          <img style="width: 100%; height: 50px; object-fit: contain; margin-bottom: 5px;" src="/images/gsi.png">
          <div style="display: flex; flex-direction: column; justify-content: center; font-size: 12px; text-align: center;">© 2019 Grupo de Sistemas Inteligentes GSI-UPM </div>
        </div>
       </div>




  </template>


  <script>
  var ready = false;
    Polymer({
      is: 'dashboard-gsicrawler',
      properties: {
        selected: {
          type: Number,
          value: 0
        },
        selectedpage: {
          type: Number,
          value: 0
        },
        query: {
          type: String,
        },
        ids:{
          type: Array

        },
        data:{
          type: Object
        },
        client: {
          type: Object,
          notify: true,
          observer: '_clientChanged'
        },
        fields: {
          type: Array,
          value: function() { return []; }
        },
        search:{
          type:Object
        },
        filters: {
          type: Array,
          notify: true,
          value: function() { return []; }
        }
      },

      observers: [
      '_filtersChange(filters.*)',
      'entityselection(selectedentity)'
      ],
      behaviors: [
        Polymer.IronA11yKeysBehavior
      ],
      keyBindings: {
        'enter': '_search'
      },

      ready: function(){
        //console.log("ready");
      },

      _clientChanged: function() {
        //console.log("ClientChanged");
        ready = true;
        this._query();
      },

      getEntities: function(datos){
            console.log("getEntities")
            if(this.entityselection){
              var entities = []
              var hits = this.data.aggregations['entities.name'].buckets;
              console.log(hits)
              hits.forEach(function(entry){
                entities.push(entry.key.split('/')[4]);
              });
              console.log(entities)
              return entities
            }
          },

      entityselection: function(entity){
        console.log("entityselection")
        console.log(entity)
        if(entity == null){
          this.filters = [];
        } else this.push('filters', { terms: { 'entities.schema:name': entity.toLowerCase().split(" ")}})
      },

      getPlaces: function(data){
        var places = []
        console.log("getPlace")
        data.hits.hits.forEach( function (entry){
          entry._source.entities.forEach(function(entity){
            if ('latitude' in entity) {
              places.push({'lat': entity.latitude, 'lon': entity.longitude, 'name': entry._source['http://schema.org/headline'][0]['@value']})
            }
          })
        })
        return places
      },
      sourceResponse:function(e){
        //console.log(e.detail.response[0])
        this.sources=e.detail.response[0]
        //console.log(this.sources)
      },
      _search: function(){
        console.log("search fired")
        console.log(this.selectedentity)

        if (this.query.length == 0){
          //console.log("default search fired")
          this.filters = []
          this._query()
        }
        else {
          console.log("Filter")
          this.push('filters', {terms: {'schema:articleBody': this.query.toLowerCase().split(" ")}});
          //this.search={"schema:articleBody":{"query":this.query,"operator":"and"}}
          this._query()
        }

      },
      _filtersChange: function() {
        this._query();
       },
       _query: function() {
        //console.log("_query")
        var that = this;
        //console.log("Ready?: ", ready);
        if(ready){
          this.client.search({
          // undocumented params are appended to the query string
          index: "gsicrawler",
          body: {
            size: 200,
            query: {
              bool: {
                must: this.filters,
              }/*,
              match: this.search*/
            },
            aggs: {
             type: {
               terms: {
                 field: "@type.keyword",
                 order: {
                   _count: "desc"
                 }
               }
             },
             'schema:author': {
               terms: {
                 field: "schema:author.keyword",
                 order: {
                   _count: "desc"
                 }
               }
             },
             'entities.name': {
                terms: {
                  field: "entities.schema:name.keyword",
                  size : 16,
                  order: {
                    _count: "desc"
                  }
                },
                aggs: {
                  'entity_type': {
                      "top_hits": {
                          "_source": {
                              "includes": [ "entities.@type","entities.schema:name"]
                          },
                          "size" : 1
                      }
                  }
                }
             },
             'categories.name': {
                terms: {
                  field: "taxonomies.rdfs:label.keyword",
                  size:10,
                  order: {
                    _count:"desc"
                  }
                }
             },
             sentiment: {
               terms: {
                 field: "sentiments.marl:hasPolarity.keyword",
                 order: {
                   _count: "desc"
                   }
               }
              },
              'schema:datePublished': {
                date_histogram : {
                  field : "schema:datePublished",
                  format: "MM-yyyy",
                  interval : "month"
                },
                aggs:{
                  source:{
                    terms: {
                     field: "schema:author.keyword",
                     order: {
                       _count: "desc"
                     }
                    }
                  }
                }
              }
          }
        }
        }).then(function (resp) {
          var myids = []
          resp.hits.hits.forEach(function(entry){myids.push(entry._id)})
          that.ids = myids;

          that.data = resp;

          //console.log(that.data);

          });
        }
      }
    });


  </script>

</dom-module>