Refactoring v0.15.1

See CHANGELOG.md for a full list of changes

* Removed nxsim
* Refactored `agents.NetworkAgent` and `agents.BaseAgent`
* Refactored exporters
* Added stats to history

CHANGELOG.md

@@ -3,6 +3,29 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.15.1]
+### Added
+* read-only `History`
+### Fixed
+* Serialization problem with the `Environment` on parallel mode.
+* Analysis functions now work as they should in the tutorial
+## [0.15.0]
+### Added
+* Control logging level in CLI and simulation
+* `Stats` to calculate trial and simulation-wide statistics
+* Simulation statistics are stored in a separate table in history (see `History.get_stats` and `History.save_stats`, as well as `soil.stats`)
+* Aliased `NetworkAgent.G` to `NetworkAgent.topology`.
+### Changed
+* Templates in config files can be given as dictionaries in addition to strings
+* Samplers are used more explicitly
+* Removed nxsim dependency. We had already made a lot of changes, and nxsim has not been updated in 5 years.
+* Exporter methods renamed to `trial` and `end`. Added `start`.
+* `Distribution` exporter now a stats class
+* `global_topology` renamed to `topology`
+* Moved topology-related methods to `NetworkAgent`
+### Fixed
+* Temporary files used for history in dry_run mode are not longer left open 
+
 ## [0.14.9]
 ### Changed
 * Seed random before environment initialization

docs/conf.py

@@ -31,7 +31,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = []
+extensions = ['IPython.sphinxext.ipython_console_highlighting']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -69,7 +69,7 @@ language = None
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints']
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'

docs/configuration.rst

@@ -218,3 +218,24 @@ These agents are programmed in much the same way as network agents, the only dif
 
 You may use environment agents to model events that a normal agent cannot control, such as natural disasters or chance.
 They are also useful to add behavior that has little to do with the network and the interactions within that network.
+
+Templating
+==========
+
+Sometimes, it is useful to parameterize a simulation and run it over a range of values in order to compare each run and measure the effect of those parameters in the simulation.
+For instance, you may want to run a simulation with different agent distributions.
+
+This can be done in Soil using **templates**.
+A template is a configuration where some of the values are specified with a variable.
+e.g.,  ``weight: "{{ var1 }}"`` instead of ``weight: 1``.
+There are two types of variables, depending on how their values are decided:
+
+* Fixed. A list of values is provided, and a new simulation is run for each possible value. If more than a variable is given, a new simulation will be run per combination of values.
+* Bounded/Sampled. The bounds of the variable are provided, along with a sampler method, which will be used to compute all the configuration combinations.
+
+When fixed and bounded variables are mixed, Soil generates a new configuration per combination of fixed values and bounded values.
+
+Here is an example with a single fixed variable and two bounded variable:
+
+.. literalinclude:: ../examples/template.yml
+   :language: yaml

examples/NewsSpread.ipynb

@@ -500,7 +500,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.5"
+   "version": "3.8.5"
   },
   "toc": {
    "colors": {

examples/Untitled.ipynb

@@ -80800,7 +80800,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.5"
+   "version": "3.8.6"
   }
  },
  "nbformat": 4,

examples/rabbits/rabbit_agents.py

-from soil.agents import FSM, state, default_state, BaseAgent
+from soil.agents import FSM, state, default_state, BaseAgent, NetworkAgent
 from enum import Enum
 from random import random, choice
 from itertools import islice
@@ -80,7 +80,7 @@ class RabbitModel(FSM):
                 self.env.add_edge(self['mate'], child.id)
                 # self.add_edge()
                 self.debug('A BABY IS COMING TO LIFE')
-                self.env['rabbits_alive'] = self.env.get('rabbits_alive', self.global_topology.number_of_nodes())+1
+                self.env['rabbits_alive'] = self.env.get('rabbits_alive', self.topology.number_of_nodes())+1
                 self.debug('Rabbits alive: {}'.format(self.env['rabbits_alive']))
                 self['offspring'] += 1
                 self.env.get_agent(self['mate'])['offspring'] += 1
@@ -97,12 +97,14 @@ class RabbitModel(FSM):
         return
 
 
-class RandomAccident(BaseAgent):
+class RandomAccident(NetworkAgent):
 
     level = logging.DEBUG
 
     def step(self):
-        rabbits_total = self.global_topology.number_of_nodes()
+        rabbits_total = self.topology.number_of_nodes()
+        if 'rabbits_alive' not in self.env:
+            self.env['rabbits_alive'] = 0
         rabbits_alive = self.env.get('rabbits_alive', rabbits_total)
         prob_death = self.env.get('prob_death', 1e-100)*math.floor(math.log10(max(1, rabbits_alive)))
         self.debug('Killing some rabbits with prob={}!'.format(prob_death))
@@ -116,5 +118,5 @@ class RandomAccident(BaseAgent):
                 self.log('Rabbits alive: {}'.format(self.env['rabbits_alive']))
                 i.set_state(i.dead)
         self.log('Rabbits alive: {}/{}'.format(rabbits_alive, rabbits_total))
-        if self.count_agents(state_id=RabbitModel.dead.id) == self.global_topology.number_of_nodes():
+        if self.count_agents(state_id=RabbitModel.dead.id) == self.topology.number_of_nodes():
             self.die()

examples/template.yml

 ---
-vars:
-  bounds:
-    x1: [0, 1]
-    x2: [1, 2]
-  fixed:
-    x3: ["a", "b", "c"]
-sampler: "SALib.sample.morris.sample"
-samples: 10
-template: |
+sampler:
+  method: "SALib.sample.morris.sample"
+  N: 10
+template:
   group: simple
   num_trials: 1
   interval: 1
@@ -19,11 +14,17 @@ template: |
     n: 10
   network_agents:
     - agent_type: CounterModel
-      weight: {{ x1 }}
+      weight: "{{ x1 }}"
       state:
         id: 0
     - agent_type: AggregatedCounter
-      weight: {{ 1 - x1 }}
+      weight: "{{ 1 - x1 }}"
   environment_params:
-    name: {{ x3 }}
+    name: "{{ x3 }}"
   skip_test: true
+vars:
+  bounds:
+    x1: [0, 1]
+    x2: [1, 2]
+  fixed:
+    x3: ["a", "b", "c"]

examples/terrorism/TerroristNetworkModel.py

@@ -195,14 +195,14 @@ class TerroristNetworkModel(TerroristSpreadModel):
                     break
 
     def get_distance(self, target):
-        source_x, source_y = nx.get_node_attributes(self.global_topology, 'pos')[self.id]
-        target_x, target_y = nx.get_node_attributes(self.global_topology, 'pos')[target]
+        source_x, source_y = nx.get_node_attributes(self.topology, 'pos')[self.id]
+        target_x, target_y = nx.get_node_attributes(self.topology, 'pos')[target]
         dx = abs( source_x - target_x )
         dy = abs( source_y - target_y )
         return ( dx ** 2 + dy ** 2 ) ** ( 1 / 2 )
 
     def shortest_path_length(self, target):
         try:
-            return nx.shortest_path_length(self.global_topology, self.id, target)
+            return nx.shortest_path_length(self.topology, self.id, target)
         except nx.NetworkXNoPath:
             return float('inf')

requirements.txt

-nxsim>=0.1.2
-simpy
-networkx>=2.0,<2.4
+simpy>=4.0
+networkx>=2.5
 numpy
 matplotlib
 pyyaml>=5.1

soil/VERSION

-0.14.9
\ No newline at end of file
+0.15.1
\ No newline at end of file

soil/__init__.py

@@ -17,12 +17,12 @@ from .environment import Environment
 from .history import History
 from . import serialization
 from . import analysis
+from .utils import logger
 
 def main():
     import argparse
     from . import simulation
 
-    logging.basicConfig(level=logging.INFO)
     logging.info('Running SOIL version: {}'.format(__version__))
 
     parser = argparse.ArgumentParser(description='Run a SOIL simulation')
@@ -40,6 +40,8 @@ def main():
                         help='Dump GEXF graph. Defaults to false.')
     parser.add_argument('--csv', action='store_true',
                         help='Dump history in CSV format. Defaults to false.')
+    parser.add_argument('--level', type=str,
+                        help='Logging level')
     parser.add_argument('--output', '-o', type=str, default="soil_output",
                         help='folder to write results to. It defaults to the current directory.')
     parser.add_argument('--synchronous', action='store_true',
@@ -48,6 +50,7 @@ def main():
                         help='Export environment and/or simulations using this exporter')
 
     args = parser.parse_args()
+    logging.basicConfig(level=getattr(logging, (args.level or 'INFO').upper()))
 
     if os.getcwd() not in sys.path:
         sys.path.append(os.getcwd())

soil/agents/BassModel.py

@@ -9,7 +9,7 @@ class BassModel(BaseAgent):
         imitation_prob
     """
 
-    def __init__(self, environment, agent_id, state):
+    def __init__(self, environment, agent_id, state, **kwargs):
         super().__init__(environment=environment, agent_id=agent_id, state=state)
         env_params = environment.environment_params
         self.state['sentimentCorrelation'] = 0
@@ -19,7 +19,7 @@ class BassModel(BaseAgent):
 
     def behaviour(self):
         # Outside effects
-        if random.random() < self.state_params['innovation_prob']:
+        if random.random() < self['innovation_prob']:
             if self.state['id'] == 0:
                 self.state['id'] = 1
                 self.state['sentimentCorrelation'] = 1
@@ -32,7 +32,7 @@ class BassModel(BaseAgent):
         if self.state['id'] == 0:
             aware_neighbors = self.get_neighboring_agents(state_id=1)
             num_neighbors_aware = len(aware_neighbors)
-            if random.random() < (self.state_params['imitation_prob']*num_neighbors_aware):
+            if random.random() < (self['imitation_prob']*num_neighbors_aware):
                 self.state['id'] = 1
                 self.state['sentimentCorrelation'] = 1
 

soil/agents/CounterModel.py

-from . import BaseAgent
+from . import NetworkAgent
 
 
-class CounterModel(BaseAgent):
+class CounterModel(NetworkAgent):
     """
     Dummy behaviour. It counts the number of nodes in the network and neighbors
     in each step and adds it to its state.
@@ -9,14 +9,14 @@ class CounterModel(BaseAgent):
 
     def step(self):
         # Outside effects
-        total = len(list(self.get_all_agents()))
+        total = len(list(self.get_agents()))
         neighbors = len(list(self.get_neighboring_agents()))
         self['times'] = self.get('times', 0) + 1
         self['neighbors'] = neighbors
         self['total'] = total
 
 
-class AggregatedCounter(BaseAgent):
+class AggregatedCounter(NetworkAgent):
     """
     Dummy behaviour. It counts the number of nodes in the network and neighbors
     in each step and adds it to its state.
@@ -33,6 +33,6 @@ class AggregatedCounter(BaseAgent):
         self['times'] += 1
         neighbors = len(list(self.get_neighboring_agents()))
         self['neighbors'] += neighbors
-        total = len(list(self.get_all_agents()))
+        total = len(list(self.get_agents()))
         self['total'] += total
         self.debug('Running for step: {}. Total: {}'.format(self.now, total))

soil/agents/__init__.py

@@ -3,19 +3,19 @@
 # for x in range(0, settings.network_params["number_of_nodes"]):
 #     sentimentCorrelationNodeArray.append({'id': x})
 # Initialize agent states. Let's assume everyone is normal.
-    
 
-import nxsim
+
 import logging
 from collections import OrderedDict
 from copy import deepcopy
 from functools import partial
 from scipy.spatial import cKDTree as KDTree
 import json
+import simpy
 
 from functools import wraps
 
-from .. import serialization, history
+from .. import serialization, history, utils
 
 
 def as_node(agent):
@@ -24,7 +24,7 @@ def as_node(agent):
     return agent
 
 
-class BaseAgent(nxsim.BaseAgent):
+class BaseAgent:
     """
     A special simpy BaseAgent that keeps track of its state history.
     """
@@ -32,14 +32,13 @@ class BaseAgent(nxsim.BaseAgent):
     defaults = {}
 
     def __init__(self, environment, agent_id, state=None,
-                 name=None, interval=None, **state_params):
+                 name=None, interval=None):
         # Check for REQUIRED arguments
         assert environment is not None, TypeError('__init__ missing 1 required keyword argument: \'environment\'. '
                                                   'Cannot be NoneType.')
         # Initialize agent parameters
         self.id = agent_id
         self.name = name or '{}[{}]'.format(type(self).__name__, self.id)
-        self.state_params = state_params
 
         # Register agent to environment
         self.env = environment
@@ -51,10 +50,10 @@ class BaseAgent(nxsim.BaseAgent):
         self.state = real_state
         self.interval = interval
 
-        if not hasattr(self, 'level'):
-            self.level = logging.DEBUG
-        self.logger = logging.getLogger(self.env.name)
-        self.logger.setLevel(self.level)
+        self.logger = logging.getLogger(self.env.name).getChild(self.name)
+
+        if hasattr(self, 'level'):
+            self.logger.setLevel(self.level)
 
         # initialize every time an instance of the agent is created
         self.action = self.env.process(self.run())
@@ -75,14 +74,10 @@ class BaseAgent(nxsim.BaseAgent):
         for k, v in value.items():
             self[k] = v
 
-    @property
-    def global_topology(self):
-        return self.env.G
-    
     @property
     def environment_params(self):
         return self.env.environment_params
-    
+
     @environment_params.setter
     def environment_params(self, value):
         self.env.environment_params = value
@@ -135,36 +130,10 @@ class BaseAgent(nxsim.BaseAgent):
     def die(self, remove=False):
         self.alive = False
         if remove:
-            super().die()
+            self.remove_node(self.id)
 
     def step(self):
-        pass
-
-    def count_agents(self, **kwargs):
-        return len(list(self.get_agents(**kwargs)))
-
-    def count_neighboring_agents(self, state_id=None, **kwargs):
-        return len(super().get_neighboring_agents(state_id=state_id, **kwargs))
-
-    def get_neighboring_agents(self, state_id=None, **kwargs):
-        return self.get_agents(limit_neighbors=True, state_id=state_id, **kwargs)
-
-    def get_agents(self, agents=None, limit_neighbors=False, **kwargs):
-        if limit_neighbors:
-            agents = super().get_agents(limit_neighbors=limit_neighbors)
-        else:
-            agents = self.env.get_agents(agents)
-        return select(agents, **kwargs)
-
-    def log(self, message, *args, level=logging.INFO, **kwargs):
-        message = message + " ".join(str(i) for i in args)
-        message = "\t{:10}@{:>5}:\t{}".format(self.name, self.now, message)
-        for k, v in kwargs:
-            message += " {k}={v} ".format(k, v)
-        extra = {}
-        extra['now'] = self.now
-        extra['id'] = self.id
-        return self.logger.log(level, message, extra=extra)
+        return
 
     def debug(self, *args, **kwargs):
         return self.log(*args, level=logging.DEBUG, **kwargs)
@@ -192,24 +161,59 @@ class BaseAgent(nxsim.BaseAgent):
         self._state = state['_state']
         self.env = state['environment']
 
-    def add_edge(self, node1, node2, **attrs):
-        node1 = as_node(node1)
-        node2 = as_node(node2)
+class NetworkAgent(BaseAgent):
 
-        for n in [node1, node2]:
-            if n not in self.global_topology.nodes(data=False):
-                raise ValueError('"{}" not in the graph'.format(n))
-        return self.global_topology.add_edge(node1, node2, **attrs)
+    @property
+    def topology(self):
+        return self.env.G
+
+    @property
+    def G(self):
+        return self.env.G
+
+    def count_agents(self, **kwargs):
+        return len(list(self.get_agents(**kwargs)))
+
+    def count_neighboring_agents(self, state_id=None, **kwargs):
+        return len(self.get_neighboring_agents(state_id=state_id, **kwargs))
+
+    def get_neighboring_agents(self, state_id=None, **kwargs):
+        return self.get_agents(limit_neighbors=True, state_id=state_id, **kwargs)
+
+    def get_agents(self, agents=None, limit_neighbors=False, **kwargs):
+        if limit_neighbors:
+            agents = self.topology.neighbors(self.id)
+
+        agents = self.env.get_agents(agents)
+        return select(agents, **kwargs)
+
+    def log(self, message, *args, level=logging.INFO, **kwargs):
+        message = message + " ".join(str(i) for i in args)
+        message = " @{:>3}: {}".format(self.now, message)
+        for k, v in kwargs:
+            message += " {k}={v} ".format(k, v)
+        extra = {}
+        extra['now'] = self.now
+        extra['agent_id'] = self.id
+        extra['agent_name'] = self.name
+        return self.logger.log(level, message, extra=extra)
 
     def subgraph(self, center=True, **kwargs):
         include = [self] if center else []
-        return self.global_topology.subgraph(n.id for n in self.get_agents(**kwargs)+include)
+        return self.topology.subgraph(n.id for n in self.get_agents(**kwargs)+include)
 
+    def remove_node(self, agent_id):
+        self.topology.remove_node(agent_id)
 
-class NetworkAgent(BaseAgent):
+    def add_edge(self, other, edge_attr_dict=None, *edge_attrs):
+        # return super(NetworkAgent, self).add_edge(node1=self.id, node2=other, **kwargs)
+        if self.id not in self.topology.nodes(data=False):
+            raise ValueError('{} not in list of existing agents in the network'.format(self.id))
+        if other not in self.topology.nodes(data=False):
+            raise ValueError('{} not in list of existing agents in the network'.format(other))
+
+        self.topology.add_edge(self.id, other, edge_attr_dict=edge_attr_dict, *edge_attrs)
 
-    def add_edge(self, other, **kwargs):
-        return super(NetworkAgent, self).add_edge(node1=self.id, node2=other, **kwargs)
 
     def ego_search(self, steps=1, center=False, node=None, **kwargs):
         '''Get a list of nodes in the ego network of *node* of radius *steps*'''
@@ -220,14 +224,14 @@ class NetworkAgent(BaseAgent):
     def degree(self, node, force=False):
         node = as_node(node)
         if force or (not hasattr(self.env, '_degree')) or getattr(self.env, '_last_step', 0) < self.now:
-            self.env._degree = nx.degree_centrality(self.global_topology)
+            self.env._degree = nx.degree_centrality(self.topology)
             self.env._last_step = self.now
         return self.env._degree[node]
 
     def betweenness(self, node, force=False):
         node = as_node(node)
         if force or (not hasattr(self.env, '_betweenness')) or getattr(self.env, '_last_step', 0) < self.now:
-            self.env._betweenness = nx.betweenness_centrality(self.global_topology)
+            self.env._betweenness = nx.betweenness_centrality(self.topology)
             self.env._last_step = self.now
         return self.env._betweenness[node]
 
@@ -292,16 +296,22 @@ class MetaFSM(type):
         cls.states = states
 
 
-class FSM(BaseAgent, metaclass=MetaFSM):
+class FSM(NetworkAgent, metaclass=MetaFSM):
     def __init__(self, *args, **kwargs):
         super(FSM, self).__init__(*args, **kwargs)
         if 'id' not in self.state:
             if not self.default_state:
                 raise ValueError('No default state specified for {}'.format(self.id))
             self['id'] = self.default_state.id
+        self._next_change = simpy.core.Infinity
+        self._next_state = self.state
 
     def step(self):
-        if 'id' in self.state:
+        if self._next_change < self.now:
+            next_state = self._next_state
+            self._next_change = simpy.core.Infinity
+            self['id'] = next_state
+        elif 'id' in self.state:
             next_state = self['id']
         elif self.default_state:
             next_state = self.default_state.id
@@ -311,6 +321,10 @@ class FSM(BaseAgent, metaclass=MetaFSM):
             raise Exception('{} is not a valid id for {}'.format(next_state, self))
         return self.states[next_state](self)
 
+    def next_state(self, state):
+        self._next_change = self.now
+        self._next_state = state
+
     def set_state(self, state):
         if hasattr(state, 'id'):
             state = state.id
@@ -371,14 +385,18 @@ def calculate_distribution(network_agents=None,
     else:
         raise ValueError('Specify a distribution or a default agent type')
 
+    # Fix missing weights and incompatible types
+    for x in network_agents:
+        x['weight'] = float(x.get('weight', 1))
+
     # Calculate the thresholds
-    total = sum(x.get('weight', 1) for x in network_agents)
+    total = sum(x['weight'] for x in network_agents)
     acc = 0
     for v in network_agents:
         if 'ids' in v:
             v['threshold'] = STATIC_THRESHOLD
             continue
-        upper = acc + (v.get('weight', 1)/total)
+        upper = acc + (v['weight']/total)
         v['threshold'] = [acc, upper]
         acc = upper
     return network_agents
@@ -425,7 +443,7 @@ def _validate_states(states, topology):
     states = states or []
     if isinstance(states, dict):
         for x in states:
-            assert x in topology.node
+            assert x in topology.nodes
     else:
         assert len(states) <= len(topology)
     return states

soil/analysis.py

@@ -28,13 +28,13 @@ def _read_data(pattern, *args, from_csv=False, process_args=None, **kwargs):
                 df = read_csv(trial_data, **kwargs)
                 yield config_file, df, config
         else:
-            for trial_data in sorted(glob.glob(join(folder, '*.db.sqlite'))):
+            for trial_data in sorted(glob.glob(join(folder, '*.sqlite'))):
                 df = read_sql(trial_data, **kwargs)
                 yield config_file, df, config
 
 
 def read_sql(db, *args, **kwargs):
-    h = history.History(db_path=db, backup=False)
+    h = history.History(db_path=db, backup=False, readonly=True)
     df = h.read_sql(*args, **kwargs)
     return df
 
@@ -69,6 +69,13 @@ def convert_types_slow(df):
     df = df.apply(convert_row, axis=1)
     return df
 
+
+def split_processed(df):
+    env = df.loc[:, df.columns.get_level_values(1).isin(['env', 'stats'])]
+    agents = df.loc[:, ~df.columns.get_level_values(1).isin(['env', 'stats'])]
+    return env, agents
+
+
 def split_df(df):
     '''
     Split a dataframe in two dataframes: one with the history of agents,
@@ -136,7 +143,7 @@ def get_value(df, *keys, aggfunc='sum'):
     return df.groupby(axis=1, level=0).agg(aggfunc)
 
 
-def plot_all(*args, **kwargs):
+def plot_all(*args, plot_args={}, **kwargs):
     '''
     Read all the trial data and plot the result of applying a function on them.
     '''
@@ -144,14 +151,17 @@ def plot_all(*args, **kwargs):
     ps = []