Skip to content

belief_value_mapping

BeliefValueMapping

Alternate representation of a value function, particularly for pomdp models. It works by adding adding belief and associated value to the object. To evaluate this version of the value function the sawtooth algorithm is used (described in Shani G. et al., "A survey of point-based POMDP solvers")

We can also compute the Q value for a particular belief b and action using the qva function.

Parameters:

Name Type Description Default
model Model

The model on which the value function applies on

 required
corner_belief_values ValueFunction

A general value function to define the value at corner points in belief space (ie: at certainty beliefs, or when beliefs have a probability of 1 for a given state). This is usually the solution of the MDP version of the problem.

 required

Attributes:

Name Type Description
model Model
corner_belief_values ValueFunction
corner_values ndarray

Array of |S| shape, having the max value at each state based on the corner_belief_values.
beliefs Belief

Beliefs contained in the belief-value mapping.
belief_value_mapping dict[bytes, float]

Mapping of beliefs points with their associated value.
Source code in olfactory_navigation/agents/model_based_util/belief_value_mapping.py 
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
class BeliefValueMapping:
    '''
    Alternate representation of a value function, particularly for pomdp models.
    It works by adding adding belief and associated value to the object.
    To evaluate this version of the value function the sawtooth algorithm is used (described in Shani G. et al., "A survey of point-based POMDP solvers")

    We can also compute the Q value for a particular belief b and action using the qva function.


    Parameters
    ----------
    model : pomdp.Model
        The model on which the value function applies on
    corner_belief_values : ValueFunction
        A general value function to define the value at corner points in belief space (ie: at certainty beliefs, or when beliefs have a probability of 1 for a given state).
        This is usually the solution of the MDP version of the problem.

    Attributes
    ----------
    model : pomdp.Model
    corner_belief_values : ValueFunction
    corner_values : np.ndarray
        Array of |S| shape, having the max value at each state based on the corner_belief_values.
    beliefs : Belief
        Beliefs contained in the belief-value mapping.
    belief_value_mapping : dict[bytes, float]
        Mapping of beliefs points with their associated value.

    '''
    def __init__(self,
                 model: Model,
                 corner_belief_values: ValueFunction
                 ) -> None:
        xp = np if not gpu_support else cp.get_array_module(corner_belief_values.alpha_vector_array)

        self.model = model
        self.corner_belief_values = corner_belief_values

        self.corner_values = xp.max(corner_belief_values.alpha_vector_array, axis=0)

        self.beliefs = []
        self.belief_value_mapping = {}

        self._belief_array = None
        self._value_array = None


    def add(self,
            b: Belief,
            v: float
            ) -> None:
        '''
        Function to a belief point and its associated value to the belief value mappings.

        Parameters
        ----------
        b: Belief
            A belief to add the belief value mappings.
        v: float
            The value associated to the belief to be added to the mappings.
        '''
        if b not in self.beliefs:
            self.beliefs.append(b)
            self.belief_value_mapping[b.bytes_repr] = v


    @property
    def belief_array(self) -> np.ndarray:
        '''
        The beliefs represented in the form of an array.
        '''
        xp = np if not gpu_support else cp.get_array_module(self.beliefs[0].values)

        if self._belief_array is None:
            self._belief_array = xp.array([b.values for b in self.beliefs])

        return self._belief_array


    @property
    def value_array(self) -> np.ndarray:
        '''
        An array of the values.
        '''
        xp = np if not gpu_support else cp.get_array_module(self.beliefs[0].values)

        if self._value_array is None:
            self._value_array = xp.array(list(self.belief_value_mapping.values()))

        return self._value_array


    def update(self) -> None:
        '''
        Function to update the belief and value arrays to speed up computation.
        '''
        xp = np if not gpu_support else cp.get_array_module(self.beliefs[0].values)

        self._belief_array = xp.array([b.values for b in self.beliefs])
        self._value_array = xp.array(list(self.belief_value_mapping.values()))


    def evaluate(self, belief: Belief) -> float:
        '''
        Runs the sawtooth algorithm to find the value at a given belief point.

        Parameters
        ----------
        belief : Belief
        '''
        xp = np if not gpu_support else cp.get_array_module(belief.values)

        # Shortcut if belief already exists in the mapping
        if belief in self.beliefs:
            return self.belief_value_mapping[belief.bytes_repr]

        v0 = xp.dot(belief.values, self.corner_values)

        if len(self.beliefs) == 0:
            return float(v0)

        with np.errstate(divide='ignore', invalid='ignore'):
            vb = v0 + ((self.value_array - xp.dot(self.belief_array, self.corner_values)) * xp.min(belief.values / self.belief_array, axis=1))

        return float(xp.min(xp.append(vb, v0)))

belief_array: np.ndarray property

The beliefs represented in the form of an array.

value_array: np.ndarray property

An array of the values.

add(b, v)

Function to a belief point and its associated value to the belief value mappings.

Parameters:

Name Type Description Default
b Belief

A belief to add the belief value mappings.

 required
v float

The value associated to the belief to be added to the mappings.

 required
Source code in olfactory_navigation/agents/model_based_util/belief_value_mapping.py 
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
def add(self,
        b: Belief,
        v: float
        ) -> None:
    '''
    Function to a belief point and its associated value to the belief value mappings.

    Parameters
    ----------
    b: Belief
        A belief to add the belief value mappings.
    v: float
        The value associated to the belief to be added to the mappings.
    '''
    if b not in self.beliefs:
        self.beliefs.append(b)
        self.belief_value_mapping[b.bytes_repr] = v

evaluate(belief)

Runs the sawtooth algorithm to find the value at a given belief point.

Parameters:

Name Type Description Default
belief Belief
required
Source code in olfactory_navigation/agents/model_based_util/belief_value_mapping.py 
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
def evaluate(self, belief: Belief) -> float:
    '''
    Runs the sawtooth algorithm to find the value at a given belief point.

    Parameters
    ----------
    belief : Belief
    '''
    xp = np if not gpu_support else cp.get_array_module(belief.values)

    # Shortcut if belief already exists in the mapping
    if belief in self.beliefs:
        return self.belief_value_mapping[belief.bytes_repr]

    v0 = xp.dot(belief.values, self.corner_values)

    if len(self.beliefs) == 0:
        return float(v0)

    with np.errstate(divide='ignore', invalid='ignore'):
        vb = v0 + ((self.value_array - xp.dot(self.belief_array, self.corner_values)) * xp.min(belief.values / self.belief_array, axis=1))

    return float(xp.min(xp.append(vb, v0)))

update()

Function to update the belief and value arrays to speed up computation.

Source code in olfactory_navigation/agents/model_based_util/belief_value_mapping.py 
105
106
107
108
109
110
111
112
def update(self) -> None:
    '''
    Function to update the belief and value arrays to speed up computation.
    '''
    xp = np if not gpu_support else cp.get_array_module(self.beliefs[0].values)

    self._belief_array = xp.array([b.values for b in self.beliefs])
    self._value_array = xp.array(list(self.belief_value_mapping.values()))