Skip to content

GitLab

P
pymdptoolbox
Commit 6ad0c1a6 authored by Steven Cordwell's avatar Steven Cordwell
Browse files
Options

[util] Make util module more modular 

Seperate out functionality into more modular and single purpose
functions. Move exception classes to own module for this purpose. Add
new user facing functions for checking MDPs.
parent 5af7d2a5
Hide whitespace changes
Inline Side-by-side
Showing with 315 additions and 205 deletions
src/mdptoolbox/error.py 0 → 100644
View file @ 6ad0c1a6
# -*- coding: utf-8 -*-
"""Markov Decision Process (MDP) Toolbox: ``error`` module
=======================================================
The ``error`` module provides exception classes that can be raised by
the toolbox.
Available classes
-----------------
Error
Base exception class derived from ``Exception``
InvalidError
Exception for invalid definitions of an MDP
NonNegativeError
Exception for transition matrices that have negative elements
SquareError
Exception for transition matrices that are not square
StochasticError
Exception for transition matrices that are not stochastic
"""
# Copyright (c) 2015 Steven A. W. Cordwell
# Copyright (c) 2009 INRA
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# * Neither the name of the <ORGANIZATION> nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
class Error(Exception):
"""Base class for exceptions in this module."""
def __init__(self):
Exception.__init__(self)
self.message = "PyMDPToolbox - "
def __str__(self):
return repr(self.message)
class InvalidError(Error):
"""Class for invalid definitions of a MDP."""
def __init__(self, msg):
Error.__init__(self)
self.message += msg
self.args = tuple(msg)
class NonNegativeError(Error):
"""Class for transition matrix stochastic errors"""
default_msg = "The transition probability matrix is negative."
def __init__(self, msg=None):
if msg is None:
msg = self.default_msg
Error.__init__(self)
self.message += msg
self.args = tuple(msg)
class SquareError(Error):
"""Class for transition matrix square errors"""
default_msg = "The transition probability matrix is not square."
def __init__(self, msg=None):
if msg is None:
msg = self.default_msg
Error.__init__(self)
self.message += msg
self.args = tuple(msg)
class StochasticError(Error):
"""Class for transition matrix stochastic errors"""
default_msg = "The transition probability matrix is not stochastic."
def __init__(self, msg=None):
if msg is None:
msg = self.default_msg
Error.__init__(self)
self.message += msg
self.args = tuple(msg)
src/mdptoolbox/util.py
View file @ 6ad0c1a6
# -*- coding: utf-8 -*-
"""Markov Decision Process (MDP) Toolbox: ``utils`` module
=======================================================
"""Markov Decision Process (MDP) Toolbox: ``util`` module
======================================================
The ``utils`` module provides functions to check that an MDP is validly
The ``util`` module provides functions to check that an MDP is validly
described. There are also functions for working with MDPs while they are being
solved.
......@@ -14,10 +14,16 @@ checkSquareStochastic
Check that a matrix is square and stochastic
getSpan
Calculate the span of an array
isNonNegative
Check if a matrix has only non-negative elements
isSquare
Check if a matrix is square
isStochastic
Check if a matrix is row stochastic
"""
# Copyright (c) 2011-2013 Steven A. W. Cordwell
# Copyright (c) 2011-2015 Steven A. W. Cordwell
# Copyright (c) 2009 INRA
#
# All rights reserved.
......@@ -48,8 +54,9 @@ getSpan
import numpy as _np
# These need to be fixed so that we use classes derived from Error.
MDPERR = {
import mdptoolbox.error as _error
_MDPERR = {
"mat_nonneg" :
"Transition probabilities must be non-negative.",
"mat_square" :
......@@ -82,12 +89,128 @@ MDPERR = {
"actions greater than 0. i.e. R.shape = (S, A) or (A, S, S)."
}
def _checkDimensionsListLike(arrays):
"""Check that each array in a list of arrays has the same size.
"""
dim1 = len(arrays)
dim2, dim3 = arrays[0].shape
for aa in range(1, dim1):
dim2_aa, dim3_aa = arrays[aa].shape
if (dim2_aa != dim2) or (dim3_aa != dim3):
raise _error.InvalidError(_MDPERR["obj_square"])
return dim1, dim2, dim3
def _checkRewardsListLike(reward, n_actions, n_states):
"""Check that a list-like reward input is valid.
"""
try:
lenR = len(reward)
if lenR == n_actions:
dim1, dim2, dim3 = _checkDimensionsListLike(reward)
elif lenR == n_states:
dim1 = n_actions
dim2 = dim3 = lenR
else:
raise _error.InvalidError(_MDPERR["R_shape"])
except AttributeError:
raise _error.InvalidError(_MDPERR["R_shape"])
return dim1, dim2, dim3
def isSquare(matrix):
"""Check that ``matrix`` is square.
Returns
=======
is_square : bool
``True`` if ``matrix`` is square, ``False`` otherwise.
"""
try:
try:
dim1, dim2 = matrix.shape
except AttributeError:
dim1, dim2 = _np.array(matrix).shape
except ValueError:
return False
if dim1 == dim2:
return True
return False
def isStochastic(matrix):
"""Check that ``matrix`` is row stochastic.
Returns
=======
is_stochastic : bool
``True`` if ``matrix`` is row stochastic, ``False`` otherwise.
"""
try:
absdiff = (_np.abs(matrix.sum(axis=1) - _np.ones(matrix.shape[0])))
except AttributeError:
matrix = _np.array(matrix)
absdiff = (_np.abs(matrix.sum(axis=1) - _np.ones(matrix.shape[0])))
return (absdiff.max() <= 10*_np.spacing(_np.float64(1)))
def isNonNegative(matrix):
"""Check that ``matrix`` is row non-negative.
Returns
=======
is_stochastic : bool
``True`` if ``matrix`` is non-negative, ``False`` otherwise.
"""
try:
if (matrix >= 0).all():
return True
except (NotImplementedError, AttributeError, TypeError):
try:
if (matrix.data >= 0).all():
return True
except AttributeError:
matrix = _np.array(matrix)
if (matrix.data >= 0).all():
return True
return False
def checkSquareStochastic(matrix):
"""Check if ``matrix`` is a square and row-stochastic.
To pass the check the following conditions must be met:
* The matrix should be square, so the number of columns equals the
number of rows.
* The matrix should be row-stochastic so the rows should sum to one.
* Each value in the matrix must be positive.
If the check does not pass then a mdptoolbox.util.Invalid
Arguments
---------
``matrix`` : numpy.ndarray, scipy.sparse.*_matrix
A two dimensional array (matrix).
Notes
-----
Returns None if no error has been detected, else it raises an error.
"""
if not isSquare(matrix):
raise _error.SquareError
if not isStochastic(matrix):
raise _error.StochasticError
if not isNonNegative(matrix):
raise _error.NonNegativeError
def check(P, R):
"""Check if ``P`` and ``R`` define a valid Markov Decision Process (MDP).
Let ``S`` = number of states, ``A`` = number of actions.
Parameters
Arguments
---------
P : array
The transition matrices. It can be a three dimensional array with
......@@ -116,7 +239,7 @@ def check(P, R):
>>> mdptoolbox.util.check(P_invalid, R_valid) # Raises an exception
Traceback (most recent call last):
...
AssertionError: Each row of a transition probability matrix must sum to one (1).
StochasticError:...
"""
# Checking P
......@@ -124,192 +247,51 @@ def check(P, R):
if P.ndim == 3:
aP, sP0, sP1 = P.shape
elif P.ndim == 1:
# A hack so that we can go into the next try-except statement and
# continue checking from there
raise AttributeError
aP, sP0, sP1 = _checkDimensionsListLike(P)
else:
raise InvalidMDPError(MDPERR["P_shape"])
raise _error.InvalidError(_MDPERR["P_shape"])
except AttributeError:
try:
aP = len(P)
sP0, sP1 = P[0].shape
for aa in range(1, aP):
sP0aa, sP1aa = P[aa].shape
if (sP0aa != sP0) or (sP1aa != sP1):
raise InvalidMDPError(MDPERR["obj_square"])
aP, sP0, sP1 = _checkDimensionsListLike(P)
except AttributeError:
raise InvalidMDPError(MDPERR["P_shape"])
raise _error.InvalidError(_MDPERR["P_shape"])
msg = ""
if aP <= 0:
msg = "The number of actions in P must be greater than 0."
elif sP0 <= 0:
msg = "The number of states in P must be greater than 0."
if msg:
raise _error.InvalidError(msg)
# Checking R
try:
ndimR = R.ndim
if ndimR == 1:
# A hack so that we can go into the next try-except statement
raise AttributeError
aR, sR0, sR1 = _checkRewardsListLike(R, aP, sP0)
elif ndimR == 2:
sR0, aR = R.shape
sR1 = sR0
elif ndimR == 3:
aR, sR0, sR1 = R.shape
else:
raise InvalidMDPError(MDPERR["R_shape"])
raise _error.InvalidError(_MDPERR["R_shape"])
except AttributeError:
try:
lenR = len(R)
if lenR == aP:
aR = lenR
sR0, sR1 = R[0].shape
for aa in range(1, aR):
sR0aa, sR1aa = R[aa].shape
if (sR0aa != sR0) or (sR1aa != sR1):
raise InvalidMDPError(MDPERR["obj_square"])
elif lenR == sP0:
aR = aP
sR0 = sR1 = lenR
else:
raise InvalidMDPError(MDPERR["R_shape"])
except AttributeError:
raise InvalidMDPError(MDPERR["R_shape"])
# Checking dimensions
assert sP0 > 0, "The number of states in P must be greater than 0."
assert aP > 0, "The number of actions in P must be greater than 0."
assert sP0 == sP1, "The matrix P must be square with respect to states."
assert sR0 > 0, "The number of states in R must be greater than 0."
assert aR > 0, "The number of actions in R must be greater than 0."
assert sR0 == sR1, "The matrix R must be square with respect to states."
assert sP0 == sR0, "The number of states must agree in P and R."
assert aP == aR, "The number of actions must agree in P and R."
# Check that the P's are square and stochastic
aR, sR0, sR1 = _checkRewardsListLike(R, aP, sP0)
msg = ""
if sR0 <= 0:
msg = "The number of states in R must be greater than 0."
elif aR <= 0:
msg = "The number of actions in R must be greater than 0."
elif sR0 != sR1:
msg = "The matrix R must be square with respect to states."
elif sP0 != sR0:
msg = "The number of states must agree in P and R."
elif aP != aR:
msg = "The number of actions must agree in P and R."
if msg:
raise _error.InvalidError(msg)
# Check that the P's are square, stochastic and non-negative
for aa in range(aP):
checkSquareStochastic(P[aa])
# We are at the end of the checks, so if no exceptions have been raised
# then that means there are (hopefullly) no errors and we return None
# These are the old code comments, which need to be converted to
# information in the docstring:
#
# tranitions must be a numpy array either an AxSxS ndarray (with any
# dtype other than "object"); or, a 1xA ndarray with a "object" dtype,
# and each element containing an SxS array. An AxSxS array will be
# be converted to an object array. A numpy object array is similar to a
# MATLAB cell array.
#
# NumPy has an array type of 'object', which is roughly equivalent to
# the MATLAB cell array. These are most useful for storing sparse
# matrices as these can only have two dimensions whereas we want to be
# able to store a transition matrix for each action. If the dytpe of
# the transition probability array is object then we store this as
# P_is_object = True.
# If it is an object array, then it should only have one dimension
# otherwise fail with a message expalining why.
# If it is a normal array then the number of dimensions must be exactly
# three, otherwise fail with a message explaining why.
#
# As above but for the reward array. A difference is that the reward
# array can have either two or 3 dimensions.
#
# We want to make sure that the transition probability array and the
# reward array are in agreement. This means that both should show that
# there are the same number of actions and the same number of states.
# Furthermore the probability of transition matrices must be SxS in
# shape, so we check for that also.
#
# If the user has put their transition matrices into a numpy array
# with dtype of 'object', then it is possible that they have made a
# mistake and not all of the matrices are of the same shape. So,
# here we record the number of actions and states that the first
# matrix in element zero of the object array says it has. After
# that we check that every other matrix also reports the same
# number of actions and states, otherwise fail with an error.
# aP: the number of actions in the transition array. This
# corresponds to the number of elements in the object array.
#
# sP0: the number of states as reported by the number of rows of
# the transition matrix
# sP1: the number of states as reported by the number of columns of
# the transition matrix
#
# Now we check to see that every element of the object array holds
# a matrix of the same shape, otherwise fail.
#
# sp0aa and sp1aa represents the number of states in each
# subsequent element of the object array. If it doesn't match
# what was found in the first element, then we need to fail
# telling the user what needs to be fixed.
#
# if we are using a normal array for this, then the first
# dimension should be the number of actions, and the second and
# third should be the number of states
#
# the first dimension of the transition matrix must report the same
# number of states as the second dimension. If not then we are not
# dealing with a square matrix and it is not a valid transition
# probability. Also, if the number of actions is less than one, or the
# number of states is less than one, then it also is not a valid
# transition probability.
#
# now we check that each transition matrix is square-stochastic. For
# object arrays this is the matrix held in each element, but for
# normal arrays this is a matrix formed by taking a slice of the array
#
# if the rewarad array has an object dtype, then we check that
# each element contains a matrix of the same shape as we did
# above with the transition array.
#
# This indicates that the reward matrices are constructed per
# transition, so that the first dimension is the actions and
# the second two dimensions are the states.
#
# then the reward matrix is per state, so the first dimension is
# the states and the second dimension is the actions.
#
# this is added just so that the next check doesn't error out
# saying that sR1 doesn't exist
#
# the number of actions must be more than zero, the number of states
# must also be more than 0, and the states must agree
#
# now we check to see that what the transition array is reporting and
# what the reward arrar is reporting agree as to the number of actions
# and states. If not then fail explaining the situation
def rowsSumToOne(Z, n):
return((_np.abs(Z.sum(axis=1) - _np.ones(n))).max() <=
10 * _np.spacing(_np.float64(1)))
def checkSquareStochastic(Z):
"""Check if Z is a square stochastic matrix.
Let S = number of states.
Parameters
----------
Z : matrix
This should be a two dimensional array with a shape of (S, S). It can
possibly be sparse.
Notes
----------
Returns None if no error has been detected, else it raises an error.
"""
# try to get the shape of the matrix
try:
s1, s2 = Z.shape
except AttributeError:
raise TypeError("Matrix should be a numpy type.")
except ValueError:
raise InvalidMDPError(MDPERR["mat_square"])
# check that the matrix is square, and that each row sums to one
assert s1 == s2, MDPERR["mat_square"]
assert rowsSumToOne(Z, s2), MDPERR["mat_stoch"]
# make sure that there are no values less than zero
try:
assert (Z >= 0).all(), MDPERR["mat_nonneg"]
except (NotImplementedError, AttributeError, TypeError):
try:
assert (Z.data >= 0).all(), MDPERR["mat_nonneg"]
except AttributeError:
raise TypeError("Matrix should be a numpy type.")
def getSpan(W):
"""Return the span of W
......@@ -317,22 +299,4 @@ def getSpan(W):
sp(W) = max W(s) - min W(s)
"""
return(W.max() - W.min())
class Error(Exception):
"""Base class for exceptions in this module."""
def __init__(self):
Exception.__init__(self)
self.message = "PyMDPToolbox: "
def __str__(self):
return repr(self.message)
class InvalidMDPError(Error):
"""Class for invalid definitions of a MDP."""
def __init__(self, msg):
Error.__init__(self)
self.message += msg
self.args = tuple(msg)
return W.max() - W.min()
src/tests/test_utils.py
View file @ 6ad0c1a6
# -*- coding: utf-8 -*-
"""
Created on Sat Aug 24 14:52:17 2013
@author: steve
"""
import numpy as np
import scipy as sp
......@@ -174,19 +169,66 @@ def test_check_vector_R():
P = [np.matrix(np.eye(STATES))] * 3
assert mdptoolbox.util.check(P, R) is None
def test_check_vector_R_error():
R = np.random.rand(STATES+1)
P = [np.matrix(np.eye(STATES))] * 3
assert_raises(mdptoolbox.error.InvalidError,
mdptoolbox.util.check, P=P, R=R)
# Exception tests
def test_check_P_shape_error_1():
P = np.eye(STATES)[:STATES - 1, :STATES]
assert_raises(mdptoolbox.util.InvalidMDPError, mdptoolbox.util.check,
P=P, R=np.random.rand(10, 3))
assert_raises(mdptoolbox.error.InvalidError, mdptoolbox.util.check,
P=P, R=np.random.rand(STATES, ACTIONS))
def test_check_P_shape_error_2():
P = (np.random.rand(9, 9), np.random.rand(9, 9), np.random.rand(9, 5))
assert_raises(mdptoolbox.util.InvalidMDPError, mdptoolbox.util.check,
assert_raises(mdptoolbox.error.InvalidError, mdptoolbox.util.check,
P=P, R=np.random.rand(9))
def test_check_R_shape_error_1():
R = (np.random.rand(9, 9), np.random.rand(9, 9), np.random.rand(9, 5))
P = np.random.rand(3, 10, 10)
assert_raises(mdptoolbox.util.InvalidMDPError, mdptoolbox.util.check,
assert_raises(mdptoolbox.error.InvalidError, mdptoolbox.util.check,
P=P, R=R)
def test_isSqaure_tuple():
P = ((1, 0, 0), (0, 1, 0), (0, 0, 1))
assert mdptoolbox.util.isSquare(P)
def test_isSqaure_string():
P = "a string, the wrong type"
assert not mdptoolbox.util.isSquare(P)
def test_isStochastic_tuple():
P = ((1, 0, 0), (0, 1, 0), (0, 0, 1))
assert mdptoolbox.util.isStochastic(P)
def test_isStochastic_string():
P = "a string, the wrong type"
assert_raises(TypeError, mdptoolbox.util.isStochastic, matrix=P)
def test_isNonNegative_tuple():
P = ((1, 0, 0), (0, 1, 0), (0, 0, 1))
assert mdptoolbox.util.isStochastic(P)
def test_isNonNegative_string():
P = "a string, the wrong type"
assert_raises(TypeError, mdptoolbox.util.isStochastic, matrix=P)
def test_checkSquareStochastic_SquareError():
P = np.eye(STATES)[:STATES - 1, :STATES]
assert_raises(mdptoolbox.error.SquareError,
mdptoolbox.util.checkSquareStochastic, matrix=P)
def test_checkSquareStochastic_StochasticError():
P = np.random.rand(STATES, STATES)
assert_raises(mdptoolbox.error.StochasticError,
mdptoolbox.util.checkSquareStochastic, matrix=P)
def test_checkSquareStochastic_NonNegativeError():
P = np.eye(STATES)
P[0, 0] = -0.5
P[0, 1] = 1.5
assert_raises(mdptoolbox.error.NonNegativeError,
mdptoolbox.util.checkSquareStochastic, matrix=P)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment