...
 
Commits (317)
build/
dist/
docs/_build/
.coverage
*.db
*.py[co]
*~
*.egg-info/
language: python
branches:
only:
- master
- testing
python:
- "2.7"
- "2.7_with_system_site_packages"
- "3.2"
- "3.2_with_system_site_packages"
- "3.3"
- "3.4"
# - "pypy"
# - "pypy3"
before_install:
- sudo apt-get update -qq
# python-cvxopt package is only available for Python 2.7, so in every other
# case the build dependencies of cvxopt are installed
# Optional CVXOPT dependencies are: libatlas-base-dev libgsl0-dev fftw-dev libglpk-dev libdsdp-dev
# However, flags need to be set in the cvxopt/setup.py script which we don't
# alter with the current setup
- if [[ $TRAVIS_PYTHON_VERSION != "2.7_with_system_site_packages" ]]; then sudo apt-get install -qq gcc liblapack-dev; fi
# SciPy and NumPy compilation needs a fortran compiler, so if we aren't using
# sytem site packages then gfortran is installed first
- if [[ $TRAVIS_PYTHON_VERSION != *"with_system_site_packages" ]]; then sudo apt-get install -qq gfortran; fi
# if using system site packages then numpy, scipy and cvxopt can be installed
# with apt-get
- if [[ $TRAVIS_PYTHON_VERSION == "2.7_with_system_site_packages" ]]; then sudo apt-get install -qq python-numpy python-scipy python-cvxopt; fi
# There is no python3-cvxopt package, so it will be built by pip install
- if [[ $TRAVIS_PYTHON_VERSION == "3.2_with_system_site_packages" ]]; then sudo apt-get install -qq python3-numpy python3-scipy; fi
# Do test coverage report using coveralls
- echo -e "[report]\nomit =\n */python?.?/*\n */site-packages/nose/*" > .coveragerc
- pip install coveralls
install:
# If we aren't using system site packages then install everything from latest
# version that pip knows about using upgrade flag
- if [[ $TRAVIS_PYTHON_VERSION != *"with_system_site_packages" ]]; then pip install --upgrade "file://$PWD#egg=pymdptoolbox[LP]"; fi
# If we are using system site packages then numpy/scipy/cvxopt won't be
# installed if there is a version already available
- if [[ $TRAVIS_PYTHON_VERSION == *"with_system_site_packages" ]]; then pip install "file://$PWD#egg=pymdptoolbox[LP]"; fi
# command to run tests
script:
./runtests.sh
after_success:
coveralls
Copyright (c) 2011-2013 Steven A. W. Cordwell
Copyright (c) 2009 INRA
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the <ORGANIZATION> nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Markov Decision Process (MDP) Toolbox for Python
================================================
.. image:: https://travis-ci.org/sawcordwell/pymdptoolbox.svg?branch=master
:target: https://travis-ci.org/sawcordwell/pymdptoolbox
:alt: Build Status
.. image:: https://coveralls.io/repos/sawcordwell/pymdptoolbox/badge.png
:target: https://coveralls.io/r/sawcordwell/pymdptoolbox
:alt: Code Coverage
.. image:: https://pypip.in/py_versions/pymdptoolbox/badge.svg
:target: https://pypi.python.org/pypi/pymdptoolbox/
:alt: Supported Python versions
.. image:: https://pypip.in/implementation/pymdptoolbox/badge.svg
:target: https://pypi.python.org/pypi/pymdptoolbox/
:alt: Supported Python implementations
.. image:: https://pypip.in/license/pymdptoolbox/badge.svg
:target: https://pypi.python.org/pypi/pymdptoolbox/
:alt: License
The MDP toolbox provides classes and functions for the resolution of
descrete-time Markov Decision Processes. The list of algorithms that have been
implemented includes backwards induction, linear programming, policy iteration,
q-learning and value iteration along with several variations.
The classes and functions were developped based on the
`MATLAB <http://www.mathworks.com/products/matlab/>`_
`MDP toolbox <http://www.inra.fr/mia/T/MDPtoolbox/>`_ by the
`Biometry and Artificial Intelligence Unit <http://mia.toulouse.inra.fr/>`_ of
`INRA Toulouse <http://www.toulouse.inra.fr/>`_ (France). There are editions
available for MATLAB, GNU Octave, Scilab and R.
The suite of MDP toolboxes are described in Chades I, Chapron G, Cros M-J,
Garcia F & Sabbadin R (2014) 'MDPtoolbox: a multi-platform toolbox to solve
stochastic dynamic programming problems', *Ecography*, vol. 37, no. 9, pp.
916–920, doi `10.1111/ecog.00888 <http://dx.doi.org/10.1111/ecog.00888>`_.
Features
--------
- Eight MDP algorithms implemented
- Fast array manipulation using `NumPy <http://www.numpy.org>`_
- Full sparse matrix support using
`SciPy's sparse package <http://www.scipy.org/SciPyPackages/Sparse>`_
- Optional linear programming support using
`cvxopt <http://abel.ee.ucla.edu/cvxopt/>`_
PLEASE NOTE: the linear programming algorithm is currently unavailable except
for testing purposes due to incorrect behaviour.
Installation
------------
NumPy and SciPy must be on your system to use this toolbox. Please have a
look at their documentation to get them installed. If you are installing
onto Ubuntu or Debian and using Python 2 then this will pull in all the
dependencies:
``sudo apt-get install python-numpy python-scipy python-cvxopt``
On the other hand, if you are using Python 3 then cvxopt will have to be
compiled (pip will do it automatically). To get NumPy, SciPy and all the
dependencies to have a fully featured cvxopt then run:
``sudo apt-get install python3-numpy python3-scipy liblapack-dev libatlas-base-dev libgsl0-dev fftw-dev libglpk-dev libdsdp-dev``
The two main ways of downloading the package is either from the Python Package
Index or from GitHub. Both of these are explained below.
Python Package Index (PyPI)
~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. image:: https://pypip.in/download/pymdptoolbox/badge.svg
:target: https://pypi.python.org/pypi//pymdptoolbox/
:alt: Downloads
.. image:: https://pypip.in/version/pymdptoolbox/badge.svg
:target: https://pypi.python.org/pypi/pymdptoolbox/
:alt: Latest Version
.. image:: https://pypip.in/status/pymdptoolbox/badge.svg
:target: https://pypi.python.org/pypi/pymdptoolbox/
:alt: Development Status
.. image:: https://pypip.in/wheel/pymdptoolbox/badge.svg
:target: https://pypi.python.org/pypi/pymdptoolbox/
:alt: Wheel Status
.. image:: https://pypip.in/egg/pymdptoolbox/badge.svg
:target: https://pypi.python.org/pypi/pymdptoolbox/
:alt: Egg Status
.. image:: https://pypip.in/format/pymdptoolbox/badge.svg
:target: https://pypi.python.org/pypi/pymdptoolbox/
:alt: Download format
The toolbox's PyPI page is https://pypi.python.org/pypi/pymdptoolbox/ and there
are both zip and tar.gz archive options available that can be downloaded.
However, I recommend using `pip <https://pip.pypa.io/en/latest/>`_ to install
the toolbox if you have it available. Just type
``pip install pymdptoolbox``
at the console and it should take care of downloading and installing everything
for you. If you also want cvxopt to be automatically downloaded and installed
so that you can help test the linear programming algorithm then type
``pip install "pymdptoolbox[LP]"``
If you want it to be installed just for you rather than system wide then do
``pip install --user pymdptoolbox``
If you downloaded the package manually from PyPI
1. Extract the `*.zip` or `*.tar.gz` archive
``tar -xzvf pymdptoolbox-<VERSION>.tar.gz``
``unzip pymdptoolbox-<VERSION>``
2. Change to the PyMDPtoolbox directory
``cd pymdptoolbox``
3. Install via Setuptools, either to the root filesystem or to your home
directory if you don't have administrative access.
``python setup.py install``
``python setup.py install --user``
Read the
`Setuptools documentation <https://pythonhosted.org/setuptools/>`_ for
more advanced information.
Of course you can also use virtualenv or simply just unpack it to your working
directory.
GitHub
~~~~~~
Clone the Git repository
``git clone https://github.com/sawcordwell/pymdptoolbox.git``
and then follow from step two above. To learn how to use Git then I reccomend
reading the freely available `Pro Git book <http://git-scm.com/book>`_ written
by Scott Chacon and Ben Straub and published by Apress.
Quick Use
---------
Start Python in your favourite way. The following example shows you how to
import the module, set up an example Markov decision problem using a discount
value of 0.9, solve it using the value iteration algorithm, and then check the
optimal policy.
.. code:: python
import mdptoolbox.example
P, R = mdptoolbox.example.forest()
vi = mdptoolbox.mdp.ValueIteration(P, R, 0.9)
vi.run()
vi.policy # result is (0, 0, 0)
Documentation
-------------
Documentation is available at http://pymdptoolbox.readthedocs.org/
and also as docstrings in the module code.
If you use `IPython <http://ipython.scipy.org>`_ to work with the toolbox,
then you can view the docstrings by using a question mark ``?``. For example:
.. code:: python
import mdptoolbox
mdptoolbox?<ENTER>
mdptoolbox.mdp?<ENTER>
mdptoolbox.mdp.ValueIteration?<ENTER>
will display the relevant documentation.
Contribute
----------
Issue Tracker: https://github.com/sawcordwell/pymdptoolbox/issues
Source Code: https://github.com/sawcordwell/pymdptoolbox
Support
-------
Use the issue tracker.
License
-------
The project is licensed under the BSD license. See `<LICENSE.txt>`_ for details.
# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = _build
# User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " singlehtml to make a single large HTML file"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " applehelp to make an Apple Help Book"
@echo " devhelp to make HTML files and a Devhelp project"
@echo " epub to make an epub"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " latexpdf to make LaTeX files and run them through pdflatex"
@echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
@echo " text to make text files"
@echo " man to make manual pages"
@echo " texinfo to make Texinfo files"
@echo " info to make Texinfo files and run them through makeinfo"
@echo " gettext to make PO message catalogs"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " xml to make Docutils-native XML files"
@echo " pseudoxml to make pseudoxml-XML files for display purposes"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
@echo " coverage to run coverage check of the documentation (if enabled)"
clean:
rm -rf $(BUILDDIR)/*
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
singlehtml:
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
@echo
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in $(BUILDDIR)/htmlhelp."
qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PythonMarkovDecisionProcessToolbox.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PythonMarkovDecisionProcessToolbox.qhc"
applehelp:
$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
@echo
@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
@echo "N.B. You won't be able to view it unless you put it in" \
"~/Library/Documentation/Help or install it in your application" \
"bundle."
devhelp:
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@echo "To view the help file:"
@echo "# mkdir -p $$HOME/.local/share/devhelp/PythonMarkovDecisionProcessToolbox"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PythonMarkovDecisionProcessToolbox"
@echo "# devhelp"
epub:
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
@echo
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@echo "Run \`make' in that directory to run these through (pdf)latex" \
"(use \`make latexpdf' here to do that automatically)."
latexpdf:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
latexpdfja:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through platex and dvipdfmx..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
text:
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
@echo
@echo "Build finished. The text files are in $(BUILDDIR)/text."
man:
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
@echo
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
texinfo:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
@echo "Run \`make' in that directory to run these through makeinfo" \
"(use \`make info' here to do that automatically)."
info:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo "Running Texinfo files through makeinfo..."
make -C $(BUILDDIR)/texinfo info
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
gettext:
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
@echo
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in $(BUILDDIR)/linkcheck/output.txt."
doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
coverage:
$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
@echo "Testing of coverage in the sources finished, look at the " \
"results in $(BUILDDIR)/coverage/python.txt."
xml:
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
@echo
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
pseudoxml:
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
@echo
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
.. automodule:: mdptoolbox.example
:members:
:undoc-members:
:inherited-members:
:show-inheritance:
.. automodule:: mdptoolbox.mdp
:members:
:undoc-members:
:inherited-members:
:show-inheritance:
.. automodule:: mdptoolbox
:members:
:undoc-members:
:inherited-members:
:show-inheritance:
.. automodule:: mdptoolbox.util
:members:
:undoc-members:
:inherited-members:
:show-inheritance:
This diff is collapsed.
.. include:: ../README.rst
Contents
========
.. toctree::
:maxdepth: 2
api/mdptoolbox
api/mdp
api/util
api/example
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
@ECHO OFF
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set BUILDDIR=_build
set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
set I18NSPHINXOPTS=%SPHINXOPTS% .
if NOT "%PAPER%" == "" (
set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
)
if "%1" == "" goto help
if "%1" == "help" (
:help
echo.Please use `make ^<target^>` where ^<target^> is one of
echo. html to make standalone HTML files
echo. dirhtml to make HTML files named index.html in directories
echo. singlehtml to make a single large HTML file
echo. pickle to make pickle files
echo. json to make JSON files
echo. htmlhelp to make HTML files and a HTML help project
echo. qthelp to make HTML files and a qthelp project
echo. devhelp to make HTML files and a Devhelp project
echo. epub to make an epub
echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
echo. text to make text files
echo. man to make manual pages
echo. texinfo to make Texinfo files
echo. gettext to make PO message catalogs
echo. changes to make an overview over all changed/added/deprecated items
echo. xml to make Docutils-native XML files
echo. pseudoxml to make pseudoxml-XML files for display purposes
echo. linkcheck to check all external links for integrity
echo. doctest to run all doctests embedded in the documentation if enabled
echo. coverage to run coverage check of the documentation if enabled
goto end
)
if "%1" == "clean" (
for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
del /q /s %BUILDDIR%\*
goto end
)
REM Check if sphinx-build is available and fallback to Python version if any
%SPHINXBUILD% 2> nul
if errorlevel 9009 goto sphinx_python
goto sphinx_ok
:sphinx_python
set SPHINXBUILD=python -m sphinx.__init__
%SPHINXBUILD% 2> nul
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
:sphinx_ok
if "%1" == "html" (
%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/html.
goto end
)
if "%1" == "dirhtml" (
%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
goto end
)
if "%1" == "singlehtml" (
%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
goto end
)
if "%1" == "pickle" (
%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can process the pickle files.
goto end
)
if "%1" == "json" (
%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can process the JSON files.
goto end
)
if "%1" == "htmlhelp" (
%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can run HTML Help Workshop with the ^
.hhp project file in %BUILDDIR%/htmlhelp.
goto end
)
if "%1" == "qthelp" (
%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can run "qcollectiongenerator" with the ^
.qhcp project file in %BUILDDIR%/qthelp, like this:
echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PythonMarkovDecisionProcessToolbox.qhcp
echo.To view the help file:
echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PythonMarkovDecisionProcessToolbox.ghc
goto end
)
if "%1" == "devhelp" (
%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished.
goto end
)
if "%1" == "epub" (
%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The epub file is in %BUILDDIR%/epub.
goto end
)
if "%1" == "latex" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
if errorlevel 1 exit /b 1
echo.
echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "latexpdf" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
cd %BUILDDIR%/latex
make all-pdf
cd %~dp0
echo.
echo.Build finished; the PDF files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "latexpdfja" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
cd %BUILDDIR%/latex
make all-pdf-ja
cd %~dp0
echo.
echo.Build finished; the PDF files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "text" (
%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The text files are in %BUILDDIR%/text.
goto end
)
if "%1" == "man" (
%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The manual pages are in %BUILDDIR%/man.
goto end
)
if "%1" == "texinfo" (
%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
goto end
)
if "%1" == "gettext" (
%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
goto end
)
if "%1" == "changes" (
%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
if errorlevel 1 exit /b 1
echo.
echo.The overview file is in %BUILDDIR%/changes.
goto end
)
if "%1" == "linkcheck" (
%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
if errorlevel 1 exit /b 1
echo.
echo.Link check complete; look for any errors in the above output ^
or in %BUILDDIR%/linkcheck/output.txt.
goto end
)
if "%1" == "doctest" (
%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
if errorlevel 1 exit /b 1
echo.
echo.Testing of doctests in the sources finished, look at the ^
results in %BUILDDIR%/doctest/output.txt.
goto end
)
if "%1" == "coverage" (
%SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage
if errorlevel 1 exit /b 1
echo.
echo.Testing of coverage in the sources finished, look at the ^
results in %BUILDDIR%/coverage/python.txt.
goto end
)
if "%1" == "xml" (
%SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The XML files are in %BUILDDIR%/xml.
goto end
)
if "%1" == "pseudoxml" (
%SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
goto end
)
:end
sphinxcontrib-napoleon==0.3.1
#!/usr/bin/env sh
# Is there any difference to using ``python setup.py nosetests``?
nosetests --with-coverage --cover-package=mdptoolbox --with-doctest \
--doctest-options='+ELLIPSIS,+NORMALIZE_WHITESPACE,+IGNORE_EXCEPTION_DETAIL'
# -*- coding: utf-8 -*-
from setuptools import setup, find_packages
setup(name="pymdptoolbox",
version="4.0-b4-dev",
author="Steven A. W. Cordwell",
author_email="steven.cordwell@uqconnect.edu.au",
url="https://github.com/sawcordwell/pymdptoolbox",
description="Markov Decision Process (MDP) Toolbox",
long_description="The MDP toolbox provides classes and functions for "
"the resolution of descrete-time Markov Decision Processes. The list of "
"algorithms that have been implemented includes backwards induction, "
"linear programming, policy iteration, q-learning and value iteration "
"along with several variations.",
download_url="https://pypi.python.org/pypi/pymdptoolbox",
classifiers=[
"Development Status :: 4 - Beta",
"Environment :: Console",
"Intended Audience :: Education",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: BSD License",
"Natural Language :: English",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 2",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.2",
"Programming Language :: Python :: 3.3",
"Programming Language :: Python :: 3.4",
"Topic :: Scientific/Engineering",
"Topic :: Scientific/Engineering :: Mathematics",
"Topic :: Software Development :: Libraries :: Python Modules"],
platforms=["Any"],
license="New BSD",
packages=find_packages("src", exclude=["tests"]),
package_dir={"": "src"},
install_requires=["numpy", "scipy"],
extras_require={"LP": "cvxopt"})
Python MDP Toolbox Examples
===========================
\ No newline at end of file
# -*- coding: utf-8 -*-
"""
Created on Wed Aug 14 07:18:32 2013
@author: steve
"""
import os
import sqlite3
from numpy import arange
from numpy.random import permutation, random, randint
from scipy.sparse import dok_matrix
from mdp import ValueIteration
from mdpsql import ValueIteration as ValueIterationSQL
def setup(S, A):
P_sparse = [None] * A
R_sparse = 2*random(S) - 1
DB_sql = "MDP-big.db" % (S, A)
if os.path.exists(DB_sql):
os.remove(DB_sql)
with sqlite3.connect(DB_sql) as conn:
c = conn.cursor()
cmd = '''
CREATE TABLE info (name TEXT, value INTEGER);
INSERT INTO info VALUES('states', %s);
INSERT INTO info VALUES('actions', %s);''' % (S, A)
c.executescript(cmd)
for a in range(1, A+1):
a_sparse = a - 1
PP_sparse = dok_matrix((S, S))
cmd = '''
CREATE TABLE transition%s (row INTEGER, col INTEGER, prob REAL);
CREATE TABLE reward%s (state INTEGER PRIMARY KEY ASC, val REAL);
''' % (a, a)
c.executescript(cmd)
cmd = "INSERT INTO reward%s(val) VALUES(?)" % a
c.executemany(cmd, zip(R_sparse.tolist()))
for s in xrange(1, S+1):
s_sparse = s - 1
n = randint(1, 10)
col = (permutation(arange(1,S+1))[0:n]).tolist()
val = random(n)
val = (val / val.sum()).tolist()
PP_sparse[s_sparse, col - 1] = val
cmd = "INSERT INTO transition%s VALUES(?, ?, ?)" % a
c.executemany(cmd, zip([s] * n, col, val))
cmd = "CREATE UNIQUE INDEX Pidx%s ON transition%s (row, col);" % (a, a)
c.execute(cmd)
P_sparse[a_sparse] = PP_sparse.tocsr()
return P_sparse, R_sparse, DB_sql
if __name__ == "__main__":
P_sparse, R_sparse, DB_sql = setup(100000000, 3)
try:
sdp = ValueIteration(P_sparse, R_sparse, 0.9)
except MemoryError:
print("Killed. Sparse method ran out of memory.")
except:
raise
try:
sdp = ValueIterationSQL(DB_sql, 0.9)
except MemoryError:
print("Killed. SQL method ran out of memory.")
except:
raise
This diff is collapsed.
# -*- coding: utf-8 -*-
import cPickle as pickle
import numpy as np
from scipy.sparse import dok_matrix
from mdptoolbox import mdp
ACTIONS = 9
STATES = 3**ACTIONS
PLAYER = 1
OPPONENT = 2
WINS = ([1, 1, 1, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 1, 1, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 1, 1, 1],
[1, 0, 0, 1, 0, 0, 1, 0, 0],
[0, 1, 0, 0, 1, 0, 0, 1, 0],
[0, 0, 1, 0, 0, 1, 0, 0, 1],
[1, 0, 0, 0, 1, 0, 0, 0, 1],
[0, 0, 1, 0, 1, 0, 1, 0, 0])
# The valid number of cells belonging to either the player or the opponent:
# (player, opponent)
OWNED_CELLS = ((0, 0),
(1, 1),
(2, 2),
(3, 3),
(4, 4),
(0, 1),
(1, 2),
(2, 3),
(3, 4))
def convertIndexToTuple(state):
""""""
return(tuple(int(x) for x in np.base_repr(state, 3, 9)[-9::]))
def convertTupleToIndex(state):
""""""
return(int("".join(str(x) for x in state), 3))
def getLegalActions(state):
""""""
return(tuple(x for x in range(ACTIONS) if state[x] == 0))
def getTransitionAndRewardArrays():
""""""
P = [dok_matrix((STATES, STATES)) for a in range(ACTIONS)]
#R = spdok((STATES, ACTIONS))
R = np.zeros((STATES, ACTIONS))
# Naive approach, iterate through all possible combinations
for a in range(ACTIONS):
for s in range(STATES):
state = convertIndexToTuple(s)
if not isValid(state):
# There are no defined moves from an invalid state, so
# transition probabilities cannot be calculated. However,
# P must be a square stochastic matrix, so assign a
# probability of one to the invalid state transitioning
# back to itself.
P[a][s, s] = 1
# Reward is 0
else:
s1, p, r = getTransitionProbabilities(state, a)
P[a][s, s1] = p
R[s, a] = r
P[a] = P[a].tocsr()
#R = R.tolil()
return(P, R)
def getTransitionProbabilities(state, action):
"""
Parameters
----------
state : tuple
The state
action : int
The action
Returns
-------
s1, p, r : tuple of two lists and an int
s1 are the next states, p are the probabilities, and r is the reward
"""
#assert isValid(state)
assert 0 <= action < ACTIONS
if not isLegal(state, action):
# If the action is illegal, then transition back to the same state but
# incur a high negative reward
s1 = [convertTupleToIndex(state)]
return(s1, [1], -10)
# Update the state with the action
state = list(state)
state[action] = PLAYER
if isWon(state, PLAYER):
# If the player's action is a winning move then transition to the
# winning state and receive a reward of 1.
s1 = [convertTupleToIndex(state)]
return(s1, [1], 1)
elif isDraw(state):
s1 = [convertTupleToIndex(state)]
return(s1, [1], 0)
# Now we search through the opponents moves, and calculate transition
# probabilities based on maximising the opponents chance of winning..
s1 = []
p = []
legal_a = getLegalActions(state)
for a in legal_a:
state[a] = OPPONENT
# If the opponent is going to win, we assume that the winning move will
# be chosen:
if isWon(state, OPPONENT):
s1 = [convertTupleToIndex(state)]
return(s1, [1], -1)
elif isDraw(state):
s1 = [convertTupleToIndex(state)]
return(s1, [1], 0)
# Otherwise we assume the opponent will select a move with uniform
# probability across potential moves:
s1.append(convertTupleToIndex(state))
p.append(1.0 / len(legal_a))
state[a] = 0
# During non-terminal play states the reward is 0.
return(s1, p, 0)
def getReward(state, action):
""""""
if not isLegal(state, action):
return -100
state = list(state)
state[action] = PLAYER
if isWon(state, PLAYER):
return 1
elif isWon(state, OPPONENT):
return -1
else:
return 0
def isDraw(state):
""""""
try:
state.index(0)
return False
except ValueError:
return True
def isLegal(state, action):
""""""
if state[action] == 0:
return True
else:
return False
def isWon(state, who):
"""Test if a tic-tac-toe game has been won.
Assumes that the board is in a legal state.
Will test if the value 1 is in any winning combination.
"""
for w in WINS:
S = sum(1 if (w[k] == 1 and state[k] == who) else 0
for k in range(ACTIONS))
if S == 3:
# We have a win
return True
# There were no wins so return False
return False
def isValid(state):
""""""
# S1 is the sum of the player's cells
S1 = sum(1 if x == PLAYER else 0 for x in state)
# S2 is the sum of the opponent's cells
S2 = sum(1 if x == OPPONENT else 0 for x in state)
if (S1, S2) in OWNED_CELLS:
return True
else:
return False
if __name__ == "__main__":
P, R = getTransitionAndRewardArrays()
ttt = mdp.ValueIteration(P, R, 1)
ttt.setVerbose()
ttt.run()
f = "tictactoe.pkl"
pickle.dump(ttt.policy, open(f, "wb"))
print("Optimal policy pickled as '%s' in current directory." % f)
This diff is collapsed.
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
"""Markov Decision Process (MDP) Toolbox
=====================================
The MDP toolbox provides classes and functions for the resolution of
descrete-time Markov Decision Processes.
Available modules
-----------------
:mod:`~mdptoolbox.example`
Examples of transition and reward matrices that form valid MDPs
:mod:`~mdptoolbox.mdp`
Makov decision process algorithms
:mod:`~mdptoolbox.util`
Functions for validating and working with an MDP
How to use the documentation
----------------------------
Documentation is available both as docstrings provided with the code and
in html or pdf format from
`The MDP toolbox homepage <http://www.somewhere.com>`_. The docstring
examples assume that the ``mdptoolbox`` package is imported like so::
>>> import mdptoolbox
To use the built-in examples, then the ``example`` module must be imported::
>>> import mdptoolbox.example
Once the ``example`` module has been imported, then it is no longer neccesary
to issue ``import mdptoolbox``.
Code snippets are indicated by three greater-than signs::
>>> x = 17
>>> x = x + 1
>>> x
18
The documentation can be displayed with
`IPython <http://ipython.scipy.org>`_. For example, to view the docstring of
the ValueIteration class use ``mdp.ValueIteration?<ENTER>``, and to view its
source code use ``mdp.ValueIteration??<ENTER>``.
Acknowledgments
---------------
This module is modified from the MDPtoolbox (c) 2009 INRA available at
http://www.inra.fr/mia/T/MDPtoolbox/.
"""
# Copyright (c) 2011-2013 Steven A. W. Cordwell
# Copyright (c) 2009 INRA
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# * Neither the name of the <ORGANIZATION> nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
from . import mdp
# -*- coding: utf-8 -*-
"""Markov Decision Process (MDP) Toolbox: ``error`` module
=======================================================
The ``error`` module provides exception classes that can be raised by
the toolbox.
Available classes
-----------------
Error
Base exception class derived from ``Exception``
InvalidError
Exception for invalid definitions of an MDP
NonNegativeError
Exception for transition matrices that have negative elements
SquareError
Exception for transition matrices that are not square
StochasticError
Exception for transition matrices that are not stochastic
"""
# Copyright (c) 2015 Steven A. W. Cordwell
# Copyright (c) 2009 INRA
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# * Neither the name of the <ORGANIZATION> nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
class Error(Exception):
"""Base class for exceptions in this module."""
def __init__(self):
Exception.__init__(self)
self.message = "PyMDPToolbox - "
def __str__(self):
return repr(self.message)
class InvalidError(Error):
"""Class for invalid definitions of a MDP."""
def __init__(self, msg):
Error.__init__(self)
self.message += msg
self.args = tuple(msg)
class NonNegativeError(Error):
"""Class for transition matrix stochastic errors"""
default_msg = "The transition probability matrix is negative."
def __init__(self, msg=None):
if msg is None:
msg = self.default_msg
Error.__init__(self)
self.message += msg
self.args = tuple(msg)
class SquareError(Error):
"""Class for transition matrix square errors"""
default_msg = "The transition probability matrix is not square."
def __init__(self, msg=None):
if msg is None:
msg = self.default_msg
Error.__init__(self)
self.message += msg
self.args = tuple(msg)
class StochasticError(Error):
"""Class for transition matrix stochastic errors"""
default_msg = "The transition probability matrix is not stochastic."
def __init__(self, msg=None):
if msg is None:
msg = self.default_msg
Error.__init__(self)
self.message += msg
self.args = tuple(msg)
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
# -*- coding: utf-8 -*-
"""The Python Markov Decision Process (MDP) Toolbox Test Suite
===========================================================
These unit tests are written for the nosetests framwork. You will need to have
nosetests installed, and then run from the command line.
$ cd /path/to/pymdptoolbox
$ nostests
"""
# -*- coding: utf-8 -*-
#def test_JacksCarRental():
# S = 21 ** 2
# A = 11
# P = np.zeros((A, S, S))
# R = np.zeros((A, S, S))
# for a in range(A):
# for s in range(21):
# for s1 in range(21):
# c1s = int(s / 21)
# c2s = s - c1s * 21
# c1s1 = int(s1 / 21)
# c2s1 = s - c1s * 21
# cs = c1s + c2s
# cs1 = c1s1 + c2s1
# netmove = 5 - a
# if (s1 < s):
# pass
# else:
# pass
# P[a, s, s1] = 1
# R[a, s, s1] = 10 * (cs - cs1) - 2 * abs(a)
#
# inst = PolicyIteration(P, R, 0.9)
# #assert (inst.policy == )
#
#def test_JacksCarRental2():
# pass
#
#def test_GamblersProblem():
# inst = ValueIteration()
# #assert (inst.policy == )
# checkSquareStochastic: not square, stochastic and non-negative
#@raises(ValueError(mdperr["mat_square"]))
#def test_checkSquareStochastic_notsquare_stochastic_nonnegative_array():
# P = np.eye(STATES, STATES + 1)
# inst.checkSquareStochastic(P)
#
#@raises(ValueError(mdperr["mat_square"]))
#def test_checkSquareStochastic_notsquare_stochastic_nonnegative_matrix():
# P = np.matrix(np.eye(STATES, STATES + 1))
# inst.checkSquareStochastic(P)
#
#@raises(ValueError(mdperr["mat_square"]))
#def test_checkSquareStochastic_notsquare_stochastic_nonnegative_sparse():
# P = sp.sparse.eye(STATES, STATES + 1).tocsr()
# inst.checkSquareStochastic(P)
# checkSquareStochastic: square, not stochastic and non-negative
#def test_checkSquareStochastic_square_notstochastic_nonnegative_array():
# P = np.eye(STATES)
# i = randint(STATES)
# j = randint(STATES)
# P[i, j] = P[i, j] + 1
# try:
# inst.checkSquareStochastic(P)
# except ValueError(mdperr["mat_stoch"]):
# pass
#
#def test_checkSquareStochastic_square_notstochastic_nonnegative_matrix():
# P = np.matrix(np.eye(STATES))
# i = randint(STATES)
# j = randint(STATES)
# P[i, j] = P[i, j] + 1
# try:
# inst.checkSquareStochastic(P)
# except ValueError(mdperr["mat_stoch"]):
# pass
#
#def test_checkSquareStochastic_square_notstochastic_nonnegative_sparse():
# P = sp.sparse.eye(STATES, STATES).tolil()
# i = randint(STATES)
# j = randint(STATES)
# P[i, j] = P[i, j] + 1
# P = P.tocsr()
# try:
# inst.checkSquareStochastic(P)
# except ValueError(mdperr["mat_stoch"]):
# pass
# checkSquareStochastic: square, stochastic and negative
#def test_checkSquareStochastic_square_stochastic_negative_array():
# P = np.eye(STATES, STATES)
# i = randint(STATES)
# j = randint(STATES)
# while j == i:
# j = randint(STATES)
# P[i, i] = -1
# P[i, j] = 1
# try:
# inst.checkSquareStochastic(P)
# except ValueError(mdperr["mat_nonneg"]):
# pass
#
#def test_checkSquareStochastic_square_stochastic_negative_matrix():
# P = np.matrix(np.eye(STATES, STATES))
# i = randint(STATES)
# j = randint(STATES)
# while j == i:
# j = randint(STATES)
# P[i, i] = -1
# P[i, j] = 1
# try:
# inst.checkSquareStochastic(P)
# except ValueError(mdperr["mat_nonneg"]):
# pass
#
#def test_checkSquareStochastic_square_stochastic_negative_sparse():
# P = sp.sparse.eye(STATES, STATES)
# i = randint(STATES)
# j = randint(STATES)
# while j == i:
# j = randint(STATES)
# P[i, i] = -1
# P[i, j] = 1
# try:
# inst.checkSquareStochastic(P)
# except ValueError(mdperr["mat_nonneg"]):
# pass
#def test_check_square_stochastic_array_Rtranspose():
# P = np.array([np.eye(STATES), np.eye(STATES)])
# R = np.array([ones(STATES), ones(STATES)])
# assert inst.check(P, R) == (True, "R is wrong way")
\ No newline at end of file
# -*- coding: utf-8 -*-
"""
Created on Wed Sep 11 13:33:47 2013
@author: steve
"""
# -*- coding: utf-8 -*-
"""
Created on Wed Sep 11 13:35:08 2013
@author: steve
"""
# -*- coding: utf-8 -*-
"""
Created on Sat Aug 24 14:18:51 2013
@author: steve
"""
import numpy as np
import scipy.sparse as sp
import mdptoolbox.example
from .utils import SMALLNUM, P_small, R_small
class TestUnitsMDP(object):
def test_MDP_has_startRun_method(self):
P, R = mdptoolbox.example.small()
sdp = mdptoolbox.mdp.MDP(P, R, None, None, None)
sdp._startRun()
assert sdp.time is not None
def test_MDP_has_endRun_method(self):
P, R = mdptoolbox.example.small()
sdp = mdptoolbox.mdp.MDP(P, R, None, None, None)
sdp._startRun()
sdp.V = np.zeros(1) # to prevent AttributeError in sdp._endRun()
sdp.policy = np.zeros(1) # to prevent AttributeError in sdp._endRun()
time = sdp.time
sdp._endRun()
assert time != sdp.time
class TestMDP(object):
P = (((0.0, 0.0, 0.6, 0.4, 0.0),
(0.0, 0.0, 0.0, 0.0, 1.0),
(0.0, 0.0, 1.0, 0.0, 0.0),
(0.0, 0.0, 0.0, 1.0, 0.0),
(0.0, 0.0, 0.0, 0.0, 1.0)),
((0.0, 0.4, 0.0, 0.0, 0.6),
(0.0, 1.0, 0.0, 0.0, 0.0),
(0.0, 0.0, 0.0, 0.0, 1.0),
(0.0, 0.0, 0.0, 0.0, 1.0),
(0.0, 0.0, 0.0, 0.0, 1.0)))
R = (((0, 0, 0, 0, 0),
(0, 0, 0, 0, 0),
(0, 0, 0, 0, 0),
(0, 0, 0, 0, 0),
(0, 0, 0, 0, 0)),
((0, 0, 0, 0, 1),
(0, 0, 0, 0, 0),
(0, 0, 0, 0, 0),
(0, 0, 0, 0, 0),
(0, 0, 0, 0, 0)))
computed_R = ((0.0, 0.0, 0.0, 0.0, 0.0),
(0.6, 0.0, 0.0, 0.0, 0.0))
def test_a(self):
P = np.array(self.P)
R = np.array(self.R)
sdp = mdptoolbox.mdp.MDP(P, R, None, None, None)
assert sdp.S == 5
assert sdp.A == 2
assert (sdp.R[0] == np.array(self.computed_R[0])).all()
assert (sdp.R[1] == np.array(self.computed_R[1])).all()
def test_b(self):
P = np.array(self.P)
R = np.array(self.computed_R).T
sdp = mdptoolbox.mdp.MDP(P, R, None, None, None)
assert sdp.S == 5
assert sdp.A == 2
assert (sdp.R[0] == R[:, 0]).all()
assert (sdp.R[1] == R[:, 1]).all()
def test_c(self):