{
  "_id": "6a1ea353b25058d4daad76d1",
  "Package": "markovDP",
  "Title": "Infrastructure for Discrete-Time Markov Decision Processes (MDP)",
  "Version": "0.99.0",
  "Date": "2024-08-29",
  "Authors@R": "person(\"Michael\", \"Hahsler\", , \"mhahsler@lyle.smu.edu\", role = c(\"aut\", \"cph\", \"cre\"),\ncomment = c(ORCID = \"0000-0003-2716-1405\"))",
  "Description": "Provides the infrastructure to work with Markov Decision\nProcesses (MDPs) in R. The focus is on convenience in\nformulating MDPs, the support of sparse representations (using\nsparse matrices, lists and data.frames) and visualization of\nresults. Some key components are implemented in C++ to speed up\ncomputation. Several popular solvers are implemented.",
  "License": "GPL (>=3)",
  "URL": "https://github.com/mhahsler/markovDP",
  "BugReports": "https://github.com/mhahsler/markovDP/issues",
  "VignetteBuilder": "knitr",
  "Classification/ACM": "G.4, G.1.6, I.2.6",
  "Copyright": "Copyright (C) Michael Hahsler.",
  "Encoding": "UTF-8",
  "Roxygen": "list(markdown = TRUE)",
  "RoxygenNote": "7.3.2",
  "SystemRequirements": "C++17",
  "Collate": "'AAA_check_installed.R' 'AAA_colors.R' 'AAA_foreach_helper.R'\n'AAA_near_equal.R' 'AAA_nodots.R' 'AAA_package.R'\n'AAA_progress.R' 'AAA_sample_sparse.R' 'AAA_shorten.R'\n'AAA_which_max_random.R' 'Cliff_walking.R' 'DynaMaze.R' 'MDP.R'\n'MDPSample.R' 'Maze.R' 'Q_values.R' 'RcppExports.R'\n'Windy_gridworld.R' 'absorbing_states.R'\n'accessors_transitions.R' 'accessors_reward.R' 'accessors.R'\n'act.R' 'action.R' 'action_state_helpers.R'\n'available_actions.R' 'bellman_operator.R'\n'check_and_fix_MDP.R' 'convergence_horizon.R'\n'find_reachable_states.R' 'greedy.R' 'gridworld.R'\n'linear_function_approximation.R' 'policy.R'\n'policy_evaluation.R' 'policy_evaluation_LP.R'\n'policy_evaluation_MC.R' 'policy_evaluation_bellman.R'\n'reachable_states.R' 'regret.R' 'reward.R'\n'round_stochchastic.R' 'sample_MDP.R' 'sample_MDPSample.R'\n'schedule.R' 'solve_MDP.R' 'solve_MDP_APPROX.R'\n'solve_MDP_DP.R' 'solve_MDP_LP.R' 'solve_MDP_MC.R'\n'solve_MDP_PG.R' 'solve_MDP_SAMP.R' 'solve_MDP_TD.R'\n'sparse_helpers.R' 'start.R' 'transformation.R'\n'transition_graph.R' 'unreachable_states.R' 'value_function.R'\n'visit_probability.R' 'zzz.R'",
  "Config/pak/sysreqs": "libglpk-dev libxml2-dev",
  "Repository": "https://mhahsler.r-universe.dev",
  "Date/Publication": "2025-07-21 19:37:59 UTC",
  "RemoteUrl": "https://github.com/mhahsler/markovDP",
  "RemoteRef": "HEAD",
  "RemoteSha": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
  "NeedsCompilation": "yes",
  "Packaged": {
    "Date": "2026-06-02 09:25:01 UTC",
    "User": "root"
  },
  "Author": "Michael Hahsler [aut, cph, cre] (ORCID:\n<https://orcid.org/0000-0003-2716-1405>)",
  "Maintainer": "Michael Hahsler <mhahsler@lyle.smu.edu>",
  "MD5sum": "f400d8c6c3be6a81acafcd772a059a03",
  "_user": "mhahsler",
  "_type": "src",
  "_file": "markovDP_0.99.0.tar.gz",
  "_fileid": "4ab87019c38fe8193e2d9bb992976fd657954595bcb228495266ff9a3159b07a",
  "_filesize": 3501433,
  "_sha256": "4ab87019c38fe8193e2d9bb992976fd657954595bcb228495266ff9a3159b07a",
  "_created": "2026-06-02T09:25:01.000Z",
  "_published": "2026-06-02T09:33:07.113Z",
  "_distro": "noble",
  "_jobs": [
    {
      "job": 79040080353,
      "time": 224,
      "config": "linux-devel-arm64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "7353986629"
    },
    {
      "job": 79040080218,
      "time": 237,
      "config": "linux-devel-x86_64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "7353990057"
    },
    {
      "job": 79040080291,
      "time": 229,
      "config": "linux-release-arm64",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7353988386"
    },
    {
      "job": 79040080294,
      "time": 214,
      "config": "linux-release-x86_64",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7353982539"
    },
    {
      "job": 79040080259,
      "time": 182,
      "config": "macos-oldrel-arm64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "7353970625"
    },
    {
      "job": 79040080273,
      "time": 314,
      "config": "macos-oldrel-x86_64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "7354018228"
    },
    {
      "job": 79040080313,
      "time": 139,
      "config": "macos-release-arm64",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7353955369"
    },
    {
      "job": 79040080317,
      "time": 419,
      "config": "macos-release-x86_64",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7354056437"
    },
    {
      "job": 79039122275,
      "time": 326,
      "config": "source",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7353904897"
    },
    {
      "job": 79040080214,
      "time": 137,
      "config": "wasm-release",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7353954607"
    },
    {
      "job": 79040080236,
      "time": 273,
      "config": "windows-devel",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "7354003572"
    },
    {
      "job": 79040080245,
      "time": 221,
      "config": "windows-oldrel",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "7353984475"
    },
    {
      "job": 79040080215,
      "time": 226,
      "config": "windows-release",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7353986401"
    }
  ],
  "_buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572",
  "_status": "success",
  "_host": "GitHub-Actions",
  "_upstream": "https://github.com/mhahsler/markovDP",
  "_commit": {
    "id": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
    "author": "mhahsler <michael@hahsler.net>",
    "committer": "mhahsler <michael@hahsler.net>",
    "message": "added policy approx.\n",
    "time": 1753126679
  },
  "_maintainer": {
    "name": "Michael Hahsler",
    "email": "mhahsler@lyle.smu.edu",
    "login": "mhahsler",
    "description": "I develop packages for AI, ML, and Data Science.",
    "uuid": 14900595,
    "orcid": "0000-0003-2716-1405"
  },
  "_registered": true,
  "_dependencies": [
    {
      "package": "R",
      "version": ">= 3.5.0",
      "role": "Depends"
    },
    {
      "package": "Rcpp",
      "role": "LinkingTo"
    },
    {
      "package": "fastmap",
      "role": "Imports"
    },
    {
      "package": "foreach",
      "role": "Imports"
    },
    {
      "package": "igraph",
      "role": "Imports"
    },
    {
      "package": "lpSolve",
      "role": "Imports"
    },
    {
      "package": "Matrix",
      "role": "Imports"
    },
    {
      "package": "MatrixExtra",
      "role": "Imports"
    },
    {
      "package": "methods",
      "role": "Imports"
    },
    {
      "package": "progress",
      "role": "Imports"
    },
    {
      "package": "Rcpp",
      "role": "Imports"
    },
    {
      "package": "stats",
      "role": "Imports"
    },
    {
      "package": "doParallel",
      "role": "Suggests"
    },
    {
      "package": "gifski",
      "role": "Suggests"
    },
    {
      "package": "knitr",
      "role": "Suggests"
    },
    {
      "package": "rmarkdown",
      "role": "Suggests"
    },
    {
      "package": "testthat",
      "role": "Suggests"
    },
    {
      "package": "visNetwork",
      "role": "Suggests"
    }
  ],
  "_owner": "mhahsler",
  "_selfowned": true,
  "_usedby": 0,
  "_updates": [
    {
      "week": "2025-25",
      "n": 1
    },
    {
      "week": "2025-29",
      "n": 2
    },
    {
      "week": "2025-30",
      "n": 3
    }
  ],
  "_tags": [],
  "_topics": [
    "control-theory",
    "markov-decision-process",
    "optimization",
    "cpp"
  ],
  "_stars": 7,
  "_contributors": [
    {
      "user": "mhahsler",
      "count": 94,
      "uuid": 14900595
    }
  ],
  "_userbio": {
    "uuid": 14900595,
    "type": "user",
    "name": "Michael Hahsler",
    "description": "I develop packages for AI, ML, and Data Science."
  },
  "_downloads": {
    "count": 0,
    "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/markovDP"
  },
  "_devurl": "https://github.com/mhahsler/markovdp",
  "_searchresults": 11,
  "_rbuild": "4.6.0",
  "_assets": [
    "extra/citation.cff",
    "extra/citation.html",
    "extra/citation.json",
    "extra/citation.txt",
    "extra/contents.json",
    "extra/markovDP.html",
    "extra/NEWS.html",
    "extra/NEWS.txt",
    "extra/readme.html",
    "extra/readme.md",
    "manual.pdf"
  ],
  "_cranurl": false,
  "_exports": [
    "A",
    "absorbing_states",
    "act",
    "action",
    "action_discrepancy",
    "add_policy",
    "approx_greedy_action",
    "approx_greedy_policy",
    "approx_Q_value",
    "approx_V_plot",
    "approx_value",
    "available_actions",
    "bellman_operator",
    "bellman_update",
    "colors_continuous",
    "colors_discrete",
    "convergence_horizon",
    "create_basis_coefs",
    "curve_multiple_directed",
    "features2state",
    "find_reachable_states",
    "get_state_features",
    "greedy_action",
    "greedy_policy",
    "gw_animate",
    "gw_init",
    "gw_matrix",
    "gw_maze_MDP",
    "gw_path",
    "gw_plot",
    "gw_plot_transition_graph",
    "gw_random_maze",
    "gw_rc2s",
    "gw_read_maze",
    "gw_s2rc",
    "gw_transition_model",
    "gw_transition_model_end_state",
    "gw_transition_model_named",
    "gw_transition_model_sparse",
    "induced_reward_matrix",
    "induced_transition_matrix",
    "is_converged_MDP",
    "is_solved_MDP",
    "manual_policy",
    "MDP",
    "MDPSample",
    "normalize_action",
    "normalize_action_id",
    "normalize_action_label",
    "normalize_MDP",
    "normalize_state",
    "normalize_state_features",
    "normalize_state_id",
    "normalize_state_label",
    "P_",
    "pi_approx_linear",
    "plot_transition_graph",
    "plot_value_function",
    "policy",
    "policy_evaluation",
    "policy_evaluation_bellman",
    "policy_evaluation_LP",
    "policy_evaluation_MC",
    "q_approx_linear",
    "Q_random",
    "Q_values",
    "Q_zero",
    "R_",
    "random_policy",
    "reachable_states",
    "regret",
    "remove_unreachable_states",
    "reward",
    "reward_matrix",
    "round_stochastic",
    "s",
    "S",
    "sample_MDP",
    "schedule_exp",
    "schedule_exp2",
    "schedule_harmonic",
    "schedule_linear",
    "schedule_log",
    "solve_MDP",
    "solve_MDP_APPROX",
    "solve_MDP_DP",
    "solve_MDP_LP",
    "solve_MDP_MC",
    "solve_MDP_PG",
    "solve_MDP_SAMP",
    "solve_MDP_TD",
    "start_vector",
    "state2features",
    "transformation_fourier_basis",
    "transformation_linear_basis",
    "transformation_polynomial_basis",
    "transformation_RBF_basis",
    "transition_graph",
    "transition_matrix",
    "unreachable_states",
    "v_approx_linear",
    "V_random",
    "V_zero",
    "value_error",
    "value_function",
    "visit_probability"
  ],
  "_datasets": [
    {
      "name": "Cliff_walking",
      "title": "Cliff Walking Gridworld MDP",
      "object": "Cliff_walking",
      "file": "Cliff_walking.rda",
      "class": [
        "MDPModel",
        "MDP"
      ],
      "fields": [],
      "table": false,
      "tojson": false
    },
    {
      "name": "DynaMaze",
      "title": "The Dyna Maze",
      "object": "DynaMaze",
      "file": "DynaMaze.rda",
      "class": [
        "MDPModel",
        "MDP"
      ],
      "fields": [],
      "table": false,
      "tojson": false
    },
    {
      "name": "Maze",
      "title": "Steward Russell's 4x3 Maze Gridworld MDP",
      "object": "Maze",
      "file": "Maze.rda",
      "class": [
        "MDPModel",
        "MDP"
      ],
      "fields": [],
      "table": false,
      "tojson": false
    },
    {
      "name": "Windy_gridworld",
      "title": "Windy Gridworld MDP Windy Gridworld MDP",
      "object": "Windy_gridworld",
      "file": "Windy_gridworld.rda",
      "class": [
        "MDPModel",
        "MDP"
      ],
      "fields": [],
      "table": false,
      "tojson": false
    }
  ],
  "_help": [
    {
      "page": "absorbing_states",
      "title": "Absorbing States",
      "concept": [
        "MDP",
        "MDPSample"
      ],
      "topics": [
        "absorbing_states",
        "absorbing_states.MDP",
        "absorbing_states.MDPSample"
      ]
    },
    {
      "page": "act",
      "title": "Perform an Action",
      "concept": [
        "MDP",
        "MDPSample"
      ],
      "topics": [
        "act",
        "act.MDPModel",
        "act.MDPSample"
      ]
    },
    {
      "page": "action",
      "title": "Choose an Action Given a Policy",
      "concept": [
        "policy"
      ],
      "topics": [
        "action"
      ]
    },
    {
      "page": "action_state_helpers",
      "title": "Conversions for Action and State IDs and Labels",
      "topics": [
        "action_state_helpers",
        "features2state",
        "get_state_features",
        "normalize_action",
        "normalize_action_id",
        "normalize_action_label",
        "normalize_state",
        "normalize_state_features",
        "normalize_state_id",
        "normalize_state_label",
        "s",
        "state2features"
      ]
    },
    {
      "page": "available_actions",
      "title": "Available Actions in a State",
      "concept": [
        "MDP"
      ],
      "topics": [
        "available_actions"
      ]
    },
    {
      "page": "bellman_update",
      "title": "Bellman Update and Bellman operator",
      "concept": [
        "MDP",
        "policy"
      ],
      "topics": [
        "bellman_operator",
        "bellman_update"
      ]
    },
    {
      "page": "Cliff_walking",
      "title": "Cliff Walking Gridworld MDP",
      "concept": [
        "MDP_examples",
        "gridworld"
      ],
      "topics": [
        "Cliff_walking",
        "cliff_walking"
      ]
    },
    {
      "page": "colors",
      "title": "Default Colors for Visualization",
      "topics": [
        "colors",
        "colors_continuous",
        "colors_discrete"
      ]
    },
    {
      "page": "convergence_horizon",
      "title": "Estimate the Convergence Horizon for an Infinite-Horizon MDP",
      "concept": [
        "MDP"
      ],
      "topics": [
        "convergence_horizon"
      ]
    },
    {
      "page": "DynaMaze",
      "title": "The Dyna Maze",
      "concept": [
        "MDP_examples",
        "gridworld"
      ],
      "topics": [
        "DynaMaze",
        "dynamaze"
      ]
    },
    {
      "page": "find_reachable_states",
      "title": "Find Reachable State Space from a Transition Model Function",
      "topics": [
        "find_reachable_states"
      ]
    },
    {
      "page": "greedy_action",
      "title": "Greedy Actions and Policies",
      "concept": [
        "MDP",
        "policy"
      ],
      "topics": [
        "greedy_action",
        "greedy_policy"
      ]
    },
    {
      "page": "gridworld",
      "title": "Helper Functions for Gridworld MDPs",
      "concept": [
        "MDP",
        "gridworld"
      ],
      "topics": [
        "gridworld",
        "gw",
        "gw_animate",
        "gw_init",
        "gw_matrix",
        "gw_maze_MDP",
        "gw_path",
        "gw_plot",
        "gw_plot_transition_graph",
        "gw_random_maze",
        "gw_rc2s",
        "gw_read_maze",
        "gw_s2rc",
        "gw_transition_model",
        "gw_transition_model_end_state",
        "gw_transition_model_named",
        "gw_transition_model_sparse"
      ]
    },
    {
      "page": "linear_function_approximation",
      "title": "Linear Function Approximation",
      "topics": [
        "approx_value",
        "linear_function_approximation",
        "pi_approx_linear",
        "q_approx_linear",
        "v_approx_linear"
      ]
    },
    {
      "page": "Maze",
      "title": "Steward Russell's 4x3 Maze Gridworld MDP",
      "concept": [
        "MDP_examples",
        "gridworld"
      ],
      "topics": [
        "Maze",
        "maze"
      ]
    },
    {
      "page": "MDP",
      "title": "Define an MDP Problem",
      "concept": [
        "MDP",
        "MDP_examples"
      ],
      "topics": [
        "A",
        "is_converged_MDP",
        "is_solved_MDP",
        "MDP",
        "MDPModel",
        "P_",
        "R_",
        "S"
      ]
    },
    {
      "page": "MDPSample",
      "title": "Define an MDP With Only Sample Access",
      "concept": [
        "MDPSample"
      ],
      "topics": [
        "MDPSample"
      ]
    },
    {
      "page": "policy",
      "title": "Extract, Create Add a Policy to a Model",
      "concept": [
        "policy"
      ],
      "topics": [
        "add_policy",
        "induced_reward_matrix",
        "induced_transition_matrix",
        "manual_policy",
        "policy",
        "random_policy"
      ]
    },
    {
      "page": "policy_evaluation",
      "title": "Policy Evaluation",
      "concept": [
        "MDP",
        "policy"
      ],
      "topics": [
        "policy_evaluation",
        "policy_evaluation_bellman",
        "policy_evaluation_LP",
        "policy_evaluation_MC"
      ]
    },
    {
      "page": "Q_values",
      "title": "Q-Values",
      "concept": [
        "MDP",
        "policy"
      ],
      "topics": [
        "Q_random",
        "Q_values",
        "Q_zero"
      ]
    },
    {
      "page": "reachable_states",
      "title": "Find Reachable States",
      "concept": [
        "MDP",
        "MDPSample"
      ],
      "topics": [
        "reachable_states",
        "reachable_states.function",
        "reachable_states.MDPModel",
        "reachable_states.MDPSample"
      ]
    },
    {
      "page": "regret",
      "title": "Regret of a Policy and Related Measures",
      "concept": [
        "MDP",
        "policy"
      ],
      "topics": [
        "action_discrepancy",
        "regret",
        "value_error"
      ]
    },
    {
      "page": "reward",
      "title": "Calculate the Expected Reward of a Policy",
      "concept": [
        "policy"
      ],
      "topics": [
        "reward",
        "reward.MDP"
      ]
    },
    {
      "page": "round_stochastic",
      "title": "Round a stochastic vector or a row-stochastic matrix",
      "topics": [
        "round_stochastic"
      ]
    },
    {
      "page": "sample_MDP",
      "title": "Sample Trajectories from an MDP",
      "concept": [
        "MDP"
      ],
      "topics": [
        "sample_MDP",
        "sample_MDP.MDP"
      ]
    },
    {
      "page": "sample_MDP.MDPSample",
      "title": "Sample Trajectories from an MDPSample",
      "concept": [
        "MDPSample"
      ],
      "topics": [
        "sample_MDP.MDPSample"
      ]
    },
    {
      "page": "schedule",
      "title": "Schedules to Reduce Alpha, Epsilon and Other Parameters",
      "concept": [
        "solver"
      ],
      "topics": [
        "schedule",
        "schedule_exp",
        "schedule_exp2",
        "schedule_harmonic",
        "schedule_linear",
        "schedule_log"
      ]
    },
    {
      "page": "solve_MDP",
      "title": "Solve an MDP Problem",
      "concept": [
        "MDP",
        "MDPSample",
        "solver"
      ],
      "topics": [
        "solve_MDP",
        "solve_MDP.MDP",
        "solve_MDP.MDPSample"
      ]
    },
    {
      "page": "solve_MDP_APPROX",
      "title": "Solve MDPs with Temporal Differencing with Function Approximation",
      "concept": [
        "MDPSample",
        "solver"
      ],
      "topics": [
        "approx_greedy_action",
        "approx_greedy_policy",
        "approx_Q_value",
        "approx_V_plot",
        "solve_MDP_APPROX"
      ]
    },
    {
      "page": "solve_MDP_DP",
      "title": "Solve MDPs using Dynamic Programming",
      "concept": [
        "solver"
      ],
      "topics": [
        "solve_MDP_DP"
      ]
    },
    {
      "page": "solve_MDP_LP",
      "title": "Solve MDPs using Linear Programming",
      "concept": [
        "solver"
      ],
      "topics": [
        "solve_MDP_LP"
      ]
    },
    {
      "page": "solve_MDP_MC",
      "title": "Solve MDPs using Monte Carlo Control",
      "concept": [
        "solver"
      ],
      "topics": [
        "solve_MDP_MC"
      ]
    },
    {
      "page": "solve_MDP_PG",
      "title": "Solve MDPs with Policy Gradient Methods",
      "concept": [
        "MDPSample",
        "solver"
      ],
      "topics": [
        "solve_MDP_PG"
      ]
    },
    {
      "page": "solve_MDP_SAMP",
      "title": "Solve MDPs using Random-Sampling",
      "concept": [
        "solver"
      ],
      "topics": [
        "solve_MDP_SAMP"
      ]
    },
    {
      "page": "solve_MDP_TD",
      "title": "Solve MDPs using Tabular Temporal Differencing",
      "concept": [
        "solver"
      ],
      "topics": [
        "solve_MDP_TD"
      ]
    },
    {
      "page": "start",
      "title": "Sample a Start State",
      "concept": [
        "MDP",
        "MDPSample"
      ],
      "topics": [
        "start",
        "start.MDPModel",
        "start.MDPSample"
      ]
    },
    {
      "page": "transformation",
      "title": "Transformation Functions for Linear Function Approximation",
      "topics": [
        "create_basis_coefs",
        "transformation",
        "transformation_fourier_basis",
        "transformation_linear_basis",
        "transformation_polynomial_basis",
        "transformation_RBF_basis"
      ]
    },
    {
      "page": "transition_graph",
      "title": "Transition Graph",
      "concept": [
        "MDP"
      ],
      "topics": [
        "curve_multiple_directed",
        "plot_transition_graph",
        "transition_graph"
      ]
    },
    {
      "page": "accessors",
      "title": "Access to Parts of the Model Description",
      "concept": [
        "MDP"
      ],
      "topics": [
        "accessors",
        "normalize_MDP",
        "reward_matrix",
        "start_vector",
        "transition_matrix"
      ]
    },
    {
      "page": "unreachable_states",
      "title": "Unreachable States",
      "concept": [
        "MDP"
      ],
      "topics": [
        "remove_unreachable_states",
        "unreachable_states"
      ]
    },
    {
      "page": "value_function",
      "title": "Value Function",
      "concept": [
        "MDP",
        "policy"
      ],
      "topics": [
        "plot_value_function",
        "value_function",
        "V_random",
        "V_zero"
      ]
    },
    {
      "page": "visit_probability",
      "title": "State Visit Probability",
      "concept": [
        "policy"
      ],
      "topics": [
        "visit_probability"
      ]
    },
    {
      "page": "Windy_gridworld",
      "title": "Windy Gridworld MDP Windy Gridworld MDP",
      "concept": [
        "MDP_examples",
        "gridworld"
      ],
      "topics": [
        "Windy_gridworld",
        "windy_gridworld"
      ]
    }
  ],
  "_pkglogo": "https://github.com/mhahsler/markovDP/raw/HEAD/man/figures/logo.svg",
  "_readme": "https://github.com/mhahsler/markovDP/raw/HEAD/README.md",
  "_rundeps": [
    "cli",
    "codetools",
    "cpp11",
    "crayon",
    "fastmap",
    "float",
    "foreach",
    "glue",
    "hms",
    "igraph",
    "iterators",
    "lattice",
    "lifecycle",
    "lpSolve",
    "magrittr",
    "Matrix",
    "MatrixExtra",
    "pkgconfig",
    "prettyunits",
    "progress",
    "R6",
    "Rcpp",
    "RhpcBLASctl",
    "rlang",
    "vctrs"
  ],
  "_sysdeps": [
    {
      "shlib": "libstdc++",
      "package": "libstdc++6",
      "source": "gcc",
      "version": "14.2.0-4ubuntu2~24.04.1",
      "name": "c++",
      "homepage": "http://gcc.gnu.org/",
      "description": "GNU Standard C++ Library v3"
    }
  ],
  "_vignettes": [
    {
      "source": "Gridworlds.Rmd",
      "filename": "Gridworlds.html",
      "title": "Gridworlds as MDPs",
      "author": "Michael Hahsler",
      "engine": "knitr::rmarkdown",
      "headings": [
        "Introduction",
        "Defining a Gridworld",
        "Working with Gridworld MDPs",
        "Solving a Gridworld",
        "Experimenting with Solvers",
        "References"
      ],
      "created": "2025-02-20 20:34:01",
      "modified": "2025-02-20 20:34:01",
      "commits": 1
    },
    {
      "source": "markovDP.Rmd",
      "filename": "markovDP.html",
      "title": "Introduction to Discrete-Time Markov Decision Processes",
      "author": "Michael Hahsler",
      "engine": "knitr::rmarkdown",
      "headings": [
        "Introduction",
        "Markov Decision Processes",
        "Definition",
        "Notation Used in the Package",
        "Package Functionality",
        "Defining an MDP Problem",
        "Solving a MDP",
        "Toy Example: Steward Russell's 4x3 Maze Gridworld MDP",
        "Specifying the Stochastic Maze",
        "Solving the Maze",
        "Additional Functions",
        "Access to Model Components",
        "Value Function",
        "Policy",
        "Evaluation",
        "Sampling",
        "Acknowledgments",
        "References"
      ],
      "created": "2024-05-31 18:46:21",
      "modified": "2025-07-19 19:13:10",
      "commits": 15
    },
    {
      "source": "LinearApproximation.Rmd",
      "filename": "LinearApproximation.html",
      "title": "Solving MDPs with Linear Approximation",
      "author": "Michael Hahsler",
      "engine": "knitr::rmarkdown",
      "headings": [
        "Introduction",
        "Linear Approximation",
        "State-action Feature Vector Construction",
        "Helper Functions",
        "Episodic Semi-gradient Sarsa",
        "Example 1: A maze without walls",
        "Example 2: Stuart Russell's 3x4 Maze using Linear Approximation",
        "Linear Basis",
        "Order-1 Polynomial Basis",
        "Radial Basis Function",
        "Order-1 Fourier Basis",
        "Example 3: Wall Maze",
        "References"
      ],
      "created": "2025-02-20 20:34:01",
      "modified": "2025-06-18 21:00:21",
      "commits": 3
    },
    {
      "source": "TicTacToe.Rmd",
      "filename": "TicTacToe.html",
      "title": "Solving Tic-Tac-Toe as a MDP",
      "author": "Michael Hahsler",
      "engine": "knitr::rmarkdown",
      "headings": [
        "Introduction",
        "Defining Tic-Tac-Toe as an MDP",
        "Actions",
        "State Space",
        "Transition Model",
        "Reward Function",
        "Constructing the MDP",
        "Define Test Boards",
        "Solve",
        "LP",
        "Value Iteration",
        "Policy Iteration",
        "Q-Learning",
        "References"
      ],
      "created": "2024-09-01 04:46:42",
      "modified": "2025-05-15 21:27:36",
      "commits": 4
    }
  ],
  "_score": 4.664641975556126,
  "_indexed": true,
  "_nocasepkg": "markovdp",
  "_universes": [
    "mhahsler"
  ],
  "_binaries": [
    {
      "r": "4.7.0",
      "os": "linux",
      "version": "0.99.0",
      "date": "2026-06-02T09:27:49.000Z",
      "distro": "noble",
      "arch": "aarch64",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "3203c00e29e4e94035240cb177eea3a9ac70553fd45838d0bfb9020e9ebe7c64",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    },
    {
      "r": "4.7.0",
      "os": "linux",
      "version": "0.99.0",
      "date": "2026-06-02T09:27:51.000Z",
      "distro": "noble",
      "arch": "x86_64",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "e77cc30111aaf376cd0271a2ded5d3d8c32acf862d892c78ce177171b300e6ad",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    },
    {
      "r": "4.6.0",
      "os": "linux",
      "version": "0.99.0",
      "date": "2026-06-02T09:28:01.000Z",
      "distro": "noble",
      "arch": "aarch64",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "e2e7576739a8451cd898155abe0a5280b2c94db5fc5f3299f5a5d0e5643c872c",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    },
    {
      "r": "4.6.0",
      "os": "linux",
      "version": "0.99.0",
      "date": "2026-06-02T09:27:37.000Z",
      "distro": "noble",
      "arch": "x86_64",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "f76a537a3780794e042428c225e4195f65dd8875ee0b796d87ec8a3683489540",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    },
    {
      "r": "4.5.3",
      "os": "mac",
      "version": "0.99.0",
      "date": "2026-06-02T09:27:14.000Z",
      "arch": "aarch64",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "f52e98967196aecdf689eace82b4b2ba5c1585bc97c866fdc88d2c979c9123cf",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    },
    {
      "r": "4.5.3",
      "os": "mac",
      "version": "0.99.0",
      "date": "2026-06-02T09:28:36.000Z",
      "arch": "x86_64",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "e5fcf762b95f85d055d4180b2983359448546ec294e8c9669fb57109bad5c27e",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    },
    {
      "r": "4.6.0",
      "os": "mac",
      "version": "0.99.0",
      "date": "2026-06-02T09:26:56.000Z",
      "arch": "aarch64",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "0ddc2efd1d646d395c6220b11b01697aa280195b9e4a2737ff8c82ae4cdaa17e",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    },
    {
      "r": "4.6.0",
      "os": "mac",
      "version": "0.99.0",
      "date": "2026-06-02T09:28:31.000Z",
      "arch": "x86_64",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "3cf4d41053186575a777386fcf740e5f112e209f89cbb794f7300f0b765acfaa",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    },
    {
      "r": "4.6.0",
      "os": "wasm",
      "version": "0.99.0",
      "date": "2026-06-02T09:27:48.000Z",
      "arch": "emscripten",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "74a4198e21dfe6b309f7df9eadf03031cceea384c4049bb71d71984b477903a3",
      "status": "success",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    },
    {
      "r": "4.7.0",
      "os": "win",
      "version": "0.99.0",
      "date": "2026-06-02T09:27:41.000Z",
      "arch": "x86_64",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "948690490f89ee9628f528631ad514a072ef78d12a89a376109ed8a0794dad02",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    },
    {
      "r": "4.5.3",
      "os": "win",
      "version": "0.99.0",
      "date": "2026-06-02T09:26:51.000Z",
      "arch": "x86_64",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "d413c1dee2568aa38b25891263a865f49d6e11cf99a3fb93db7603b69611c35e",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    },
    {
      "r": "4.6.0",
      "os": "win",
      "version": "0.99.0",
      "date": "2026-06-02T09:26:56.000Z",
      "arch": "x86_64",
      "commit": "e8ab3f595ec64fc5e97be7b5faec9dea0126a505",
      "fileid": "21355640ceaa2bb7e55702390e15b9b1e036c019ac9d2fbaaa12b4d10a64a7a6",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/mhahsler/actions/runs/26810549572"
    }
  ]
}