Solutions for week 9

solutions/ex09/gambler_TODO_1.py

+        return state in [0, self.goal]
\ No newline at end of file

solutions/ex09/gambler_TODO_2.py

+        return list( range(1, min(s, self.goal - s) + 1))
\ No newline at end of file

solutions/ex09/gambler_TODO_3.py

+        r = 1 if s + a == 100 else 0
+        WIN = (s+a, r)
+        LOSS = (s-a, 0)
+        outcome_dict = {WIN: self.p_heads, LOSS: 1-self.p_heads } if WIN != LOSS else {WIN: 1.} 
\ No newline at end of file

solutions/ex09/mdp_warmup_TODO_1.py

+    q_dict = {a: sum([p*(r+ (gamma*v[sp] if not mdp.is_terminal(sp) else 0)) for (sp,r), p in mdp.Psr(s,a).items()]) for a in mdp.A(s)} 
\ No newline at end of file

solutions/ex09/mdp_warmup_TODO_2.py

+    raise NotImplementedError("Insert your solution and remove this error.")
\ No newline at end of file

solutions/ex09/mdp_warmup_TODO_3.py

+    expected_reward = sum( [r * p for (sp, r), p in mdp.Psr(s, a).items() ] ) 
\ No newline at end of file

solutions/ex09/mdp_warmup_TODO_4.py

+    V_s = sum( [Q[s,a] * p for a, p in policy.items()] ) 
\ No newline at end of file

solutions/ex09/policy_evaluation_TODO_1.py

+            q = value_function2q_function(mdp, s, gamma, v) 
+            v_, v[s] = v[s], sum( [q[a] * pi_a for a,pi_a in pi[s].items()] ) 
\ No newline at end of file

solutions/ex09/policy_iteration_TODO_1.py

+        for s in [mdp.nonterminal_states[i] for i in np.random.permutation(len(mdp.nonterminal_states))]:  
+            old_a = pi[s] # The best action we would take under the current policy
+            Qs = value_function2q_function(mdp, s, gamma, V)
+            pi[s] = max(Qs, key=Qs.get)
+            if old_a != pi[s]:
+                policy_stable = False 
\ No newline at end of file

solutions/ex09/value_iteration_TODO_1.py

+            v, V[s] = V[s], max(value_function2q_function(mdp, s, gamma, V).values()) if len(mdp.A(s)) > 0 else 0    
+            Delta = max(Delta, np.abs(v - V[s])) 
\ No newline at end of file

solutions/ex09/value_iteration_TODO_2.py

+        Q = {a: v-(1e-8*a if isinstance(a, int) else 0) for a,v in value_function2q_function(mdp, s, gamma, V).items()} 
+        pi[s] = max(Q, key=Q.get) 
\ No newline at end of file

solutions/ex09/value_iteration_agent_TODO_1.py

+        self.policy, self.v = value_iteration(mdp, gamma=gamma, **kwargs) 
\ No newline at end of file

solutions/ex09/value_iteration_agent_TODO_2.py

+            action = self.policy[s] 
\ No newline at end of file