Skip to content

Reward API Reference

IsingRewardModule

Bases: BaseRewardModule[IsingEnvState, IsingEnvParams]

Source code in gfnx/reward/ising.py 
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
class IsingRewardModule(BaseRewardModule[IsingEnvState, IsingEnvParams]):
    def init(
        self,
        rng_key: chex.PRNGKey,
        dummy_state: IsingEnvState,
    ) -> TRewardParams:
        dim = dummy_state.state.shape[-1]
        J = jnp.zeros((dim, dim))
        return IsingRewardParams(J=J)

    def reward(self, state: IsingEnvState, env_params: IsingEnvParams) -> TReward:
        return jnp.exp(self.log_reward(state, env_params))

    def log_reward(self, state: IsingEnvState, env_params: IsingEnvParams) -> TLogReward:
        """Compute log reward for Ising model states.

        Args:
        - state: IsingEnvState with state field of shape [B, dim] containing values in {0, 1}
        - env_params: Environment parameters containing reward_params with alpha and J

        Returns:
        - Log reward tensor of shape [B] for each state in the batch

        The Ising model energy is computed as:
            E = -alpha * sum_{i,j} J_{ij} * s_i * s_j
        where s_i are the spin values in {-1, 1} (transformed from {0, 1} input).

        The log reward is simply -E, so higher energy states have lower reward.
        """
        canonical = 2 * state.state - 1
        J = env_params.reward_params.J
        return jnp.einsum("bi,ij,bj->b", canonical, J, canonical)

log_reward(state, env_params)

Compute log reward for Ising model states.

Args: - state: IsingEnvState with state field of shape [B, dim] containing values in {0, 1} - env_params: Environment parameters containing reward_params with alpha and J

Returns: - Log reward tensor of shape [B] for each state in the batch

The Ising model energy is computed as

E = -alpha * sum_{i,j} J_{ij} * s_i * s_j

where s_i are the spin values in {-1, 1} (transformed from {0, 1} input).

The log reward is simply -E, so higher energy states have lower reward.

Source code in gfnx/reward/ising.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
def log_reward(self, state: IsingEnvState, env_params: IsingEnvParams) -> TLogReward:
    """Compute log reward for Ising model states.

    Args:
    - state: IsingEnvState with state field of shape [B, dim] containing values in {0, 1}
    - env_params: Environment parameters containing reward_params with alpha and J

    Returns:
    - Log reward tensor of shape [B] for each state in the batch

    The Ising model energy is computed as:
        E = -alpha * sum_{i,j} J_{ij} * s_i * s_j
    where s_i are the spin values in {-1, 1} (transformed from {0, 1} input).

    The log reward is simply -E, so higher energy states have lower reward.
    """
    canonical = 2 * state.state - 1
    J = env_params.reward_params.J
    return jnp.einsum("bi,ij,bj->b", canonical, J, canonical)