# Module `bbrl.agents.remote`

Expand source code

    # This source code is licensed under the MIT license found in the
    # LICENSE file in the root directory of this source tree.
    #

    import copy
    import time

    import torch
    import torch.multiprocessing as mp

    from bbrl.agents.agent import Agent
    from bbrl.workspace import Workspace, _SplitSharedWorkspace

    # Unused???
    def f(agent, in_queue, out_queue, seed, verbose):
        """The function that is executed in a single process"""
        out_queue.put("ok")
        running = True
        old_workspace = None
        if verbose:
            print("Seeding remote agent with ", seed)
        agent.seed(seed)
        while running:
            command = in_queue.get()
            if command[0] == "go_new_workspace":
                _, workspace, kwargs = command
                old_workspace = workspace
                agent(workspace, **kwargs)
                out_queue.put("ok")
            elif command[0] == "go_reuse_workspace":
                _, _, kwargs = command
                agent(old_workspace, **kwargs)
                out_queue.put("ok")
            elif command[0] == "exit":
                out_queue.put("ok")
                return
            elif command[0] == "eval_mode":
                agent.eval()
                out_queue.put("ok")
            elif command[0] == "train_mode":
                agent.train()
                out_queue.put("ok")

    class RemoteAgent(Agent):
        """It corresponds to an agent that is executed in another process

        Args:
            Agent ([bbrl.Agent]): the agent ot execute in another process
        """

        def __init__(self, agent, name=None, verbose=False):
            super().__init__(name=name, verbose=verbose)
            self.agent = agent
            self._is_running = False
            self.process = None
            self.last_workspace = None
            self.train_mode = True

        def get_by_name(self, n):
            if self._name == n:
                return [self] + self.agent.get_by_name(n)
            else:
                return self.agent.get_by_name(n)

        def forward(self, **kwargs):
            raise NotImplementedError

        def _create_process(self):
            if self.verbose:
                print("[RemoteAgent] starting process...")
            self.i_queue = mp.Queue()
            self.o_queue = mp.Queue()
            self.i_queue.cancel_join_thread()
            self.o_queue.cancel_join_thread()
            self.process = mp.Process(
                target=f, args=(self.agent, self.i_queue, self.o_queue, self._seed)
            )
            self.process.daemon = False
            self.process.start()

        def __call__(self, workspace, **kwargs):
            with torch.no_grad():
                assert (
                    workspace.is_shared
                ), "You must use a shared workspace when using a Remote Agent"
                if self.process is None:
                    self._create_process()
                    self.train(self.train_mode)
                if not workspace == self.last_workspace:
                    self.i_queue.put(("go_new_workspace", workspace, kwargs))
                    self.last_workspace = workspace
                    r = self.o_queue.get()
                    assert r == "ok"
                else:
                    self.i_queue.put(("go_reuse_workspace", workspace, kwargs))
                    r = self.o_queue.get()
                    assert r == "ok"

        def _asynchronous_call(self, workspace, **kwargs):
            """Non-blocking forward. To use together with `is_running`"""
            with torch.no_grad():
                self._is_running = True
                assert (
                    workspace.is_shared
                ), "You must use a shared workspace when using a Remote Agent"
                if self.process is None:
                    self._create_process()
                if not workspace == self.last_workspace:
                    self.i_queue.put(("go_new_workspace", workspace, kwargs))
                    self.last_workspace = workspace
                else:
                    self.i_queue.put(("go_reuse_workspace", workspace, kwargs))

        def train(self, f=True):
            self.train_mode = f
            if self.process is None:
                return
            if f:
                self.i_queue.put(("train_mode",))
                a = self.o_queue.get()
                assert a == "ok"
            else:
                self.eval()

        def eval(self):
            self.train_mode = False
            if self.process is None:
                return
            self.i_queue.put(("eval_mode",))
            a = self.o_queue.get()
            assert a == "ok"

        def seed(self, _seed):
            self._seed = _seed

        def _running_queue(self):
            return self.o_queue

        def is_running(self):
            if self._is_running:
                try:
                    r = self.o_queue.get(False)
                    assert r == "ok"
                    self._is_running = False
                except:
                    pass
            return self._is_running

        def close(self):
            if self.process is None:
                return

            if self.verbose:
                print("[RemoteAgent] closing process")
            self.i_queue.put(("exit",))
            self.o_queue.get()
            time.sleep(0.1)
            self.process.terminate()
            self.process.join()
            self.i_queue.close()
            self.o_queue.close()
            time.sleep(0.1)
            del self.i_queue
            del self.o_queue
            self.process = None

        def __del__(self):
            self.close()

    class NRemoteAgent(Agent):
        """Multiple agents executed in different processes. Use the `NRemoteAgent.create` function to create such an agent"""

        def __init__(self, agents, batch_dims):
            super().__init__()
            self.agents = agents
            self.batch_dims = batch_dims

        def get_by_name(self, name):
            r = []
            if self._name == name:
                r = [self]
            for a in self.agents:
                r = r + a.get_by_name(name)
            return r

        @staticmethod
        def create(agent, num_processes=0, time_size=None, **extra_kwargs):
            """Returns a NRemote agent with num_processes copies of agent in different processes
            Also returns the specific workspace to use with such an agent

            Args:
                agent ([bbrl.Agent]): The agent to execute in multiple processes
                num_processes (int, optional): Number of processes to create. If 0, then no processes are created (for debugging). Defaults to 0.
                time_size ([type], optional): If specified, it forces the created Workspace to have this particular time_size. Defaults to None.

            Returns:
                [bbrl.Agent,bbrl.SharedWorkspace]: The NRemoteAgent and the corresponding workspace
            """
            agent.seed(0)
            if num_processes == 0:
                workspace = Workspace()
                _agent = copy.deepcopy(agent)
                agent(workspace, **extra_kwargs)
                shared_workspace = workspace._convert_to_shared_workspace(
                    n_repeat=1, time_size=time_size
                )
                return _agent, shared_workspace

            workspace = Workspace()
            agents = [copy.deepcopy(agent) for t in range(num_processes)]
            agent(workspace, **extra_kwargs)
            b = workspace.batch_size()
            batch_dims = [(k * b, k * b + b) for k, a in enumerate(agents)]
            shared_workspace = workspace._convert_to_shared_workspace(
                n_repeat=num_processes, time_size=time_size
            )
            agents = [RemoteAgent(a) for a in agents]
            return NRemoteAgent(agents, batch_dims), shared_workspace

        def __call__(self, workspace, **kwargs):
            assert workspace.is_shared
            for k in range(len(self.agents)):
                _workspace = _SplitSharedWorkspace(workspace, self.batch_dims[k])
                self.agents[k]._asynchronous_call(_workspace, **kwargs)
            for a in self.agents:
                ok = a._running_queue().get()
                assert ok == "ok"

        def seed(self, seed, inc=1):
            s = seed
            for a in self.agents:
                a.seed(s)
                s += inc

        def _asynchronous_call(self, workspace, **kwargs):
            assert workspace.is_shared
            for k in range(len(self.agents)):
                _workspace = _SplitSharedWorkspace(workspace, self.batch_dims[k])
                self.agents[k]._asynchronous_call(_workspace, **kwargs)

        def is_running(self):
            for a in self.agents:
                if a.is_running():
                    return True
            return False

        def train(self, f=True):
            for a in self.agents:
                a.train(f)

        def eval(self):
            for a in self.agents:
                a.eval()

        def close(self):
            for a in self.agents:
                a.close()

## Functions

`def f(agent, in_queue, out_queue, seed, verbose)`

  The function that is executed in a single process

  Expand source code

      def f(agent, in_queue, out_queue, seed, verbose):
          """The function that is executed in a single process"""
          out_queue.put("ok")
          running = True
          old_workspace = None
          if verbose:
              print("Seeding remote agent with ", seed)
          agent.seed(seed)
          while running:
              command = in_queue.get()
              if command[0] == "go_new_workspace":
                  _, workspace, kwargs = command
                  old_workspace = workspace
                  agent(workspace, **kwargs)
                  out_queue.put("ok")
              elif command[0] == "go_reuse_workspace":
                  _, _, kwargs = command
                  agent(old_workspace, **kwargs)
                  out_queue.put("ok")
              elif command[0] == "exit":
                  out_queue.put("ok")
                  return
              elif command[0] == "eval_mode":
                  agent.eval()
                  out_queue.put("ok")
              elif command[0] == "train_mode":
                  agent.train()
                  out_queue.put("ok")

## Classes

`class NRemoteAgent (agents, batch_dims)`

  Multiple agents executed in different processes. Use the `NRemoteAgent.create()` function to create such an agent

  To create a new Agent

  ## Args

  `name` :`[type]`, optional
    An agent can have a name that will allow to perform operations

  on agents that are composed into more complex agents.

  Expand source code

      class NRemoteAgent(Agent):
          """Multiple agents executed in different processes. Use the `NRemoteAgent.create` function to create such an agent"""

          def __init__(self, agents, batch_dims):
              super().__init__()
              self.agents = agents
              self.batch_dims = batch_dims

          def get_by_name(self, name):
              r = []
              if self._name == name:
                  r = [self]
              for a in self.agents:
                  r = r + a.get_by_name(name)
              return r

          @staticmethod
          def create(agent, num_processes=0, time_size=None, **extra_kwargs):
              """Returns a NRemote agent with num_processes copies of agent in different processes
              Also returns the specific workspace to use with such an agent

              Args:
                  agent ([bbrl.Agent]): The agent to execute in multiple processes
                  num_processes (int, optional): Number of processes to create. If 0, then no processes are created (for debugging). Defaults to 0.
                  time_size ([type], optional): If specified, it forces the created Workspace to have this particular time_size. Defaults to None.

              Returns:
                  [bbrl.Agent,bbrl.SharedWorkspace]: The NRemoteAgent and the corresponding workspace
              """
              agent.seed(0)
              if num_processes == 0:
                  workspace = Workspace()
                  _agent = copy.deepcopy(agent)
                  agent(workspace, **extra_kwargs)
                  shared_workspace = workspace._convert_to_shared_workspace(
                      n_repeat=1, time_size=time_size
                  )
                  return _agent, shared_workspace

              workspace = Workspace()
              agents = [copy.deepcopy(agent) for t in range(num_processes)]
              agent(workspace, **extra_kwargs)
              b = workspace.batch_size()
              batch_dims = [(k * b, k * b + b) for k, a in enumerate(agents)]
              shared_workspace = workspace._convert_to_shared_workspace(
                  n_repeat=num_processes, time_size=time_size
              )
              agents = [RemoteAgent(a) for a in agents]
              return NRemoteAgent(agents, batch_dims), shared_workspace

          def __call__(self, workspace, **kwargs):
              assert workspace.is_shared
              for k in range(len(self.agents)):
                  _workspace = _SplitSharedWorkspace(workspace, self.batch_dims[k])
                  self.agents[k]._asynchronous_call(_workspace, **kwargs)
              for a in self.agents:
                  ok = a._running_queue().get()
                  assert ok == "ok"

          def seed(self, seed, inc=1):
              s = seed
              for a in self.agents:
                  a.seed(s)
                  s += inc

          def _asynchronous_call(self, workspace, **kwargs):
              assert workspace.is_shared
              for k in range(len(self.agents)):
                  _workspace = _SplitSharedWorkspace(workspace, self.batch_dims[k])
                  self.agents[k]._asynchronous_call(_workspace, **kwargs)

          def is_running(self):
              for a in self.agents:
                  if a.is_running():
                      return True
              return False

          def train(self, f=True):
              for a in self.agents:
                  a.train(f)

          def eval(self):
              for a in self.agents:
                  a.eval()

          def close(self):
              for a in self.agents:
                  a.close()

  ### Ancestors

  - Agent
  - torch.nn.modules.module.Module

  ### Class variables

  `var dump_patches : bool`
  `var training : bool`

  ### Static methods

  `def create(agent, num_processes=0, time_size=None, **extra_kwargs)`

    Returns a NRemote agent with num_processes copies of agent in different processes Also returns the specific workspace to use with such an agent

    ## Args

    `agent` :`[bbrl.Agent]`
      The agent to execute in multiple processes
    `num_processes` :`int`, optional
      Number of processes to create. If 0, then no processes are created (for debugging). Defaults to 0.
    `time_size` :`[type]`, optional
      If specified, it forces the created Workspace to have this particular time_size. Defaults to None.

    ## Returns

    `[bbrl.Agent,bbrl.SharedWorkspace]`
      The NRemoteAgent and the corresponding workspace
    Expand source code

        @staticmethod
        def create(agent, num_processes=0, time_size=None, **extra_kwargs):
            """Returns a NRemote agent with num_processes copies of agent in different processes
            Also returns the specific workspace to use with such an agent

            Args:
                agent ([bbrl.Agent]): The agent to execute in multiple processes
                num_processes (int, optional): Number of processes to create. If 0, then no processes are created (for debugging). Defaults to 0.
                time_size ([type], optional): If specified, it forces the created Workspace to have this particular time_size. Defaults to None.

            Returns:
                [bbrl.Agent,bbrl.SharedWorkspace]: The NRemoteAgent and the corresponding workspace
            """
            agent.seed(0)
            if num_processes == 0:
                workspace = Workspace()
                _agent = copy.deepcopy(agent)
                agent(workspace, **extra_kwargs)
                shared_workspace = workspace._convert_to_shared_workspace(
                    n_repeat=1, time_size=time_size
                )
                return _agent, shared_workspace

            workspace = Workspace()
            agents = [copy.deepcopy(agent) for t in range(num_processes)]
            agent(workspace, **extra_kwargs)
            b = workspace.batch_size()
            batch_dims = [(k * b, k * b + b) for k, a in enumerate(agents)]
            shared_workspace = workspace._convert_to_shared_workspace(
                n_repeat=num_processes, time_size=time_size
            )
            agents = [RemoteAgent(a) for a in agents]
            return NRemoteAgent(agents, batch_dims), shared_workspace

  ### Methods

  `def close(self)`
    Expand source code

        def close(self):
            for a in self.agents:
                a.close()

  `def eval(self)`

    Sets the module in evaluation mode.

    This has any effect only on certain modules. See documentations of particular modules for details of their behaviors in training/evaluation mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`, etc.

    This is equivalent with :meth:`self.train(False) <torch.nn.Module.train>`.

    See :ref:`locally-disable-grad-doc` for a comparison between `.eval()` and several similar mechanisms that may be confused with it.

    ## Returns

    `Module`
      self
    Expand source code

        def eval(self):
            for a in self.agents:
                a.eval()

  `def train(self, f=True)`

    Sets the module in training mode.

    This has any effect only on certain modules. See documentations of particular modules for details of their behaviors in training/evaluation mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`, etc.

    ## Args

    `mode` :`bool`
      whether to set training mode (`True`) or evaluation mode (`False`). Default: `True`.

    ## Returns

    `Module`
      self
    Expand source code

        def train(self, f=True):
            for a in self.agents:
                a.train(f)

  ### Inherited members

  - `Agent`:
    - `clone`
    - `forward`
    - `get`
    - `get_by_name`
    - `get_name`
    - `get_time_truncated`
    - `is_running`
    - `load_model`
    - `save_model`
    - `seed`
    - `set`
    - `set_name`
`class RemoteAgent (agent, name=None, verbose=False)`

  It corresponds to an agent that is executed in another process

  ## Args

  `Agent` :`[bbrl.Agent]`
    the agent ot execute in another process

  To create a new Agent

  ## Args

  `name` :`[type]`, optional
    An agent can have a name that will allow to perform operations

  on agents that are composed into more complex agents.

  Expand source code

      class RemoteAgent(Agent):
          """It corresponds to an agent that is executed in another process

          Args:
              Agent ([bbrl.Agent]): the agent ot execute in another process
          """

          def __init__(self, agent, name=None, verbose=False):
              super().__init__(name=name, verbose=verbose)
              self.agent = agent
              self._is_running = False
              self.process = None
              self.last_workspace = None
              self.train_mode = True

          def get_by_name(self, n):
              if self._name == n:
                  return [self] + self.agent.get_by_name(n)
              else:
                  return self.agent.get_by_name(n)

          def forward(self, **kwargs):
              raise NotImplementedError

          def _create_process(self):
              if self.verbose:
                  print("[RemoteAgent] starting process...")
              self.i_queue = mp.Queue()
              self.o_queue = mp.Queue()
              self.i_queue.cancel_join_thread()
              self.o_queue.cancel_join_thread()
              self.process = mp.Process(
                  target=f, args=(self.agent, self.i_queue, self.o_queue, self._seed)
              )
              self.process.daemon = False
              self.process.start()

          def __call__(self, workspace, **kwargs):
              with torch.no_grad():
                  assert (
                      workspace.is_shared
                  ), "You must use a shared workspace when using a Remote Agent"
                  if self.process is None:
                      self._create_process()
                      self.train(self.train_mode)
                  if not workspace == self.last_workspace:
                      self.i_queue.put(("go_new_workspace", workspace, kwargs))
                      self.last_workspace = workspace
                      r = self.o_queue.get()
                      assert r == "ok"
                  else:
                      self.i_queue.put(("go_reuse_workspace", workspace, kwargs))
                      r = self.o_queue.get()
                      assert r == "ok"

          def _asynchronous_call(self, workspace, **kwargs):
              """Non-blocking forward. To use together with `is_running`"""
              with torch.no_grad():
                  self._is_running = True
                  assert (
                      workspace.is_shared
                  ), "You must use a shared workspace when using a Remote Agent"
                  if self.process is None:
                      self._create_process()
                  if not workspace == self.last_workspace:
                      self.i_queue.put(("go_new_workspace", workspace, kwargs))
                      self.last_workspace = workspace
                  else:
                      self.i_queue.put(("go_reuse_workspace", workspace, kwargs))

          def train(self, f=True):
              self.train_mode = f
              if self.process is None:
                  return
              if f:
                  self.i_queue.put(("train_mode",))
                  a = self.o_queue.get()
                  assert a == "ok"
              else:
                  self.eval()

          def eval(self):
              self.train_mode = False
              if self.process is None:
                  return
              self.i_queue.put(("eval_mode",))
              a = self.o_queue.get()
              assert a == "ok"

          def seed(self, _seed):
              self._seed = _seed

          def _running_queue(self):
              return self.o_queue

          def is_running(self):
              if self._is_running:
                  try:
                      r = self.o_queue.get(False)
                      assert r == "ok"
                      self._is_running = False
                  except:
                      pass
              return self._is_running

          def close(self):
              if self.process is None:
                  return

              if self.verbose:
                  print("[RemoteAgent] closing process")
              self.i_queue.put(("exit",))
              self.o_queue.get()
              time.sleep(0.1)
              self.process.terminate()
              self.process.join()
              self.i_queue.close()
              self.o_queue.close()
              time.sleep(0.1)
              del self.i_queue
              del self.o_queue
              self.process = None

          def __del__(self):
              self.close()

  ### Ancestors

  - Agent
  - torch.nn.modules.module.Module

  ### Class variables

  `var dump_patches : bool`
  `var training : bool`

  ### Methods

  `def close(self)`
    Expand source code

        def close(self):
            if self.process is None:
                return

            if self.verbose:
                print("[RemoteAgent] closing process")
            self.i_queue.put(("exit",))
            self.o_queue.get()
            time.sleep(0.1)
            self.process.terminate()
            self.process.join()
            self.i_queue.close()
            self.o_queue.close()
            time.sleep(0.1)
            del self.i_queue
            del self.o_queue
            self.process = None

  `def eval(self)`

    Sets the module in evaluation mode.

    This has any effect only on certain modules. See documentations of particular modules for details of their behaviors in training/evaluation mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`, etc.

    This is equivalent with :meth:`self.train(False) <torch.nn.Module.train>`.

    See :ref:`locally-disable-grad-doc` for a comparison between `.eval()` and several similar mechanisms that may be confused with it.

    ## Returns

    `Module`
      self
    Expand source code

        def eval(self):
            self.train_mode = False
            if self.process is None:
                return
            self.i_queue.put(("eval_mode",))
            a = self.o_queue.get()
            assert a == "ok"

  `def train(self, f=True)`

    Sets the module in training mode.

    This has any effect only on certain modules. See documentations of particular modules for details of their behaviors in training/evaluation mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`, etc.

    ## Args

    `mode` :`bool`
      whether to set training mode (`True`) or evaluation mode (`False`). Default: `True`.

    ## Returns

    `Module`
      self
    Expand source code

        def train(self, f=True):
            self.train_mode = f
            if self.process is None:
                return
            if f:
                self.i_queue.put(("train_mode",))
                a = self.o_queue.get()
                assert a == "ok"
            else:
                self.eval()

  ### Inherited members

  - `Agent`:
    - `clone`
    - `forward`
    - `get`
    - `get_by_name`
    - `get_name`
    - `get_time_truncated`
    - `is_running`
    - `load_model`
    - `save_model`
    - `seed`
    - `set`
    - `set_name`