vipl_priv_conn_mgr.c

Go to the documentation of this file.

/********************************************************
 * Myricom VI-GM  networking software and documentation *
 * Copyright (c) 2001 by Myricom, Inc.                  *
 * All rights reserved.                                 *
 * See the file `COPYING' for copyright notice.         *
 ********************************************************/

#ifndef WIN32
#include <unistd.h>
#endif
#include <string.h>

#include "vipl_priv.h"


typedef struct _VIP_CONN_MGR_ENTRY
{
  VIP_NET_UINT32 net_srv_magic;
  gm_alarm_t alarm;
}
VIP_ENDPOINT;


void
vip_conn_mgr_dead_endpoint (void *ptr)
{
  VIP_ENDPOINT *vip_endpoint_ptr;
  VIP_DEBUG_LABEL (("vip_conn_mgr_dead_endpoint"));

  vip_endpoint_ptr = (VIP_ENDPOINT *) ptr;
  VIP_ASSERT (VIP_NTOH_UINT32 (vip_endpoint_ptr->net_srv_magic) != 0);
  vip_endpoint_ptr->net_srv_magic = VIP_HTON_UINT32 (0);
  VIP_DEBUG (("Keep alive timeout, endpoint is deregistered"));
}

void
vip_conn_mgr_send_callback (struct gm_port *gm_port,
                            void *context, 
                            gm_status_t status)
{
  if (status != GM_SUCCESS)
    {
      VIP_PACKET_CONN_LIST *pkt_ptr;
      unsigned int gm_id;
      unsigned int port_id;
      
      pkt_ptr = (VIP_PACKET_CONN_LIST *) context;
      gm_unique_id_to_node_id (gm_port, pkt_ptr->gm_unique_id, &gm_id);
      port_id = pkt_ptr->port_id;
      fprintf (stderr, "Connection manager: error sending message "
               "to GM id %d / GM port %d (%s)\n", gm_id, port_id,
               gm_strerror (status));
      gm_resume_sending (gm_port, GM_LOW_PRIORITY, gm_id, port_id,
                         vip_conn_mgr_send_callback, context);
      return;
    }

  gm_provide_receive_buffer (gm_port, context, VI_GM_CTRL_GM_SIZE,
                             GM_LOW_PRIORITY);
}


int
main (void)
{
  struct gm_port *gm_port;
  gm_status_t gm_status;
  gm_alarm_t *alarm_ptr;
  gm_recv_event_t *gm_event;
  VIP_UCHAR port_id;
  VIP_UINT16 gm_id;
  unsigned int node_id;
  VIP_UINT32 i;
  VIP_UINT32 j;
  VIP_UINT32 recv_tokens;
  VIP_PACKET *pkt_ptr;
  VIP_PVOID recv_buffers;
  VIP_UCHAR gm_hostname[GM_MAX_HOST_NAME_LEN];
  VIP_UCHAR gm_nicname[GM_MAX_HOST_NAME_LEN];
  VIP_ENDPOINT endpoints[VI_GM_MAX_NICS][VI_GM_MAX_PORTS];
  VIP_UINT32 board_gm_ids[VI_GM_MAX_NICS];
  VIP_DEBUG_LABEL (("Connection manager"));


  /* open the GM port */
  i = 15;
  do
    {
      vip_millisleep ((15-i) * 1000);
      gm_status = gm_open (&gm_port, 0, VI_GM_PORT_CONN_MANAGER,
                           "VI-GM  Connection Manager", GM_API_VERSION_1_6);
    }
  while ((i-- > 0) && (gm_status == GM_STILL_SHUTTING_DOWN));

  if (gm_status != GM_SUCCESS)
    {
      fprintf (stderr, "Connection manager: Cannot open GM port %d "
               "on the Myrinet board 0 (%s)\n",
               VI_GM_PORT_CONN_MANAGER, gm_strerror (gm_status));
      goto out;
    }

  /* init endpoints matrix */
  gm_bzero (endpoints, (VI_GM_MAX_NICS * VI_GM_MAX_PORTS
                        * sizeof (VIP_ENDPOINT)));

  /* get hostame */
  gm_status = gm_get_host_name (gm_port, gm_hostname);
  if (gm_status != GM_SUCCESS)
    {
      fprintf (stderr, "Connection manager: Cannot get GM hostname (%s)\n",
               gm_strerror (gm_status));
      goto out_with_gm_port;
    }

  /* get the gm id */
  gm_status = gm_get_node_id (gm_port, &(board_gm_ids[0]));
  if (gm_status != GM_SUCCESS)
    {
      fprintf (stderr, "Connection manager: Cannot get GM id (%s)\n",
               gm_strerror (gm_status));
      goto out_with_gm_port;
    }

  /* fill GM id of boards, if any */
  for (i = 1; i < VI_GM_MAX_NICS; i++)
    {
      sprintf (gm_nicname, "%s:%d", gm_hostname, i);
      board_gm_ids[i] = gm_host_name_to_node_id (gm_port, gm_nicname);
    }

  /* init recv buffers */
  recv_tokens = gm_num_send_tokens (gm_port);
  recv_buffers = gm_dma_malloc (gm_port, recv_tokens
                                * sizeof (VIP_PACKET_CONN_LIST));

  if (recv_buffers == NULL)
    {
      fprintf (stderr, "Connection manager: Cannot allocate GM "
               "receive buffers, not enough DMA-able memory available\n");
      goto out_with_gm_port;
    }

  /* post recv buffers */
  for (i = 0; i < recv_tokens; i++)
    {
      gm_provide_receive_buffer (gm_port, &(((VIP_PACKET_CONN_LIST *)
                                             recv_buffers)[i]),
                                 VI_GM_CTRL_GM_SIZE, GM_LOW_PRIORITY);
    }

  /* recv loop */
  while (1)
    {
    begin_loop:
      gm_event = gm_blocking_receive_no_spin (gm_port);

      switch (gm_ntohc (gm_event->recv.type))
        {
        case GM_NO_RECV_EVENT:
          break;

        case GM_RECV_EVENT:
        case GM_HIGH_RECV_EVENT:
        case GM_PEER_RECV_EVENT:
        case GM_HIGH_PEER_RECV_EVENT:
          pkt_ptr = (VIP_PACKET *) (gm_ntohp (gm_event->recv.buffer));

          switch (VIP_NTOH_UCHAR (pkt_ptr->type))
            {
            case VIP_ALIVE_PKT:
              gm_id = gm_ntohs (gm_event->recv.sender_node_id);
              port_id = gm_ntohc (gm_event->recv.sender_port_id);
              VIP_DEBUG (("Received a ALIVE packet from %d/%d", gm_id,
                          port_id));

              for (i = 0; i < VI_GM_MAX_NICS; i++)
                {
                  if (board_gm_ids[i] == gm_id)
                    {
                      /* found the associated board index */
                      alarm_ptr = &(endpoints[i][port_id].alarm);
                      if (VIP_NTOH_UINT32
                          (endpoints[i][port_id].net_srv_magic) == 0)
                        {
                          VIP_DEBUG (("New VI-GM endpoint registered"));
                          endpoints[i][port_id].net_srv_magic =
                            pkt_ptr->vip_pkt_alive.net_srv_magic;
                          gm_initialize_alarm (alarm_ptr);
                        }
                      else
                        {
                          if (memcmp (&(endpoints[i][port_id].net_srv_magic),
                                      &(pkt_ptr->vip_pkt_alive.net_srv_magic),
                                      sizeof (VIP_NET_UINT32)) != 0)
                            {
                              VIP_DEBUG
                                (("New VI-GM endpoint replace old one"));
                              endpoints[i][port_id].net_srv_magic =
                                pkt_ptr->vip_pkt_alive.net_srv_magic;
                            }
                        }

                      /* set an alarm to remove dead endpoint */
                      gm_set_alarm (gm_port, alarm_ptr,
                                    (VI_GM_CONN_MGR_TTL * 1000),
                                    vip_conn_mgr_dead_endpoint,
                                    &(endpoints[i][port_id]));
                      i = VI_GM_MAX_NICS;
                    }
                }
              break;

            case VIP_CONN_LIST_REQ_PKT:
              gm_unique_id_to_node_id(gm_port, pkt_ptr->vip_pkt_conn_list.gm_unique_id, &node_id);
              gm_id = node_id;
              VIP_DEBUG (("Received a CONN_LIST request about GM_id %d "
                          "from (%d/%d)", gm_id,
                          gm_ntohs (gm_event->recv.sender_node_id), 
                          gm_ntohc (gm_event->recv.sender_port_id)));
              pkt_ptr->vip_pkt_conn_list.type =
                VIP_HTON_UCHAR (VIP_CONN_LIST_REPLY_PKT);

              for (i = 0; i < VI_GM_MAX_NICS; i++)
                {
                  if (board_gm_ids[i] == gm_id)
                    {
                      /* found the associated board index */
                      gm_id = gm_ntohs (gm_event->recv.sender_node_id);
                      port_id = gm_ntohc (gm_event->recv.sender_port_id);

                      for (j = 0; j < VI_GM_MAX_PORTS; j++)
                        {
                          pkt_ptr->vip_pkt_conn_list.net_srv_magic[j] =
                            endpoints[i][j].net_srv_magic;
                        }

                      pkt_ptr->vip_pkt_conn_list.port_id = port_id;
                      gm_node_id_to_unique_id(gm_port, gm_id, pkt_ptr->vip_pkt_conn_list.gm_unique_id);
                      gm_send_with_callback (gm_port, pkt_ptr,
                                             VI_GM_CTRL_GM_SIZE,
                                             sizeof (VIP_PACKET_CONN_LIST),
                                             GM_LOW_PRIORITY,
                                             gm_id, port_id,
                                             vip_conn_mgr_send_callback,
                                             pkt_ptr);
                      VIP_DEBUG (("Send a CONN_LIST reply for board %d", i));
                      goto begin_loop;
                    }
                }
              break;

            default:
              VIP_DEBUG (("Unknown packet type received (%d)",
                          VIP_NTOH_UCHAR (pkt_ptr->type)));
            }

          gm_provide_receive_buffer (gm_port, pkt_ptr,
                                     VI_GM_CTRL_GM_SIZE, GM_LOW_PRIORITY);
          break;

        default:
          gm_unknown (gm_port, gm_event);
        }
    }

  gm_dma_free (gm_port, recv_buffers);

out_with_gm_port:
  gm_close (gm_port);
out:
  return 1;
}