vipopennic.c

Go to the documentation of this file.

/********************************************************
 * Myricom VI-GM  networking software and documentation *
 * Copyright (c) 2001 by Myricom, Inc.                  *
 * All rights reserved.                                 *
 * See the file `COPYING' for copyright notice.         *
 ********************************************************/

#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <string.h>
#ifndef WIN32
#include <sys/time.h>
#include <unistd.h>
#endif

#include "vipl_priv.h"

#ifndef WIN32

extern const char *_gm_version;
#endif

VIP_ENTRY_POINT VIP_RETURN
VipOpenNic (const VIP_CHAR * DeviceName, VIP_NIC_HANDLE * NicHandle)
{
  VIP_NIC *vip_nic_ptr = NULL;
  VIP_GM *vip_gm_ptr = NULL;
  VIP_UINT16 gm_board_id, gm_port_id;
  VIP_UINT32 i;
  VIP_RETURN vip_status;
  struct timeval current_time;
  gm_status_t gm_status = GM_FAILURE;
  VIP_DEBUG_LABEL (("VipOpenNic"));

  if (DeviceName == NULL)
    {
      VIP_DEBUG (("DeviceName is null"));
      return VIP_INVALID_PARAMETER;
    }

  if (NicHandle == NULL)
    {
      VIP_DEBUG (("NicHandle is null"));
      return VIP_INVALID_PARAMETER;
    }
  
#ifndef WIN32
  /* check GM version */
  if (strncmp (_gm_version, "1.5", 3) > 0)
    {
      goto gm_version_ok;
    }
  else
    {
      if (strncmp (_gm_version, "1.5", 3) < 0)
        {
          fprintf(stderr, "VI-GM: Invalid GM version (minimum 1.5.1, "
                  "current is %s\n", _gm_version);
          return VIP_ERROR_RESOURCE;
        }
      else
        {
          if (_gm_version[3] != '.')
            {
              fprintf(stderr, "VI-GM: Invalid GM version (minimum 1.5.1, "
                      "current is %s\n", _gm_version);
              return VIP_ERROR_RESOURCE;
            }
          else
            {
              goto gm_version_ok;
            }
        }
    }
#endif
  
 gm_version_ok:

  /* check device name */
  if (sscanf (DeviceName, "VINIC%hd", &gm_board_id) != 1)
    {
      if (strcmp ("VINIC", DeviceName) == 0)
        {
          gm_board_id = 0;
        }
      else
        {
          VIP_DEBUG (("Invalid DeviceName (%s)", DeviceName));
          return VIP_INVALID_PARAMETER;
        }
    }

  /* check NIC number limit */
  if (gm_board_id >= VI_GM_MAX_NICS)
    {
      VIP_DEBUG (("Invalid NIC %d (only %d NICs supported)",
                  gm_board_id, VI_GM_MAX_NICS));
      return VIP_INVALID_PARAMETER;
    }

  VIP_MUTEX_LOCK (&vip_glob_lock);

  /* look for previously initialized NIC */
  if (vip_glob_gm_ptrs[gm_board_id] == NULL)
    {
      VIP_DEBUG (("No previously initialized GM port"));

      /* alloc the VIP_GM struct */
      vip_gm_ptr = (VIP_GM *) calloc (1, sizeof (VIP_GM));
      if (vip_gm_ptr == NULL)
        {
          VIP_DEBUG (("VIP_GM struct alloc failed"));
          goto out_with_vip_gm_lock;
        }
      vip_glob_gm_ptrs[gm_board_id] = vip_gm_ptr;

      /* open the GM port */
      gm_port_id = 2;
      while (gm_port_id <= VI_GM_MAX_PORTS)
        {
          if (gm_port_id == VI_GM_PORT_CONN_MANAGER)
            {
              gm_port_id++;
              continue;
            }

          gm_status = gm_open (&(vip_gm_ptr->gm_port),
                               gm_board_id, gm_port_id,
                               "VI-GM over GM", GM_API_VERSION_1_6);
          if (gm_status == GM_SUCCESS)
            {
              break;
            }
          gm_port_id++;
        }

      if (gm_status != GM_SUCCESS)
        {
          VIP_DEBUG (("Unable to open any GM ports on the Myrinet board %d",
                      gm_board_id));
          goto out_with_gm_ptr;
        }

      gm_set_enable_nack_down (vip_gm_ptr->gm_port, 1);

      /* allow remote memory access */
      gm_status = gm_allow_remote_memory_access (vip_gm_ptr->gm_port);
      if (gm_status != GM_SUCCESS)
        {
          VIP_DEBUG_GM (("gm_allow_remote_memory_access() failed"));
          goto out_with_gm_port;
        }

      /* get the gm id */
      gm_status = gm_get_node_id (vip_gm_ptr->gm_port, &(vip_gm_ptr->gm_id));
      if (gm_status != GM_SUCCESS)
        {
          VIP_DEBUG_GM (("Unable to get the GM id of the GM port %d "
                         "of the board %d", gm_port_id, gm_board_id));
          goto out_with_gm_port;
        }

      /* get the mac address */
      gm_status = gm_get_unique_board_id (vip_gm_ptr->gm_port,
                                          vip_gm_ptr->mac_address);
      if (gm_status != GM_SUCCESS)
        {
          VIP_DEBUG_GM (("Unable to get the MAC address of the "
                         "Myrinet board %d", gm_board_id));
          goto out_with_gm_port;
        }

      /* get the connection manager gm id */
      if (gm_board_id == 0)
        {
          vip_gm_ptr->conn_mgr_gm_id = vip_gm_ptr->gm_id;
        }
      else
        {
          VIP_UCHAR name[GM_MAX_HOST_NAME_LEN];

          gm_status = gm_get_host_name (vip_gm_ptr->gm_port, name);
          if (gm_status != GM_SUCCESS)
            {
              VIP_DEBUG_GM (("Unable to get the GM host name of "
                             "the Myrinet board %d", gm_board_id));
              goto out_with_gm_port;
            }

          *strchr (name, ':') = '\0';
          vip_gm_ptr->conn_mgr_gm_id =
            gm_host_name_to_node_id (vip_gm_ptr->gm_port, name);
          if (vip_gm_ptr->conn_mgr_gm_id == GM_NO_SUCH_NODE_ID)
            {
              VIP_DEBUG (("Unable to get the GM id of the Myrinet board 0"));
              goto out_with_gm_port;
            }
        }

      /* init recv buffers */
      vip_gm_ptr->max_recv_tokens =
        gm_num_receive_tokens (vip_gm_ptr->gm_port);
      vip_gm_ptr->recv_tokens = vip_gm_ptr->max_recv_tokens;
      vip_gm_ptr->recv_buffers =
        (VIP_PVOID **) calloc (vip_gm_ptr->max_recv_tokens,
                               sizeof (VIP_PVOID *));
      if (vip_gm_ptr->recv_buffers == NULL)
        {
          VIP_DEBUG (("Error allocating GM recv buffers array"));
          goto out_with_gm_port;
        }

      for (i = 0; i < vip_gm_ptr->max_recv_tokens; i++)
        {
          vip_gm_ptr->recv_buffers[i] =
            (VIP_PVOID *) gm_dma_malloc (vip_gm_ptr->gm_port,
                                         VI_GM_EAGER_LENGTH);
          if (vip_gm_ptr->recv_buffers[i] == NULL)
            {
              VIP_DEBUG (("Error allocating GM receive buffer"));
              goto out_with_recv_bufs;
            }
          gm_provide_receive_buffer_with_tag (vip_gm_ptr->gm_port,
                                              vip_gm_ptr->recv_buffers[i],
                                              VI_GM_CTRL_GM_SIZE,
                                              GM_LOW_PRIORITY, i + 1);
          vip_gm_ptr->recv_tokens--;
        }

      /* init send buffers */
      vip_gm_ptr->max_send_tokens = gm_num_send_tokens (vip_gm_ptr->gm_port);
      vip_gm_ptr->max_send_tokens--;    /* for the heartbeat message */
      vip_gm_ptr->send_tokens = vip_gm_ptr->max_send_tokens;
      vip_gm_ptr->send_buffers_free = NULL;

      vip_gm_ptr->vip_send_req_lookaside
        = gm_create_lookaside (sizeof (VIP_SEND_REQ), VI_GM_NUMBER_SEND_BUF);

      if (vip_gm_ptr->vip_send_req_lookaside == NULL)
        {
          VIP_DEBUG (("Error allocating send buffer lookaside"));
          goto out_with_recv_bufs;
        }

      vip_gm_ptr->send_buffers =
        (VIP_PVOID **) calloc (VI_GM_NUMBER_SEND_BUF, sizeof (VIP_PVOID *));
      if (vip_gm_ptr->send_buffers == NULL)
        {
          VIP_DEBUG (("Error allocating GM send buffers array"));
          goto out_with_send_lookaside;
        }

      for (i = 0; i < VI_GM_NUMBER_SEND_BUF; i++)
        {
          vip_gm_ptr->send_buffers[i] =
            (VIP_PVOID *) gm_dma_malloc (vip_gm_ptr->gm_port,
                                         VI_GM_EAGER_LENGTH);
          if (vip_gm_ptr->send_buffers[i] == NULL)
            {
              VIP_DEBUG (("Error allocating GM receive buffer"));
              goto out_with_send_bufs;
            }

          if (i == 0)
            {
              *(vip_gm_ptr->send_buffers[0]) = NULL;
            }
          else
            {
              *(vip_gm_ptr->send_buffers[i]) =
                vip_gm_ptr->send_buffers[i - 1];
            }
        }
      vip_gm_ptr->send_buffers_free = vip_gm_ptr->send_buffers[i - 1];

      /* set an alarm to keep the conn mgr registration alive */
      vip_gm_ptr->alive_buffer = (VIP_PACKET_ALIVE *)
        gm_dma_malloc (vip_gm_ptr->gm_port, sizeof (VIP_PACKET_ALIVE));
      if (vip_gm_ptr->alive_buffer == NULL)
        {
          VIP_DEBUG (("Error allocating alive buffer"));
          goto out_with_send_bufs;
        }
      vip_gm_ptr->alive_buffer->net_srv_magic = VIP_HTON_UINT32 (0);
      gm_initialize_alarm (&(vip_gm_ptr->alive_alarm));
      
      /* initialize the regmem avl tree stuff */
      VIP_AVL_TREE_INIT (vip_gm_ptr->regmem_avl_tree);
      vip_gm_ptr->regmem_avl_lookaside =
        gm_create_lookaside (sizeof (VIP_AVL_RANGE), VI_GM_INITIAL_AVL);
      if (vip_gm_ptr->regmem_avl_lookaside == NULL)
        {
          VIP_DEBUG (("Error allocating regmem AVL lookaside list"));
          goto out_with_alive_alarm;
        }

      /* initialize the notify looaside */
      vip_gm_ptr->notify_lookaside =
        gm_create_lookaside (sizeof (VIP_NOTIFY), VI_GM_INITIAL_NOTIFY);
      if (vip_gm_ptr->notify_lookaside == NULL)
        {
          VIP_DEBUG (("Error allocating notify handler lookaside list"));
          goto out_with_avl_tree;;
        }

      /* delayed send queue init */
      vip_gm_ptr->send_fifo_head = NULL;
      vip_gm_ptr->send_fifo_tail = NULL;
      vip_gm_ptr->send_fifo_queued = 0;

      /* update misc */
      vip_gm_ptr->memory_registered = 0;
      vip_gm_ptr->send_buf_free_cnt = VI_GM_NUMBER_SEND_BUF;
      vip_gm_ptr->ref_count = 0;
      vip_gm_ptr->gm_board_id = gm_board_id;
      vip_gm_ptr->gm_port_id = gm_port_id;
      vip_gettimeofday (&current_time, NULL);
      gm_srand ((VIP_UINT32) (current_time.tv_sec * current_time.tv_usec));
      vip_gm_ptr->magic = gm_rand () * gm_rand ();
      vip_gm_ptr->head_waiting_conn_ptr = NULL;
      vip_gm_ptr->tail_waiting_conn_ptr = NULL;
      vip_gm_ptr->periodic_plumber_cancelled = VIP_FALSE;
      vip_gm_ptr->blocking_plumber_cancelled = VIP_FALSE;
      gm_bzero (vip_gm_ptr->vip_nic_ptrs,
                (VI_GM_MAX_NIC_HANDLES + 1) * sizeof (VIP_NIC *));

      /* init mutex locks */
      VIP_MUTEX_INIT (&(vip_gm_ptr->lock));
      VIP_MUTEX_INIT (&(vip_gm_ptr->recv_lock));
      VIP_MUTEX_LOCK (&(vip_gm_ptr->lock));
      
      /* init blocking tokens */
      VIP_SEMAPHORE_INIT (&(vip_gm_ptr->blocking_plumber_tokens), 0);

      /* periodic plumber thread creation */
      if (VIP_THREAD_CREATE (&(vip_gm_ptr->plumber_periodic),
                             vip_plumber_periodic, vip_gm_ptr) != 0)
        {
          VIP_DEBUG (("Unable to spawn plumber periodic thread"));
          goto out_with_notify_lookaside;
        }

      /* blocking plumber thread creation */
      if (VIP_THREAD_CREATE (&(vip_gm_ptr->plumber_blocking),
                             vip_plumber_blocking, vip_gm_ptr) != 0)
        {
          VIP_DEBUG (("Unable to spawn plumber blocking thread"));
          goto out_with_periodic_plumber;
        }
    }
  else
    {
      vip_gm_ptr = vip_glob_gm_ptrs[gm_board_id];
      VIP_MUTEX_LOCK (&(vip_gm_ptr->lock));
    }

  /* check the NIC handle limit */
  if (vip_gm_ptr->ref_count >= VI_GM_MAX_NIC_HANDLES)
    {
      VIP_ASSERT (vip_gm_ptr->ref_count == VI_GM_MAX_NIC_HANDLES);
      VIP_DEBUG (("NIC handles limit reached"));
      goto out_with_blocking_plumber;
    }

  /* initialize the NIC lookaside, if needed */
  if (vip_glob_nic_lookaside == NULL)
    {
      vip_glob_nic_lookaside = gm_create_lookaside (sizeof (VIP_NIC), 1);
      if (vip_glob_nic_lookaside == NULL)
        {
          VIP_DEBUG (("Unable to initialize NIC lookaside list"));
          goto out_with_blocking_plumber;
        }
    }

  /* allocate a VIP_NIC */
  vip_nic_ptr = (VIP_NIC *) gm_lookaside_zalloc (vip_glob_nic_lookaside);
  if (vip_nic_ptr == NULL)
    {
      VIP_DEBUG (("VIP_NIC lookaside alloc failed"));
      goto out_with_blocking_plumber;
    }

  /* init (allocate) the handle sets */
  vip_status = vip_init_handle_set (&(vip_nic_ptr->vip_vi_set),
                                    sizeof (VIP_VI),
                                    VI_GM_MAX_VI, VIP_VI_MAGIC);
  if (vip_status != VIP_SUCCESS)
    {
      VIP_DEBUG (("VI handle set init failed"));
      goto out_with_nic_ptr;
    }
  vip_nic_ptr->vip_vi_set.vip_nic_ptr = vip_nic_ptr;

  vip_status = vip_init_handle_set (&(vip_nic_ptr->vip_mem_set),
                                    sizeof (VIP_MEM),
                                    VI_GM_MAX_REGISTER_REGIONS,
                                    VIP_MEM_MAGIC);
  if (vip_status != VIP_SUCCESS)
    {
      VIP_DEBUG (("REGMEM handle set init failed"));
      goto out_with_vi_set;
    }
  vip_nic_ptr->vip_mem_set.vip_nic_ptr = vip_nic_ptr;

  vip_status = vip_init_handle_set (&(vip_nic_ptr->vip_ptag_set),
                                    sizeof (VIP_PTAG),
                                    VI_GM_MAX_PTAGS, VIP_PTAG_MAGIC);
  if (vip_status != VIP_SUCCESS)
    {
      VIP_DEBUG (("PTAG handle set init failed"));
      goto out_with_mem_set;
    }
  vip_nic_ptr->vip_ptag_set.vip_nic_ptr = vip_nic_ptr;

  vip_status = vip_init_handle_set (&(vip_nic_ptr->vip_cq_set),
                                    sizeof (VIP_CQ),
                                    VI_GM_MAX_CQ, VIP_CQ_MAGIC);
  if (vip_status != VIP_SUCCESS)
    {
      VIP_DEBUG (("CQ handle set init failed"));
      goto out_with_ptag_set;
    }
  vip_nic_ptr->vip_cq_set.vip_nic_ptr = vip_nic_ptr;

  vip_status = vip_init_handle_set (&(vip_nic_ptr->vip_conn_set),
                                    sizeof (VIP_CONN),
                                    VI_GM_MAX_CONN, VIP_CONN_MAGIC);
  if (vip_status != VIP_SUCCESS)
    {
      VIP_DEBUG (("CONN handle set init failed"));
      goto out_with_cq_set;
    }
  vip_nic_ptr->vip_conn_set.vip_nic_ptr = vip_nic_ptr;

  /* initialize the MEM handle index */
  vip_nic_ptr->vip_mem_handle_free = 1;
  vip_nic_ptr->vip_mem_handle_ptrs[0] = NULL;
  for (i = 1; i <= VI_GM_MAX_REGISTER_REGIONS; i++)
    {
      vip_nic_ptr->vip_mem_handle_ptrs[i] = 
        (VIP_MEM *) ((VIP_UINTPTR) (i + 1));
    }
  vip_nic_ptr->vip_mem_handle_ptrs[VI_GM_MAX_REGISTER_REGIONS] = NULL;

  /* initialize the VI handle index */
  vip_nic_ptr->vip_vi_index_free = 1;
  vip_nic_ptr->vip_vi_ptrs[0] = NULL;
  for (i = 1; i <= VI_GM_MAX_VI; i++)
    {
      vip_nic_ptr->vip_vi_ptrs[i] = (VIP_VI *) ((VIP_UINTPTR) (i + 1));
    }
  vip_nic_ptr->vip_vi_ptrs[VI_GM_MAX_VI] = NULL;

  /* validate the NIC handle */
  vip_nic_ptr->magic = VIP_NIC_MAGIC;
  vip_nic_ptr->ptag_last_id = 1;
  vip_nic_ptr->error_callback = vip_default_error_callback;
  vip_nic_ptr->error_callback_context = NULL;
  VIP_MUTEX_INIT (&(vip_nic_ptr->lock));
  vip_nic_ptr->ns_initialized = VIP_FALSE;

  /* link with the GM port */
  vip_nic_ptr->vip_gm_ptr = vip_gm_ptr;
  vip_gm_ptr->ref_count++;
  for (i = 1; i <= VI_GM_MAX_NIC_HANDLES; i++)
    {
      if (vip_gm_ptr->vip_nic_ptrs[i] == NULL)
        {
          vip_gm_ptr->vip_nic_ptrs[i] = vip_nic_ptr;
          vip_nic_ptr->index = i;

          *NicHandle = (VIP_NIC_HANDLE) vip_nic_ptr;
          if (vip_gm_ptr->ref_count == 1)
            {
              /* notify the new endpoint to the conn mgr */
              vip_alive_alarm_callback (vip_gm_ptr);
            }

          VIP_MUTEX_UNLOCK (&(vip_gm_ptr->lock));
          VIP_MUTEX_UNLOCK (&vip_glob_lock);
          VIP_DEBUG (("Success"));
          return VIP_SUCCESS;
        }
    }

out_with_cq_set:
  vip_destroy_cq_handle_set (&(vip_nic_ptr->vip_cq_set));
out_with_ptag_set:
  vip_destroy_ptag_handle_set (&(vip_nic_ptr->vip_ptag_set));
out_with_mem_set:
  vip_destroy_mem_handle_set (&(vip_nic_ptr->vip_mem_set));
out_with_vi_set:
  vip_destroy_vi_handle_set (&(vip_nic_ptr->vip_vi_set));
out_with_nic_ptr:
  gm_lookaside_free (vip_nic_ptr);

out_with_blocking_plumber:
  if (vip_gm_ptr->ref_count == 0)
    {
      vip_gm_ptr->blocking_plumber_cancelled = VIP_TRUE;
      VIP_SEMAPHORE_POST (&(vip_gm_ptr->blocking_plumber_tokens));
      VIP_THREAD_JOIN (vip_gm_ptr->plumber_blocking);
    }
  else
    {
      /* no GM structure to clean */
      VIP_MUTEX_UNLOCK (&(vip_gm_ptr->lock));
      VIP_MUTEX_UNLOCK (&vip_glob_lock);
      return VIP_ERROR_RESOURCE;
    }

out_with_periodic_plumber:
  vip_gm_ptr->periodic_plumber_cancelled = VIP_TRUE;
  VIP_THREAD_JOIN (vip_gm_ptr->plumber_periodic);

out_with_notify_lookaside:
  VIP_SEMAPHORE_DESTROY (&(vip_gm_ptr->blocking_plumber_tokens));
  VIP_MUTEX_UNLOCK (&(vip_gm_ptr->lock));
  VIP_MUTEX_DESTROY (&(vip_gm_ptr->lock));
  gm_destroy_lookaside (vip_gm_ptr->notify_lookaside);

out_with_avl_tree:
  VIP_AVL_TREE_DESTROY (vip_gm_ptr->regmem_avl_tree);
  gm_destroy_lookaside (vip_gm_ptr->regmem_avl_lookaside);

out_with_alive_alarm:
  gm_cancel_alarm (&(vip_gm_ptr->alive_alarm));
  gm_dma_free (vip_gm_ptr->gm_port, vip_gm_ptr->alive_buffer);

out_with_send_bufs:
  for (i = 0; i < VI_GM_NUMBER_SEND_BUF; i++)
    {
      if (vip_gm_ptr->send_buffers[i] != NULL)
        {
          gm_dma_free (vip_gm_ptr->gm_port, vip_gm_ptr->send_buffers[i]);
        }
    }
  free (vip_gm_ptr->send_buffers);

out_with_send_lookaside:
  gm_destroy_lookaside (vip_gm_ptr->vip_send_req_lookaside);

out_with_recv_bufs:
  for (i = 0; i < vip_gm_ptr->max_recv_tokens; i++)
    {
      if (vip_gm_ptr->recv_buffers[i] != NULL)
        {
          gm_dma_free (vip_gm_ptr->gm_port, vip_gm_ptr->recv_buffers[i]);
        }
    }
  free (vip_gm_ptr->recv_buffers);

out_with_gm_port:
  gm_close (vip_gm_ptr->gm_port);
out_with_gm_ptr:
  free (vip_gm_ptr);
  vip_glob_gm_ptrs[gm_board_id] = NULL;
out_with_vip_gm_lock:
  VIP_MUTEX_UNLOCK (&vip_glob_lock);

  return VIP_ERROR_RESOURCE;
}