
#/*##########################################################################
# Copyright (C) 2001-2013 European Synchrotron Radiation Facility
#
#              PyHST2  
#  European Synchrotron Radiation Facility, Grenoble,France
#
# PyHST2 is  developed at
# the ESRF by the Scientific Software  staff.
# Principal author for PyHST2: Alessandro Mirone.
#
# This program is free software; you can redistribute it and/or modify it 
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option) 
# any later version.
#
# PyHST2 is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# PyHST2; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
# Suite 330, Boston, MA 02111-1307, USA.
#
# PyHST2 follows the dual licensing model of Trolltech's Qt and Riverbank's PyQt
# and cannot be used as a free plugin for a non-free program. 
#
# Please contact the ESRF industrial unit (industry@esrf.fr) if this license 
# is a problem for you.
#############################################################################*/
#/*##########################################################################
# Copyright (C) 2004-2010 European Synchrotron Radiation Facility
#
# This file is part of the PyHST  Toolkit developed at
# the ESRF by the SciSoft  group.
#
# This toolkit is free software; you can redistribute it and/or modify it 
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option) 
# any later version.
#
# PyHST is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# PyHST; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
# Suite 330, Boston, MA 02111-1307, USA.
#
# PyHST follows the dual licensing model of Trolltech's Qt and Riverbank's PyQt
# and cannot be used as a free plugin for a non-free program. 
#
# Please contact the ESRF industrial unit (industry@esrf.fr) if this license 
# is a problem for you.
#############################################################################*/



#include <stdio.h>
#include <stdlib.h>
#include<math.h>

#include <sys/types.h>
#include <unistd.h>
#include <sys/stat.h>


// #include <nvcore/Library.h>
#include <cuda.h>
// #include <unistd.h>
#include <fcntl.h>

void get_possible_cards(int **ncard, int *ncardlen, int min_major, int min_minor ) {
  int count1;
  int major, minor;
  CUdevice  dev;
  
  cuInit(0);
  cuDeviceGetCount( &count1 );
  
  *ncard=new int [count1];
  *ncardlen=0;

  for(int i=0; i<count1; i++) {
    cuDeviceGet(&dev, i);
    cuDeviceComputeCapability(&major, &minor,dev);
    if(major>=min_major ||(  major==min_major   && minor>=min_minor ) ) {
      (*ncard)[*ncardlen]=i;
      (*ncardlen)++;
    }
  }
}



int c_cudaSetDevice(int device)
{
  return cudaSetDevice(device);
}


int cudadistribute(int min_major, int min_minor, int option=-1) {
  
  static int gia_inizializzata=0;
  static int cardtouse=-1;
#define MAXLBUFF 200
  struct flock fl;
  int fd;
  FILE *fdstream;
  mode_t oldvalue;
  struct stat st;
  size_t bufferlen;
  char *buffer;
  int nprocess, activenprocess;
  int *proc_id;
  int *card;
  #define NCARDS 1000
  int cardhisto[NCARDS];
  int mincards;
  int *nCard, nCardlen;
  int iaux;
  int i;

  bufferlen=100;
  buffer=(char * ) malloc( bufferlen);

  if(gia_inizializzata==0) {  

    if(option!=-1) {
      cardtouse=option;
      c_cudaSetDevice(cardtouse);
      return cardtouse;
    }
    

    get_possible_cards( &nCard, &nCardlen, min_major , min_minor ) ;
    
    
    if( nCardlen==0) {
      printf(" ERROR, no GPUs found \n");
      exit(1);
    }
    
    fl.l_type   = F_WRLCK;  /* F_RDLCK, F_WRLCK, F_UNLCK    */
    fl.l_whence = SEEK_SET; /* SEEK_SET, SEEK_CUR, SEEK_END */
    fl.l_start  = 0;        /* Offset from l_whence         */
    fl.l_len    = 0;        /* length, 0 = to EOF           */
    fl.l_pid    = getpid(); /* our PID                      */
    
    oldvalue=umask(0x000);
    fd = open("/tmp/GPU_versus_process.txt", O_RDWR | O_CREAT,   S_IRWXU | S_IRWXG | S_IRWXO  );
    
    if(fd!=-1) {
      fcntl(fd, F_SETLKW, &fl);  /* F_GETLK, F_SETLK, F_SETLKW */
      
      

      fdstream=fdopen(fd, "r");

      
      iaux=getline(&buffer,&bufferlen , fdstream );
      
      
      if(iaux>0) {
	iaux=sscanf(buffer,"%d",&nprocess);
	if(iaux!=1) {
	  nprocess=0;
	}
      } else {
	nprocess=0;
      }
    } else {
      printf(" WARNING : could not open : %s in routine cudadistribute \n", "/tmp/GPU_versus_process.txt");
      nprocess=0;
    }
   
    proc_id=(int *) malloc((nprocess+1)*sizeof(int));
    card   =(int *) malloc((nprocess+1)*sizeof(int));
    
    for(i=0; i<nCardlen ; i++) {
      cardhisto[i]=0;
    }

    activenprocess=0;

    for(i=0; i<nprocess; i++) {
      
      getline(&buffer,&bufferlen ,    fdstream );
      

      bufferlen=10000;
      buffer=(char * ) realloc(buffer, bufferlen);

      sscanf(buffer,"%d %d",&(proc_id[i]),&(card[i] ) );
      sprintf(buffer,"/proc/%d", proc_id[i]);

      if(stat(buffer,&st) != 0) {
	card[i]=-1;
      } else {
	cardhisto[card[i]]+=1;
	activenprocess+=1;
      }
    }
    
    mincards=100000;
    for(i=0; i<nCardlen  ; i++) {
      if(cardhisto[i]<mincards) {
	mincards=cardhisto[i];
	cardtouse=i;
      }
    }
    

    
    if( fd!=-1) {
      
      lseek( fd  ,  0 , SEEK_SET);
      sprintf(buffer,"%d\n", activenprocess+1 );
      write(fd, buffer, strlen(buffer));
      
      proc_id[nprocess ]=  getpid(); 
      card[nprocess ]   =  cardtouse ; 
      
      for(i=0; i<nprocess+1; i++) {
	if( card[i]>=0) {
	  sprintf(buffer,"%d %d \n",proc_id[i ],  card[i ]   );
	  write(fd, buffer, strlen(buffer));
	}
      }
      
      fl.l_type   = F_UNLCK;  /* tell it to unlock the region */
      fcntl(fd, F_SETLK, &fl); /* set the region to unlocked */
      
      // close(fd);
      fclose(fdstream);
      umask(oldvalue);
    }
    
    free(proc_id);
    free(card);
    free(buffer);
    gia_inizializzata=1;
    
  }
  c_cudaSetDevice(nCard[cardtouse]);
  return cardtouse;
}
