///////
   //    MySQL Database class for ht://Check
   //    File: HtmysqlDB.cc
   //
   //    Copyright (c) 1999-2004 Comune di Prato - Prato - Italy
   //    Some Portions Copyright (c) 2008 Devise.IT srl <http://www.devise.it/>
   //    Author: Gabriele Bartolini - Prato - Italy <angusgb@users.sourceforge.net>
   //
   //    For copyright details, see the file COPYING in your distribution
   //    or the GNU General Public License version 2 or later 
   //    <http://www.gnu.org/copyleft/gpl.html>
   //
   //    $Id: HtmysqlDB.cc,v 1.91 2009/08/26 12:25:57 angusgb Exp $
   //
   //    Started: 28.06.1999
///////

#ifdef HAVE_CONFIG_H
#include "htconfig.h"
#endif /* HAVE_CONFIG_H */

#ifdef HAVE_STD
#include <iostream>
#include <sstream>
#include <iomanip>
#ifdef HAVE_NAMESPACES
using namespace std;
#endif
#else
#include <stdlib.h>
#include <ctype.h>
#include <iostream.h>
#include <iomanip.h>
#include <sstream.h>
#endif /* HAVE_STD */

#include "HtmysqlDB.h"
#include "StringList.h"
#include "_Url.h"

#ifdef HAVE_LOAD_DEFAULTS
// Local function (declared as static)
static bool getmysqlconfvalue(const char *source, const char *pattern,
   std::string &destination);
#endif

#define URL_INDEX_LENGTH 64

#define ESCAPE_STRING(mysql, start, length) \
	char* to = new char[length*2+1]; \
	mysql_real_escape_string(&mysql, to, start, length); \
	Dest += '\'' + to + '\''; \
	if (to) { \
		delete[] to; \
	}

#define ESCAPE_OSTRING(mysql, start, length) \
	char* to = new char[length*2+1]; \
	mysql_real_escape_string(&mysql, to, start, length); \
	Dest << '\'' << to << "'"; \
	if (to) { \
		delete[] to; \
	}

///////
   //    Construction
///////

#if 0   
HtmysqlDB::HtmysqlDB (const std::string &host, const std::string &db,
         const std::string &user, const std::string &passwd)
{

   MySQLHost = host;
   MySQLDB = db;
   MySQLUser = user;
   MySQLPasswd = passwd;

   // Set the default length for the Index regarding
   // the Url field in the Schedule and Url tables
   URL_Index_Length = URL_INDEX_LENGTH;
   
}
#endif

HtmysqlDB::HtmysqlDB(const std::string &db,
#ifdef HAVE_LOAD_DEFAULTS
    const std::string &File,
#endif
    const std::string& Group,
    const std::string &_ClientCharset, const std::string& _DBCharset,
    int *argc, char ***argv)
:
   MySQLDB(db),
#ifdef HAVE_LOAD_DEFAULTS
   MySQLHost(),
   MySQLUser(),
   MySQLPasswd(),
   MySQLPort(),
   MySQLSocket(),
#endif
   URL_Index_Length(URL_INDEX_LENGTH),
   DBSignature(db),
   ClientCharset(),
   DBCharset(DEFAULT_DB_CHARSET),
   AvailableCharsets()

{
   // Set the default length for the Index regarding
   // the Url field in the Schedule and Url tables
   
#ifdef HAVE_LOAD_DEFAULTS
   LoadDefaults(File, Group.c_str(), argc, argv);
   
   if (MySQLHost.length())
      DBSignature += "@" + MySQLHost;
   if (MySQLPort)
      DBSignature += ":" + MySQLPort;
#else
   if (debug > 0)
      cout << "  Reading MySQL options for group '" << Group << "'" << endl;
   ReadDefaultGroup(Group);
#endif

   if (_DBCharset.size() && _DBCharset != "default") {
       DBCharset = _DBCharset;
   }

   if (_ClientCharset.size() && _ClientCharset != "default") {
       ClientCharset = _ClientCharset;
   }
   
}

///////
   //    Destruction
///////

HtmysqlDB::~HtmysqlDB ()
{
}

#ifdef HAVE_LOAD_DEFAULTS

void HtmysqlDB::LoadDefaults(const std::string &File,
   MYSQL_LOAD_DEFAULTS_ARGTWO Group, int *argc, char ***argv)
{

   MYSQL_LOAD_DEFAULTS_ARGTWO group[] = {0, 0};
   std::string strPort;
   
   group[0] = Group;
   
   // Use the MySQL function for getting the default settings
   // for the connection
   
   if (debug >0)
      cout << " Reading default MySQL option file: " << File
         << " [" << Group << "]" << endl;

   load_defaults(File.c_str(), group, argc, argv);

   for (int i=0; i < *argc; i++)
   {
      if (getmysqlconfvalue((*argv)[i], "user", MySQLUser))
      {
         if (debug > 0)
            cout << "  Found MySQL User: " << MySQLUser << endl;
      }
      else if (getmysqlconfvalue((*argv)[i], "host", MySQLHost))
      {
         if (debug > 0)
            cout << "  Found MySQL Host: " << MySQLHost << endl;
      }
      else if (getmysqlconfvalue((*argv)[i], "password", MySQLPasswd))
      {
         if (debug > 0)
            cout << "  Found MySQL Password: <shhh>" << endl;
      }
      else if (getmysqlconfvalue((*argv)[i], "port", strPort))
      {
         MySQLPort = atoi(strPort.c_str());
         
         if (debug > 0)
            cout << "  Found MySQL Port Number: " << MySQLPort << endl;
      }
      else if (getmysqlconfvalue((*argv)[i], "socket", MySQLSocket))
      {
         if (debug > 0)
            cout << "  Found MySQL Socket: " << MySQLSocket << endl;
      }
   }

}
#endif


#ifdef HAVE_LOAD_DEFAULTS
bool getmysqlconfvalue(const char * source, const char *pattern,
   std::string &destination)
{
   const char *p=source;

   for (; *p && *p=='-'; p++);   // Skip the trailing '-'

   if (!mystrncasecmp(pattern, p, strlen(pattern)))
   {
      destination.clear();

         p+=strlen(pattern)+1;   // Go over the '='

      for (; *p && isspace(*p); p++);
      
      destination = p;
      
      return true;
   }
   
   return false;
}
#endif


///////
   //    Connection to the host specified with user and password.
   //    Here we don't connect to a precise database ... We will do it
   //    later using SelectDB method.
///////

int HtmysqlDB::Connect ()
{

#ifdef HAVE_LOAD_DEFAULTS

   const char *host;
   const char *user;
   const char *passwd;
   const char *socket;

///////
   //    Converting empty strings into NULL pointer
   //    This allows MySQL server to act as default
///////

   if (! MySQLHost.length()) host=NULL;
   else
       host = MySQLHost.c_str();

   if (! MySQLUser.length())
       user=0;
   else
       user = MySQLUser.c_str();

   if (! MySQLPasswd.length())
       passwd=0;
   else
       passwd = MySQLPasswd.c_str();

   if (! MySQLSocket.length())
       socket=0;
   else
       socket = MySQLSocket.c_str();

   if (debug > 0)
      cout << "Connecting to MySQL server on " << (host?host:"localhost")
         << " as " << (user?user:"session") << " user" << endl;

///////
   //    Connecting using Htmysql::Connect Interface method
///////

   if (! Htmysql::Connect (host, user, passwd, 0, MySQLPort, socket))
      return 0;   // Something has gone wrong ... Let's check it outside here
#else

///////
   //    Connecting using Htmysql::Connect Interface method (using default attributes)
///////

   if (! Htmysql::Connect())
      return 0;   // Something has gone wrong ... Let's check it outside here

#endif

   // Set the client encoding
  if (ClientCharset.length() > 0) {
      std::string SQLStatement = "SET NAMES " + ClientCharset;

      if (Query (SQLStatement) == -1) {
	  cout << "Check the value of the 'mysql_client_charset' configuration option: "
	      << ClientCharset << endl;
         return 0;   // An error occured
      }
  }
   
   return 1;   
}


///////
   //    Creating a new database for ht://Check
   //    Return 0 if an error occured, 1 OK
///////

int HtmysqlDB::CreateDatabase()
{

   std::ostringstream SQLStatement;
   int result;

   // This variable holds the length of the URL Index
   // in the URL and Schedule tables   
   std::ostringstream SQL_Url_Index;
   
   SQL_Url_Index << "INDEX Idx_Url (Url";
   
   if (URL_Index_Length > 0)
   {
      // Let's set the length
      SQL_Url_Index << '(' << URL_Index_Length << ')';
   }

   SQL_Url_Index << ')';
   
   
///////
   //    Check if the db already exists
///////

   result = Exists (MySQLDB);
   
   if (result == -1) return 0;   // An error occured
   
   // Database already exists
   // Let's drop it before creating a new one with the same name
   if (result > 0)
   {
      if (drop_database)
         DropDatabase(); // Drop the database
      else
      {
         // Keep the existant database, but drop the tables
         DropTables();

         if (debug >0)
            cout << "Keep alive previous ht://Check database '" << MySQLDB << "'" << endl;
   
      }
   }
   else
      drop_database = true; // Database has been virtually 'dropped' (did not exist)


   // If a database was not found or it has already dropped, we have to create it
   if (drop_database)
   {
      ///////
         //    Database creation
      ///////
   
      if (debug >0)
         cout << "Creating ht://Check database '" << MySQLDB << "'" << endl;
  
      SQLStatement << "CREATE DATABASE " << MySQLDB;

      if (DBCharset.length() > 0)
          SQLStatement << " CHARACTER SET " << DBCharset;
   
      // Executing Database creation 

      if (Query (SQLStatement.str()) == -1)
         return 0;   // An error occured
   }
   
   // Select the database
   SelectDB (MySQLDB);

   if (debug >0)
      cout << "Database '" << MySQLDB << "' now selected" << endl;
   
///////
   //    Creation of the 'Schedule' table
///////

   if (debug >1)
      cout << " |- Creating 'htCheck' table" << endl;


   // Write the SQL statement
   
   SQLStatement.str(""); // Initialize it again.
   SQLStatement << "CREATE TABLE " << MySQLDB << ".htCheck (" << "\n"
      << "  Version VARCHAR(32) DEFAULT '" << VERSION << "' NOT NULL, " << "\n"
      << "  StartTime DATETIME DEFAULT '0000-00-00 00:00:00' NOT NULL, " << "\n"
      << "  EndTime DATETIME DEFAULT '0000-00-00 00:00:00' NOT NULL, " << "\n"
      << "  ScheduledUrls MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL, " << "\n"
      << "  TotUrls MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL, " << "\n"
      << "  RetrievedUrls MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL, " << "\n"
      << "  TCPConnections MEDIUMINT UNSIGNED  DEFAULT '0' NOT NULL, " << "\n"
      << "  ServerChanges MEDIUMINT UNSIGNED  DEFAULT '0' NOT NULL, " << "\n"
      << "  HTTPRequests MEDIUMINT UNSIGNED  DEFAULT '0' NOT NULL, " << "\n"
      << "  HTTPSeconds MEDIUMINT UNSIGNED  DEFAULT '0' NOT NULL, " << "\n"
      << "  HTTPBytes BIGINT UNSIGNED  DEFAULT '0' NOT NULL, " << "\n"
      << "  AccessibilityChecks TINYINT UNSIGNED  DEFAULT '1' NOT NULL, " << "\n"
      << "  HtDigNotification TINYINT UNSIGNED  DEFAULT '0' NOT NULL, " << "\n"
      << "  User VARCHAR(255) DEFAULT '' NOT NULL," << "\n"
      << "  PRIMARY KEY (StartTime, EndTime)" << "\n"
      << ")" << "\n";

   // Executing Table creation 

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured


///////
   //    Creation of the 'Schedule' table
///////

   if (debug >1)
      cout << " |- Creating 'Schedule' table" << endl;


   // Write the SQL statement
   
   SQLStatement.str(""); // Initialize it again.
   SQLStatement << "CREATE TABLE " << MySQLDB << ".Schedule (" << "\n"
      << "  IDUrl MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL, " << "\n"
      << "  IDServer SMALLINT UNSIGNED DEFAULT '0' NOT NULL, " << "\n"
      << "  Url VARCHAR(" << URL_DB_SIZE << ") BINARY DEFAULT '' NOT NULL, " << "\n"
      << "  Status ENUM('ToBeRetrieved', 'Retrieved', 'CheckIfExists',"
      << " 'Checked', 'BadQuerystd::string', 'BadExtension', 'MaxHopCount',"
      << " 'FileProtocol', 'EMail', 'Javascript', 'NotValidService',"
      << " 'Malformed', 'MaxUrlsCount') "
             << "DEFAULT 'ToBeRetrieved' NOT NULL, " << "\n"
      << "  Domain ENUM('Internal', 'External') DEFAULT NULL," << "\n"
      << "  CreationTime DATETIME DEFAULT '0000-00-00 00:00:00' NOT NULL, " << "\n"
      << "  IDReferer MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL, " << "\n"
      << "  HopCount TINYINT UNSIGNED DEFAULT '0' NOT NULL, " << "\n"
      << "  PRIMARY KEY (IDUrl), " << "\n"
      << "  INDEX Idx_IDServer (IDServer), " << "\n"
      << "  " << SQL_Url_Index.str() << ", " << "\n"
      << "  INDEX Idx_Status (Status) " << "\n"
      << ")" << "\n";

   // Executing Table creation 

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured


///////
   //    Creation of the 'Server' table
///////

   if (debug >1)
      cout << " |- Creating 'Server' table" << endl;


   // Write the SQL statement
   
   SQLStatement.str(""); // Initialize it again.
   SQLStatement << "CREATE TABLE " << MySQLDB << ".Server (" << "\n"
      << "  IDServer SMALLINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  Server VARCHAR(255) DEFAULT '' NOT NULL," << "\n"
      << "  IPAddress VARCHAR(15)," << "\n"
      << "  Port SMALLINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  HttpServer VARCHAR(255) DEFAULT '' NOT NULL," << "\n"
      << "  HttpVersion VARCHAR(255) DEFAULT '' NOT NULL," << "\n"
      << "  PersistentConnection TINYINT(1) UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  Requests SMALLINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  PRIMARY KEY (IDServer), " << "\n"
      << "  INDEX Idx_Server (Server(24)), " << "\n"
      << "  INDEX Idx_Requests (Requests) " << "\n"
      << ")" << "\n";

   // Executing table creation 

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured


///////
   //    Creation of the 'Url' table
///////

   if (debug >1)
      cout << " |- Creating 'Url' table" << endl;


   // Write the SQL statement

   // First builds the Charset field as an enumeration of possible values
   std::string tmpcharsets;
   for (CharsetsMap::const_iterator cs(AvailableCharsets.begin());
	cs != AvailableCharsets.end(); ++cs) {
      tmpcharsets += '\'' + (*cs) + '\'' + ',';
   }
   
   SQLStatement.str(""); // Initialize it again.
   SQLStatement << "CREATE TABLE " << MySQLDB << ".Url (" << "\n"
      << "  IDUrl MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  IDServer SMALLINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  Url VARCHAR(" << URL_DB_SIZE << ") BINARY DEFAULT '' NOT NULL," << "\n"
      << "  HTTPContentType VARCHAR(32) DEFAULT '' NOT NULL," << "\n"
      << "  ContentType VARCHAR(32) DEFAULT '' NOT NULL," << "\n"
      << "  ConnStatus ENUM('OK', 'NoHeader', 'NoHost', 'NoPort', "
         << "'NoConnection', 'ConnectionDown', 'ServiceNotValid', "
         << "'OtherError', 'ServerError') "
         << "DEFAULT 'OK' NOT NULL, " << "\n"
      << "  ContentLanguage VARCHAR(16) DEFAULT '' NOT NULL," << "\n"
      << "  TransferEncoding VARCHAR(32) DEFAULT '' NOT NULL," << "\n"
      << "  LastModified DATETIME DEFAULT '0000-00-00 00:00:00' NOT NULL," << "\n"
      << "  LastAccess DATETIME DEFAULT '0000-00-00 00:00:00' NOT NULL," << "\n"
      << "  Size INT DEFAULT '0' NOT NULL," << "\n"
      << "  StatusCode SMALLINT DEFAULT '0' NOT NULL," << "\n"
      << "  ReasonPhrase VARCHAR(32) DEFAULT '' NOT NULL," << "\n"
      << "  Location VARCHAR(" << URL_DB_SIZE << ") BINARY DEFAULT '' NOT NULL," << "\n"
      << "  Title VARCHAR(255) DEFAULT '' NOT NULL," << "\n"
      << "  Contents MEDIUMTEXT DEFAULT NULL," << "\n"
      << "  DocType ENUM('not-public', 'not-html', " << "\n"
	  	<< " 'xhtml-11', 'xhtml-10', 'xhtml-10-transitional', 'xhtml-10-frameset'," << "\n"
		<< " 'html-401', 'html-401-transitional', 'html-401-frameset'," << "\n"
		<< " 'html-40', 'html-40-transitional', 'html-40-frameset'," << "\n"
		<< " 'html-32', 'html-20', 'html-20-level2', 'html-20-level1'," << "\n"
		<< " 'html-20-strict', 'html-20-strict-level1', 'html-iso-iec-15445-2000'," << "\n"
		<< " 'unknown') DEFAULT NULL," << "\n"
      << "  HTTPCharset ENUM(" << tmpcharsets << "'unknown') DEFAULT NULL," << "\n"
      << "  Charset ENUM(" << tmpcharsets << "'unknown') DEFAULT NULL," << "\n"
      << "  Description VARCHAR(255) DEFAULT NULL," << "\n"
      << "  Keywords VARCHAR(255) DEFAULT NULL," << "\n"
      << "  HtDigEmail VARCHAR(255) DEFAULT NULL," << "\n"
      << "  HtDigEmailSubject VARCHAR(255) DEFAULT NULL," << "\n"
      << "  HtDigNotificationDate DATE DEFAULT NULL," << "\n"
      << "  SizeAdd INT DEFAULT '0' NOT NULL," << "\n"
      // << "  OutgoingLinks SMALLINT DEFAULT '0' NOT NULL," << "\n"
      // << "  IncomingLinks SMALLINT DEFAULT '0' NOT NULL," << "\n"
      // << "  IncomingUrls SMALLINT DEFAULT '0' NOT NULL," << "\n"
      // << "  OutgoingDocuments SMALLINT DEFAULT '0' NOT NULL," << "\n"
      << "  PRIMARY KEY (IDUrl), " << "\n"
      << "  INDEX Idx_IDServer (IDServer), " << "\n"
      << "  " << SQL_Url_Index.str() << ", " << "\n"
      << "  INDEX Idx_ContentType (ContentType(16)), " << "\n"
      << "  INDEX Idx_StatusCode (StatusCode), " << "\n"
      << "  INDEX Idx_HTTPCharset (HTTPCharset)," << "\n"
      << "  INDEX Idx_Charset (Charset)," << "\n"
      << "  INDEX Idx_HtDigNotificationDate (HtDigNotificationDate)" << "\n"
      << ")" ;

   // Executing table creation 

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured


///////
   //    Creation of the 'HtmlStatement' table
///////

   if (debug >1)
      cout << " |- Creating 'HtmlStatement' table" << endl;


   // Write the SQL statement
   
   SQLStatement.str(""); // Initialize it again.
   SQLStatement << "CREATE TABLE " << MySQLDB << ".HtmlStatement (" << "\n"
      << "  IDUrl MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  TagPosition SMALLINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  Row MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  Col MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  Tag VARCHAR(32) DEFAULT '' NOT NULL," << "\n"
      << "  Statement VARCHAR(255)," << "\n"
      << "  LinkTagPosition SMALLINT UNSIGNED," << "\n"
      << "  LinkDescription VARCHAR(255)," << "\n"
      << "  PRIMARY KEY (IDUrl, TagPosition)," << "\n"
      << "  INDEX Idx_Tag (Tag(4))," << "\n"
      << "  INDEX Idx_Statement (Tag(8))" << "\n"
      << ")" << "\n";

   // Executing table creation 

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured


///////
   //    Creation of the 'HtmlAttribute' table
///////

   if (debug >1)
      cout << " |- Creating 'HtmlAttribute' table" << endl;


   // Write the SQL statement
   
   SQLStatement.str(""); // Initialize it again.
   SQLStatement << "CREATE TABLE " << MySQLDB << ".HtmlAttribute (" << "\n"
      << "  IDUrl  MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  TagPosition SMALLINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  AttrPosition TINYINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  Attribute VARCHAR(32) DEFAULT '' NOT NULL," << "\n"
      << "  Content VARCHAR(255) DEFAULT '' NOT NULL," << "\n"
      << "  PRIMARY KEY (IDUrl, TagPosition, AttrPosition)," << "\n"
      << "  INDEX Idx_Attribute (Attribute(8))," << "\n"
      << "  INDEX Idx_Content (Content(8))" << "\n"
      << ")" << "\n";

   // Executing table creation 

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured


///////
   //    Creation of the 'Link' table
///////

   if (debug >1)
      cout << " |- Creating 'Link' table" << endl;


   // Write the SQL statement
   
   SQLStatement.str(""); // Initialize it again.
   SQLStatement << "CREATE TABLE " << MySQLDB << ".Link (" << "\n"
      << "  IDUrlSrc MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  IDUrlDest MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  TagPosition SMALLINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  AttrPosition TINYINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  Anchor VARCHAR(255) BINARY DEFAULT '' NOT NULL," << "\n"
      << "  LinkType ENUM('Normal', 'Direct', 'Redirection') "
         << "DEFAULT 'Normal' NOT NULL, " << "\n"
      << "  LinkResult "
         << "ENUM('NotChecked', 'NotRetrieved', 'OK', 'Broken', "
         << "'AnchorNotFound', 'Redirected', 'NotAuthorized', "
	 << "'EMail', 'Javascript', 'BadEncoded') "
         << "DEFAULT 'NotChecked' NOT NULL, " << "\n"
      << "  LinkDomain "
         << "ENUM('SameServer', 'Internal', 'External') "
         << "DEFAULT NULL, " << "\n"
      << "  PRIMARY KEY (IDUrlSrc, IDUrlDest, TagPosition, AttrPosition)" << "\n"
      << ")" << "\n";


   // Executing table creation 

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

///////
   //    Creation of the 'Cookies' table
///////

   if (debug >1)
      cout << " |- Creating 'Cookies' table" << endl;


   // Write the SQL statement
   
   SQLStatement.str(""); // Initialize it again.
   SQLStatement << "CREATE TABLE " << MySQLDB << ".Cookies (" << "\n"
      << "  IDCookie MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  Name VARCHAR(255) DEFAULT '' NOT NULL," << "\n"
      << "  Value TEXT DEFAULT '' NOT NULL," << "\n"
      << "  Path VARCHAR(255) DEFAULT '' NOT NULL," << "\n"
      << "  Domain VARCHAR(255) DEFAULT '' NOT NULL," << "\n"
      << "  MaxAge MEDIUMINT DEFAULT '-1' NOT NULL," << "\n"
      << "  Version TINYINT DEFAULT '0' NOT NULL," << "\n"
      << "  SrcUrl VARCHAR(" << URL_DB_SIZE << ") DEFAULT '' NOT NULL," << "\n"
      << "  Expires DATETIME DEFAULT '0000-00-00 00:00:00' NOT NULL, " << "\n"
      << "  Secure TINYINT DEFAULT '0' NOT NULL, " << "\n"
      << "  DomainValid TINYINT DEFAULT '0' NOT NULL, " << "\n"
      << "  PRIMARY KEY (IDCookie)" << "\n"
      << ")" << "\n";


   // Executing table creation 

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

///////
   //    Creation of the 'Accessibility' table
///////

   if (debug >1)
      cout << " |- Creating 'Accessibility' table" << endl;


   // Write the SQL statement
   
   SQLStatement.str(""); // Initialize it again.
   SQLStatement << "CREATE TABLE " << MySQLDB << ".Accessibility (" << "\n"
      << "  IDCheck MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  IDUrl MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  TagPosition SMALLINT UNSIGNED DEFAULT '0' NULL," << "\n"
      << "  AttrPosition TINYINT UNSIGNED DEFAULT '0' NULL," << "\n"
      << "  Code SMALLINT UNSIGNED DEFAULT '0' NULL," << "\n"
      << "  PRIMARY KEY (IDCheck)," << "\n"
      << "  INDEX (IDUrl, TagPosition, AttrPosition)," << "\n"
      << "  INDEX (Code, IDUrl, TagPosition)" << "\n"
      << ")" << "\n";


   // Executing table creation 

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

///////
   //    Creation of the 'TmpAnchors' temporary table
///////

   if (debug >1)
      cout << " |- Creating 'TmpAnchors' temporary table" << endl;


   // Write the SQL statement
   
   SQLStatement.str(""); // Initialize it again.
   SQLStatement << "CREATE TABLE " << MySQLDB << ".TmpAnchors (" << "\n"
      << "  IDUrl MEDIUMINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  TagPosition SMALLINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  AttrPosition TINYINT UNSIGNED DEFAULT '0' NOT NULL," << "\n"
      << "  Anchor VARCHAR(255) BINARY DEFAULT '' NOT NULL," << "\n"
      << "  PRIMARY KEY (IDUrl, TagPosition, AttrPosition), " << "\n"
      << "  INDEX Idx_Anchor (Anchor(16))" << "\n"
      << ")" << "\n";

   // Executing table creation 

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;

}


///////
   //    Dropping a database for ht://Check
   //    Return 0 if an error occured, 1 OK
///////

int HtmysqlDB::DropDatabase()
{

   std::ostringstream SQLStatement;
   
   if (debug >0)
      cout << "Dropping a previous ht://Check database called '"
      << MySQLDB << "'" << endl;

   SQLStatement << "DROP DATABASE " << MySQLDB;

   // Executing Database dropping

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
}


///////
   //    Keep the database for ht://Check, but recreate the tables
   //    Return 0 if an error occured, 1 OK
///////

int HtmysqlDB::DropTables()
{

   std::ostringstream SQLStatement;
   
   if (debug >0)
      cout << "Dropping the tables of a previous ht://Check database called '"
      << MySQLDB << "'" << endl;

   SQLStatement << "DROP TABLE IF EXISTS " 
      << MySQLDB << ".HtmlAttribute, " 
      << MySQLDB << ".HtmlStatement, " 
      << MySQLDB << ".Link, " 
      << MySQLDB << ".Schedule, " 
      << MySQLDB << ".Server, " 
      << MySQLDB << ".Url, " 
      << MySQLDB << ".htCheck, " 
      << MySQLDB << ".TmpAnchors, " 
      << MySQLDB << ".Cookies, "
      << MySQLDB << ".Accessibility ";


   // Executing Database dropping

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
}


///////
   //    Set the SQL BIG TABLES option (for huge queries)
///////

int HtmysqlDB::SetSQLBigTableOption ()
{

   if (debug >0)
      cout << "Setting option for big tables" << endl;
   
   if (Query ("SET OPTION SQL_BIG_TABLES = 1") == -1)
      return 0;

   return 1;   
}


///////
   //    Optimize all the tables of the Database
///////

int HtmysqlDB::Optimize()
{

   std::ostringstream SQLStatement;
   HtDateTime OptimizeTime;
   
   if (debug >0)
      cout << "Optimizing Database '" << MySQLDB << "'"
         << " - " << OptimizeTime.GetAscTime()<< endl;
   
///////
   //    Optimization of the 'Schedule' table
///////

   if (debug >1)
      cout << " |- 'Schedule' table optimization"
         << " - " << OptimizeTime.GetAscTime()<< endl;

   if (Query ( "OPTIMIZE TABLE Schedule" ) == -1)
      return 0;

///////
   //    Optimization of the 'Url' table
///////

   OptimizeTime.SettoNow();
   if (debug >1)
      cout << " |- 'Url' table optimization"
         << " - " << OptimizeTime.GetAscTime()<< endl;

   if (Query ( "OPTIMIZE TABLE Url" ) == -1)
      return 0;

///////
   //    Optimization of the 'HtmlStatement' table
///////

   OptimizeTime.SettoNow();
   if (debug >1)
      cout << " |- 'HtmlStatement' table optimization"
         << " - " << OptimizeTime.GetAscTime()<< endl;
      
   if (Query ( "OPTIMIZE TABLE HtmlStatement" ) == -1)
      return 0;

///////
   //    Optimization of the 'HtmlAttribute' table
///////

   OptimizeTime.SettoNow();
   if (debug >1)
      cout << " |- 'HtmlAttribute' table optimization"
         << " - " << OptimizeTime.GetAscTime()<< endl;
      
   if (Query ("OPTIMIZE TABLE HtmlAttribute") == -1)
      return 0;

///////
   //    Optimization of the 'Link' table
///////

   OptimizeTime.SettoNow();
   if (debug >1)
      cout << " |- 'Link' table optimization"
         << " - " << OptimizeTime.GetAscTime()<< endl;
      
   if (Query ( "OPTIMIZE TABLE Link" ) == -1)
      return 0;

///////
   //    Optimization of the 'Server' table
///////

   OptimizeTime.SettoNow();
   if (debug >1)
      cout << " |- 'Server' table optimization"
         << " - " << OptimizeTime.GetAscTime()<< endl;
      
   if (Query ( "OPTIMIZE TABLE Server" ) == -1)
      return 0;

///////
   //    Optimization of the 'Cookies' table
///////

   OptimizeTime.SettoNow();
   if (debug >1)
      cout << " |- 'Cookies' table optimization"
         << " - " << OptimizeTime.GetAscTime()<< endl;
      
   if (Query ("OPTIMIZE TABLE Cookies") == -1)
      return 0;

///////
   //    Optimization of the 'htCheck' table
///////

   OptimizeTime.SettoNow();
   if (debug >1)
      cout << " |- 'htCheck' table optimization"
         << " - " << OptimizeTime.GetAscTime()<< endl;
      
   if (Query ("OPTIMIZE TABLE htCheck") == -1)
      return 0;

   return 1;
   
}

///////
   //    Insert a Server into the relative table
   //    Returns 0 if an error occured
///////

int HtmysqlDB::Insert(const _Server &server)
{

    std::ostringstream SQLStatement;
   
   if (debug >4)
      cout << "Inserting a new server into '"
      << MySQLDB << "': " << server.host() << ":" << server.port() << endl;

   SQLStatement << "Insert into " << MySQLDB << ".Server ( "
      << "IDServer, Server, IPAddress, Port, HttpServer, "
      << "HttpVersion, PersistentConnection, Requests"
      << " ) values ( "
      << server.GetID() << ", ";

   // Escape safe host name
   AppendSQLTextField(SQLStatement, server.host().get());

   SQLStatement << ", "
      << "'" << server.GetIPAddress() << "'" << ", "
      << server.port() << ", ";

   // Escape safe host HTTP server
   AppendSQLTextField(SQLStatement, server.GetHttpServer());

   SQLStatement << ", ";

   // Escape safe host HTTP version
   AppendSQLTextField(SQLStatement, server.GetHttpVersion());

   SQLStatement << ", "
      << server.IsPersistentConnectionAllowed() << ", "
      << server.GetRequests()
      << ")";

   // Executing Insert query

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
}


///////
   //    Insert an Url into the relative table
   //    Returns 0 if an error occured
///////

int HtmysqlDB::Insert(const _Url &url)
{

   std::ostringstream SQLStatement;
   std::string Status="";
   std::string DocType="";
   
   // Retrieve the std::string value of the Status
   url.RetrieveConnStatus(Status);   

   // Retrieve the string value for the doctype
   url.RetrieveDocType(DocType);   
   
   if (debug >4)
      cout << "Inserting a new url into '"
      << MySQLDB << "': " << url.get() << endl;

   SQLStatement << "Insert into " << MySQLDB << ".Url ( "
      << "IDUrl, IDServer, Url, HTTPContentType, ContentType, ConnStatus, TransferEncoding, "
      << "LastModified, LastAccess, Size, StatusCode, "
      << "ReasonPhrase, Location, Title, ContentLanguage, "
	  << "Contents, HTTPCharset, Charset, DocType, "
	  << "Description, Keywords"
#ifdef HTDIG_NOTIFICATION
     << ", HtDigEmail, HtDigEmailSubject, HtDigNotificationDate"
#endif
      << " ) values ( "
      << url.GetID() << ", "
      << url.GetIDServer() << ", ";

   // Escape safe URL
   AppendSQLTextField(SQLStatement, url.get());

   SQLStatement << ", ";

   // Escape safe URL content-type (HTTP)
   AppendSQLTextField(SQLStatement, url.GetHTTPContentType());

   SQLStatement << ", ";

   // Escape safe URL content-type
   AppendSQLTextField(SQLStatement, url.GetContentType());

   SQLStatement << ", "
      << "'" << Status << "'" << ", ";

   // TransferEncoding
   AppendSQLTextField(SQLStatement, url.GetTransferEncoding());

   SQLStatement << ", "
      << "'" << (url.GetLastModified()?(url.GetLastModified())->GetTimeStamp():"") << "'" << ", "
      << "'" << (url.GetLastAccess()?(url.GetLastAccess())->GetTimeStamp():"") << "'" << ", "
      << url.GetSize() << ", "
      << url.GetStatusCode() << ", ";

   // Reason Phrase
   AppendSQLTextField(SQLStatement, url.GetReasonPhrase());

   SQLStatement << ", ";

   // Url redirection
   AppendSQLTextField(SQLStatement, url.GetLocation());

   SQLStatement << ", ";

   // Url Title
   AppendSQLTextField(SQLStatement, url.GetTitle());

   SQLStatement << ", ";

   // ContentLanguage
   AppendSQLTextField(SQLStatement, url.GetContentLanguage());

   SQLStatement << ", ";

   // Contents
   const std::string* doc(url.GetContents());
    if (doc) {
	AppendSQLTextField(SQLStatement, doc->c_str());
    }
   else
      SQLStatement << "NULL";

   SQLStatement << ", ";

   // HTTPCharset
   if (url.GetHTTPCharset().length())
   {
      std::string cs;
      for (std::string::const_iterator c(cs.begin()); c != cs.end(); ++c) {
	  cs.push_back (tolower(*c));
      }

      if (AvailableCharsets.find(cs) == AvailableCharsets.end())
         cs = "unknown";

      AppendSQLTextField(SQLStatement, cs);
   }
   else
      SQLStatement << "NULL";

   SQLStatement << ", ";

   // Charset
   if (url.GetCharset().length())
   {
      std::string cs;
      for (std::string::const_iterator c(cs.begin()); c != cs.end(); ++c) {
	  cs.push_back (tolower(*c));
      }
      if (AvailableCharsets.find(cs) == AvailableCharsets.end())
         cs = "unknown";

      AppendSQLTextField(SQLStatement, cs);
   }
   else
      SQLStatement << "NULL";

   SQLStatement << ", ";

   // DocType
   if (DocType.length())
      AppendSQLTextField(SQLStatement, DocType);
   else
      SQLStatement << "NULL";

   SQLStatement << ", ";

   // Description
   if (url.GetDescription().length())
      AppendSQLTextField(SQLStatement, url.GetDescription());
   else
      SQLStatement << "NULL";

   SQLStatement << ", ";

   // Keywords
   if (url.GetKeywords().length())
      AppendSQLTextField(SQLStatement, url.GetKeywords());
   else
      SQLStatement << "NULL";

#ifdef HTDIG_NOTIFICATION
   SQLStatement << ", ";

   // HtDig email
   if (url.GetHtDigEmail().length())
      AppendSQLTextField(SQLStatement, url.GetHtDigEmail());
   else
      SQLStatement << "NULL";
   SQLStatement << ", ";

   // HtDig email subject
   if (url.GetHtDigEmailSubject().length())
      AppendSQLTextField(SQLStatement, url.GetHtDigEmailSubject());
   else
      SQLStatement << "NULL";
   SQLStatement << ", ";

   // HtDig email subject
   if (url.GetHtDigHtDigNotificationDate().length())
      AppendSQLTextField(SQLStatement, url.GetHtDigHtDigNotificationDate());
   else
      SQLStatement << "NULL";

#endif
   SQLStatement << ")";

   // Executing Insert query

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
}


///////
   //    Insert a Schedule into the relative table
   //    Returns 0 if an error occured
///////

int HtmysqlDB::Insert(SchedulerEntry &s)
{

   std::ostringstream SQLStatement;
   std::string Status="";
   std::string Domain="";

   // Retrieve the std::string value of the Status and the domain
   s.RetrieveStatus(Status);   
   s.RetrieveDomain(Domain);   
   
   if (debug >4)
      cout << "Inserting a new scheduler entry into '"
      << MySQLDB << "': " << s << endl;

   SQLStatement << "Insert into " << MySQLDB << ".Schedule ( "
      << "IDUrl, IDServer, Url, CreationTime, Status, IDReferer, HopCount, Domain"
      << " ) values ( "
      << s.GetIDSchedule() << ", "
      << s.GetIDServer() << ", ";

   AppendSQLTextField(SQLStatement, s.GetScheduleUrl());

   SQLStatement << ", "
      << "NOW(), "
      << "'" << Status << "', "
      << s.GetIDReferer() << ", "
      << s.GetHopCount() << ", ";

   if (Domain.length())
      SQLStatement << "'" << Domain << "')";
   else
      SQLStatement << "NULL" << ")";

   // Executing Insert query

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
}


///////
   //    Insert the info into the htCheck table
///////

int HtmysqlDB::Insert(const RunInfo &runinfo)
{

   std::ostringstream SQLStatement;

   if (debug >4)
      cout << "Inserting the retrieval info into '"
      << MySQLDB << "'" << endl;

   SQLStatement << "Insert into " << MySQLDB << ".htCheck ( "
      << "StartTime, EndTime, ScheduledUrls, TotUrls, "
      << "RetrievedUrls, TCPConnections, ServerChanges, "
      << "HTTPRequests, HTTPSeconds, HTTPBytes, AccessibilityChecks, "
      << "HtDigNotification, User"
      << " ) values ( "
      << "'" << runinfo.StartTime.GetTimeStamp() << "', ";

   SQLStatement << "'" << runinfo.FinishTime.GetTimeStamp() << "', "
      << runinfo.ScheduledUrls << ", "
      << runinfo.TotUrls << ", "
      << runinfo.RetrievedUrls << ", "
      << runinfo.TCPConnections << ", "
      << runinfo.ServerChanges << ", "
      << runinfo.HTTPRequests << ", "
      << runinfo.HTTPSeconds << ", "
      << runinfo.HTTPBytes << ", "
      << runinfo.AccessibilityChecks << ", "
      << runinfo.HtDigNotification << ", "
      << "USER()"
      << ")";

   // Executing Insert query

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
   
}


///////
   //    Insert a Cookie into the relative table
   //    Returns 0 if an error occured
///////

int HtmysqlDB::Insert(const HtCookie &cookie)
{
   static unsigned id_cookie = 0;
   std::ostringstream SQLStatement;

   ++id_cookie;   // Increment the index
      
   if (debug >4)
      cout << "Inserting a new cookie into '"
      << MySQLDB << "': " << cookie.GetName() << endl;

   SQLStatement << "Insert into " << MySQLDB << ".Cookies ( "
      << "IDCookie, Name, Value, Path, Domain, MaxAge, Version, SrcUrl, Secure, DomainValid";
   
   if (cookie.GetExpires())
      SQLStatement << ", Expires";
      
   SQLStatement << ") values ( "
      << id_cookie << ", ";

   // Escape safe cookie name
   AppendSQLTextField(SQLStatement, cookie.GetName());

   SQLStatement << ", ";

   // Escape safe cookie value
   AppendSQLTextField(SQLStatement, cookie.GetValue());

   SQLStatement << ", ";

   // Escape safe cookie path
   AppendSQLTextField(SQLStatement, cookie.GetPath());

   SQLStatement << ", ";

   // Escape safe cookie domain
   AppendSQLTextField(SQLStatement, cookie.GetDomain());


   // Insert the max-age attribute
   SQLStatement << ", "
      << cookie.GetMaxAge();

   // Insert the max-age attribute
   SQLStatement << ", "
      << cookie.GetVersion();

   SQLStatement << ", ";

   // Escape safe cookie source URL
   AppendSQLTextField(SQLStatement, cookie.GetSrcURL());

   SQLStatement << ", "
      << (cookie.getIsSecure()?1:0);

   SQLStatement << ", "
      << (cookie.getIsDomainValid()?1:0);

   if (cookie.GetExpires())
      SQLStatement << ", '" << cookie.GetExpires()->GetTimeStamp() << "'";

      
   SQLStatement << ")";

   // Executing Insert query

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
}


///////
   //    Look for a schedule entries, given a filter and a
   //     HtmysqlQuery Result object. Returns -1 if an error occurs
   //     else returns the number of records found.
///////

int HtmysqlDB::Search(SchedulerEntry &filter, HtmysqlQueryResult &result)
{

   std::ostringstream SQLStatement;

   // SQL statement construction
   
   SQLStatement << "Select "
      << "IDUrl"     << ", "     // Url identifier
      << "IDServer"  << ", "     // Server identifier
      << "Url"       << ", "     // Url
      << "Status"    << ", "     // Schedule Status
      << "IDReferer" << ", "     // ID of the referring Url
      << "HopCount"  << "  "     // Hop Count
      << " from Schedule ";

   // Create the SQL 'Where' statement given a filter
   CreateFilter(SQLStatement, filter);

   // Execute and store the Query
   
   return Query(SQLStatement.str(), &result);

}


///////
   //    Get next Element from the query result
   //    Returns 0 if the end has been reached.
///////

int HtmysqlDB::GetNextElement (SchedulerEntry &dest, HtmysqlQueryResult &result)
{

   MYSQL_ROW Rows;
   
   if ((Rows = result.GetNextRecord()) == 0) // end of query reached
      return 0;

   // Reset the destination
   dest.Reset();

   // Set the ID      
   dest.SetIDSchedule(atoi(Rows[0]));

   // Set the ID of the server
   dest.SetIDServer(atoi(Rows[1]));
   
   // Set the Url
   dest.SetScheduleUrl((char *)Rows[2]);

   // Set the Status
   dest.SetStatus((char *)Rows[3]);
      
   // Set the ID of the Referring URL
   dest.SetIDReferer(atoi(Rows[4]));
      
   // Set the hop count number
   dest.SetHopCount(atoi(Rows[5]));
      
   return 1;
}


///////
   //    Create a SQL filter string on a Scheduler Entry
///////

void HtmysqlDB::CreateFilter(std::ostringstream &SQLStatement, SchedulerEntry &filter)
{
   // Search for any filter to be applied to the query
   
   int flag = 0;

   // Applying Url Identifier
      
   if (filter.GetIDSchedule() != 0)   // specified an ID Url
   {
      if (flag)   // already specified a filter
         SQLStatement << "And ";
      else
      {
         SQLStatement << "Where ";
         flag ++; // Add an occurrence to flag
      }

      // Write the sentence
      SQLStatement << "IDUrl = " << filter.GetIDSchedule() << " ";
   }


   // Applying Server
      
   if (filter.GetIDServer() != 0)   // specified a server
   {
      if (flag)   // already specified a filter
         SQLStatement << "And ";
      else
      {
         SQLStatement << "Where ";
         flag ++; // Add an occurrence to flag
      }

      // Write the sentence
      SQLStatement << "IDServer = " << filter.GetIDServer() << " ";
   }
         

   // Applying Url
      
   if (filter.GetScheduleUrl().length())   // specified an Url
   {
      if (flag)   // already specified a filter
         SQLStatement << "And ";
      else
      {
         SQLStatement << "Where ";
         flag ++; // Add an occurrence to flag
      }

      // Write the sentence
      SQLStatement << "Url = ";
   
      AppendSQLTextField(SQLStatement, filter.GetScheduleUrl());
      
   }
         

   // Applying Status
      

   if (filter.GetStatus() != SchedulerEntry::Url_Empty)
   {
      // A Status has been specified
      // Retrieve the std::string value of the Status
      std::string Status="";

      filter.RetrieveStatus(Status);   
      
      if (flag)   // already specified a filter
         SQLStatement << "And ";
      else
      {
         SQLStatement << "Where ";
         flag ++; // Add an occurrence to flag
      }

      // Write the sentence
      SQLStatement << "Status = '" << Status << "' ";
   }

   // Applying Referring Url ID
      
   if (filter.GetIDReferer() != 0)   // specified an ID Referer Url
   {
      if (flag)   // already specified a filter
         SQLStatement << "And ";
      else
      {
         SQLStatement << "Where ";
         flag ++; // Add an occurrence to flag
      }

      // Write the sentence
      SQLStatement << "IDReferer = " << filter.GetIDReferer() << " ";
   }

}


///////
   //    Execute a generic query and stores the results
   //    Returns -1 if an error occured
   //    Else the number of rows retrieved if we want to store a result
   //    or 0 if not.
///////

int HtmysqlDB::Query(const std::string &SQLStatement, HtmysqlQueryResult *result,
   Query_Type qt)
{
   
   if (debug >5)
      cout << "SQL Statement: " << SQLStatement << endl;

   // Executing Select query

   if ( ExecQuery ( SQLStatement))
      return -1;   // An error occured

   // Stores the result of the query
   if (result)
   {
      switch(qt)
      {
         case Htmysql_Stored: // stored query - slower
               if (debug>5) cout << "Stored query" << endl;
               if (! StoreResult(*result)) return -1;
               else return result->GetRows();
               break;
         case Htmysql_Temporary: // faster
               if (debug>5) cout << "Direct query" << endl;
               if (! UseResult(*result)) return -1;
               else return 0;
               break;
      }
      
   }

   return 0;

}


///////
   //    Insert a HtmlStatement into the relative table
   //    Returns 0 if an error occured
///////

int HtmysqlDB::Insert(const HtmlStatement& htmlstatement)
{

   std::ostringstream SQLStatement;
   
   if (debug >4)
      cout << "Inserting a new HtmlStatement into '"
      << MySQLDB << "': " << htmlstatement << endl;

   SQLStatement << "Insert into " << MySQLDB << ".HtmlStatement ( "
      << "IDUrl, Row, Col, TagPosition, Tag, Statement, LinkTagPosition"
      << " ) values ( "
      << htmlstatement.GetIDUrl() << ", "
      << htmlstatement.GetRow() << ", "
      << htmlstatement.GetCol() << ", "
      << htmlstatement.GetTagPosition() << ", ";

   AppendSQLTextField(SQLStatement, htmlstatement.GetTag());
   
   SQLStatement << ", ";

   AppendSQLTextField(SQLStatement, htmlstatement.GetStatement());

   SQLStatement << ", ";

   if (htmlstatement.GetTagPosition() != htmlstatement.GetLinkTagPosition()
      && htmlstatement.GetLinkTagPosition() > 0)
         SQLStatement << htmlstatement.GetLinkTagPosition();
   else
	  SQLStatement << "NULL";

   SQLStatement << ")";

   // Executing Insert query

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
}


///////
   //    Insert a Link description into the HtmlStatement table
   //    Returns 0 if an error occured
///////

int HtmysqlDB::InsertHtmlStatementLinkDescription(const unsigned int IDUrl,
    const unsigned int TagPosition, const std::string& LinkDescription)
{

   std::ostringstream SQLStatement;
   
   if (debug >4)
      cout << "Inserting a new Link description into '"
      << MySQLDB << "': tag " << TagPosition << endl;

   SQLStatement << "Update " << MySQLDB << ".HtmlStatement "
      << " SET LinkDescription=";

   AppendSQLTextField(SQLStatement, LinkDescription);
   
   SQLStatement << " WHERE IDUrl=" << IDUrl << " AND TagPosition=" << TagPosition;

   // Executing Insert query

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
}


///////
   //    Insert a HtmlAttribute into the relative table
   //    Returns 0 if an error occured
///////

int HtmysqlDB::Insert(const HtmlAttribute& htmlattribute)
{

   std::ostringstream SQLStatement;
   
   if (debug >4)
      cout << "Inserting a new HtmlAttribute into '"
      << MySQLDB << "': " << htmlattribute << endl;

   SQLStatement << "Insert into " << MySQLDB << ".HtmlAttribute ( "
      << "IDUrl, TagPosition, AttrPosition, Attribute, Content"
      << " ) values ( "
      << htmlattribute.GetIDUrl() << ", "
      << htmlattribute.GetTagPosition() << ", "
      << htmlattribute.GetAttrPosition() << ", ";
      
   AppendSQLTextField(SQLStatement, htmlattribute.GetAttribute());

   SQLStatement << ", ";
   
   AppendSQLTextField(SQLStatement, htmlattribute.GetContent());

   SQLStatement << ")";

   // Executing Insert query

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
}


///////
   //    Insert a AccessibilityCheck into the relative table
   //    Returns 0 if an error occured
///////

int HtmysqlDB::Insert(const AccessibilityCheck& accessibilitycheck)
{

   std::ostringstream SQLStatement;
   
   if (debug >4)
      cout << "Inserting a new Accessibility into '"
      << MySQLDB << "': " << accessibilitycheck << endl;

   SQLStatement << "Insert into " << MySQLDB << ".Accessibility ( "
      << "IDCheck, IDUrl, TagPosition, AttrPosition, Code"
      << " ) values ( "
      << accessibilitycheck.GetIDCheck() << ", "
      << accessibilitycheck.GetIDUrl() << ", ";

   if (accessibilitycheck.GetTagPosition() == 0)
      SQLStatement << "NULL";
   else
      SQLStatement << accessibilitycheck.GetTagPosition();

   SQLStatement << ", ";

   if (accessibilitycheck.GetAttrPosition() == 0)
      SQLStatement << "NULL";
   else
      SQLStatement << accessibilitycheck.GetAttrPosition();

   SQLStatement << ", " << accessibilitycheck.GetCode() << ")";
      
   // Executing Insert query

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
}


///////
   //    Insert a Link into the relative table
   //    Returns 0 if an error occured
///////

int HtmysqlDB::Insert(const Link& link)
{

   std::ostringstream SQLStatement;
   std::string LinkType="";
   std::string LinkResult="";
   std::string LinkDomain="";
   
   // Retrieve the std::string value of the type
   link.RetrieveLinkType(LinkType);   
   
   // Retrieve the std::string value of the result
   link.RetrieveLinkResult(LinkResult);   
   
   // Retrieve the std::string value of the domain
   link.RetrieveLinkDomain(LinkDomain);   
   
   
   if (debug >4)
      cout << "Inserting a new Link into '"
      << MySQLDB << "': " << link << endl;

   SQLStatement << "Insert into " << MySQLDB << ".Link ( "
      << "IDUrlSrc, IDUrlDest, TagPosition, AttrPosition, "
      << "Anchor, LinkType, LinkResult, LinkDomain"
      << " ) values ( "
      << link.GetIDUrlSrc() << ", "
      << link.GetIDUrlDest() << ", "
      << link.GetTagPosition() << ", "
      << link.GetAttrPosition() << ", ";

   AppendSQLTextField(SQLStatement, link.GetAnchor());

   SQLStatement << ", "
      << "'" << LinkType << "', "
      << "'" << LinkResult << "', ";
      
   if (LinkDomain.length())
      SQLStatement << "'" << LinkDomain << "')";
   else
      SQLStatement << "NULL" << ")";

   // Executing Insert query

   if (Query (SQLStatement.str()) == -1)
      return 0;   // An error occured

   return 1;
}


// This method just make a string to be escape safe, which means
// that it makes it get ready to be put into the database through
// an INSERT SQL statement. The string character used is ' (single
// quote).

//void HtmysqlDB::AppendSQLTextField(std::string &Dest, const char *source)

void HtmysqlDB::AppendSQLTextField(std::string &Dest, const char* source)
{

	const char* start (source);
	const long length( strlen(source) );

	ESCAPE_STRING(mysql, start, length);

}

void HtmysqlDB::AppendSQLTextField(std::string &Dest, const std::string source)
{

	const char* start (source.c_str());
	const long length( source.length() );

	ESCAPE_STRING(mysql, start, length);

}

void HtmysqlDB::AppendSQLTextField(std::ostringstream &Dest, const std::string source)
{

	const char* start (source.c_str());
	const long length( source.length() );

	ESCAPE_OSTRING(mysql, start, length);
}

void HtmysqlDB::AppendSQLTextField(std::ostringstream &Dest, const char* source)
{

	const char* start (source);
	const long length( strlen(source) );

	ESCAPE_OSTRING(mysql, start, length);

}


// Alter the Link table, by creating the Indexes - not used
// before now (this could save a lot of time when adding new
// link entries while crawling

int HtmysqlDB::CreateLinkTableIndexes()
{

   int result = 0;
   std::ostringstream SQLStatement;

   if (debug>0)
      cout << "Create indexes for the Link table" << endl;

   // IDUrlDest index
   SQLStatement << "CREATE INDEX Idx_IDUrlDest ON " 
      << MySQLDB << ".Link " 
      << "(IDUrlDest)" << "\n";

   if (debug>1)
      cout << "   > Idx_IDUrlDest on IDUrlDest" << endl;

   if ((result=Query (SQLStatement.str())) == -1)
      return result;

   // Anchor index
   SQLStatement.str("");
   SQLStatement << "CREATE INDEX Idx_Anchor ON " 
      << MySQLDB << ".Link " 
      << "(Anchor(8))" << "\n";

   if (debug>1)
      cout << "   > Idx_Anchor on Anchor" << endl;

   if ((result=Query (SQLStatement.str())) == -1)
      return result;

   // LinkType index
   SQLStatement.str("");
   SQLStatement << "CREATE INDEX Idx_LinkType ON "
      << MySQLDB << ".Link "
      << "(LinkType)" << "\n";

   if (debug>1)
      cout << "   > LinkType on LinkType" << endl;

   if ((result=Query (SQLStatement.str())) == -1)
      return result;

   // LinkResult index
   SQLStatement.str("");
   SQLStatement << "CREATE INDEX Idx_LinkResult ON " << "\n"
      << "   " << MySQLDB << ".Link " << "\n"
      << "   (LinkResult)" << "\n";

   if (debug>1)
      cout << "   > LinkResult on LinkResult" << endl;

   if ((result=Query (SQLStatement.str())) == -1)
      return result;

   // Exec the query
   return result;
}


// Create the temporary table for anchors

int HtmysqlDB::CreateAnchorsTable()
{

   // Create the table with all the anchors;

   std::ostringstream SQLFillTemporaryTable;
   SQLFillTemporaryTable << "INSERT INTO TmpAnchors "
      << "SELECT HtmlAttribute.IDUrl, HtmlAttribute.TagPosition, "
      << "HtmlAttribute.AttrPosition, HtmlAttribute.Content FROM HtmlAttribute, "
      << "HtmlStatement WHERE HtmlStatement.IDUrl=HtmlAttribute.IDUrl AND "
      << "HtmlAttribute.TagPosition=HtmlStatement.TagPosition AND "
      << "((HtmlAttribute.Attribute='name' AND HtmlStatement.Tag='a') "
      << "OR HtmlAttribute.Attribute='id')";

   if (debug>2)
      cout << "Filling temporary table TmpAnchors" << endl;

   // Exec the query
   return Query(SQLFillTemporaryTable.str());


}


// Create and fill the temporary table for managing anchors

int HtmysqlDB::AnchorsTable(ostream& output)
{
   std::ostringstream SQLStatement;
   Link LinkTmp;  // Temporary link object

   // Create the temporary table
   CreateAnchorsTable();

   // Now the temporary table is filled with all the anchors found

   SQLStatement << "SELECT DISTINCT Link.IDUrlSrc, Link.IDUrlDest, "
      << "Link.TagPosition, Link.AttrPosition, Link.Anchor, TmpAnchors.Anchor "
      << "FROM Schedule, Link LEFT JOIN TmpAnchors ON "
      << "(TmpAnchors.IDUrl=Link.IDUrlDest "
      << "AND TmpAnchors.Anchor=Link.Anchor) "
      << "WHERE Link.Anchor != '' AND LCASE(Link.Anchor) != 'top' "
      << "AND Link.LinkResult='OK' "
      << "AND Link.IDUrlDest=Schedule.IDUrl AND Schedule.Status='Retrieved'";
  
   HtmysqlQueryResult ResultTmp;
   
   int NumAnchors = 0;
   int NumAnchorsNotFound = 0;

   if (Query (SQLStatement.str(), &ResultTmp) == -1)
      return -1;

   MYSQL_ROW LinkRow;
   
   while ((LinkRow = (ResultTmp.GetNextRecord())))
   {
      ++NumAnchors;
      
      if (! LinkRow[5])
      {
         // Anchor Not found
         
         ++NumAnchorsNotFound;
         LinkTmp.Reset();  
         LinkTmp.SetIDUrlSrc((unsigned int) atoi(LinkRow[0]));
         LinkTmp.SetIDUrlDest((unsigned int) atoi(LinkRow[1]));
         LinkTmp.SetTagPosition((unsigned int) atoi(LinkRow[2]));
         LinkTmp.SetAttrPosition((unsigned int) atoi(LinkRow[3]));
         LinkTmp.SetAnchor(LinkRow[4]);

         LinkTmp.SetLinkResult(Link::Link_AnchorNotFound);
         
         if (debug>2)
               output << "Anchor not found: " << LinkTmp
                  << "#" << LinkTmp.GetAnchor() << endl;

         // Update the link table - setting the results

         if(SetAnchorsResults(LinkTmp) == -1) // A database error occured
            return -1;
      }

   }

   ResultTmp.Free();      

   if (debug>0)
      output << "Anchors checked: " << NumAnchors
         << " - Not found: " << NumAnchorsNotFound << endl;


///////
   //    Drop the 'TmpAnchors' temporary table
///////

   if (debug >1)
      output << " |- Dropping 'TmpAnchors' temporary table" << endl;

   // Write the SQL statement
   SQLStatement.str("");
   SQLStatement << "DROP TABLE " << MySQLDB << ".TmpAnchors";

   // Executing table dropping
   return Query (SQLStatement.str());

}

int HtmysqlDB::AnchorsNotFound(ostream& output)
{
   // Show broken anchors

   std::ostringstream SQLStatement;

   SQLStatement << "SELECT UrlSrc.IDUrl as IDUrlSrc, UrlDest.IDUrl as IDUrlDest,"
      << "UrlSrc.Url as UrlSrc, UrlDest.Url as UrlDest, "
      << "Link.LinkType, Link.Anchor, HtmlStatement.Statement "
      << "FROM Url UrlDest, Url UrlSrc, Link "
      << "LEFT JOIN HtmlStatement "
      << "ON (HtmlStatement.IDUrl = Link.IDUrlSrc "
      << "AND HtmlStatement.TagPosition = Link.TagPosition) "
      << "WHERE Link.LinkResult = 'AnchorNotFound' "
      << "AND UrlSrc.IDUrl = Link.IDUrlSrc "
      << "AND UrlDest.IDUrl = Link.IDUrlDest "
      << "ORDER BY UrlSrc, UrlDest, Link.TagPosition, "
      << "Link.AttrPosition";

   HtmysqlQueryResult ResultTmp;
   
   int NumUrlsNotFound = 0;
   int NumAnchorsNotFound = 0;
   int OldIDSrc=0;
   int OldIDDest=0;
   std::string OldAnchor;

   output << endl << "Checking anchors not found"
          << endl << "==========================" << endl;

   if (Query (SQLStatement.str(), &ResultTmp) == -1)
      return -1;

   MYSQL_ROW AnchorsNotFound;
   
   while ((AnchorsNotFound = (ResultTmp.GetNextRecord())))
   {

      ++NumAnchorsNotFound;

      if (OldIDDest != atoi(AnchorsNotFound[1]) ||
         mystrcasecmp(OldAnchor.c_str(), (const char *) AnchorsNotFound[5]))
      {

         // New URL not found
         ++NumUrlsNotFound;
         output << endl;
         output << "Anchor not found: " << (char *) AnchorsNotFound[3]
            << "#" << (char *) AnchorsNotFound[5] << endl;

         OldIDDest = atoi(AnchorsNotFound[1]);      // New assignment
         OldAnchor = (char *) AnchorsNotFound[5];   // New assignment
         OldIDSrc = 0;
         
      }

      if (OldIDSrc != atoi(AnchorsNotFound[0]))
      {
         output << "|- Referenced by: " << (char *) AnchorsNotFound[2] << endl;
         OldIDSrc = atoi(AnchorsNotFound[0]); // New assignment
      }
      
      if (! (char *) AnchorsNotFound[8])
      {
         // No Statement. It's an HTTP redirection
         output << "| |- " << (char *) AnchorsNotFound[4] << endl;
      }
      else
      {
         output << "| |- Tag        : <" << (char *) AnchorsNotFound[6] << ">" << endl;
      }

   }

   ResultTmp.Free();

   if (NumUrlsNotFound)
      output << endl;
   
   output << "Anchor not Found: " << NumUrlsNotFound << endl;
   output << "Broken Links    : " << NumAnchorsNotFound << endl;

   return 0;
}

// Set the result type for anchors

int HtmysqlDB::SetAnchorsResults(Link &LinkTmp)
{

   std::string strResult;
   LinkTmp.RetrieveLinkResult(strResult);

   std::ostringstream SQLUpdateStatement;
   
   SQLUpdateStatement << "Update Link SET LinkResult = '" << strResult << "'"
      << " WHERE IDUrlSrc = " << LinkTmp.GetIDUrlSrc()
      << " AND IDUrlDest = " << LinkTmp.GetIDUrlDest()
      << " AND Anchor = ";

   AppendSQLTextField(SQLUpdateStatement, LinkTmp.GetAnchor());

   return Query(SQLUpdateStatement.str());

}


///////
   //    Set the results for the links. It is called by the Scheduler at the end
   //    of the crawl.
   //    Look for all the links that have 'NotChecked' flag in the LinkResult
   //    field. For each of them, it checks all of the links, and see if
   //    they are eithrt found or not retrieved or broken or redirected.
   //    Then it updates the Link table with the right flag.
   //    Now the BadEncoded case is treated as well.
///////


int HtmysqlDB::SetLinkResults(ostream& output)
{

   // Set the link results

   std::ostringstream SQLStatement;
   Link LinkTmp;  // Temporary link object
   
   SQLStatement << "SELECT distinct IDUrlSrc, IDUrlDest, "
      << "Url.StatusCode, Link.LinkResult "
      << "FROM Link LEFT JOIN Url ON (Link.IDUrlDest=Url.IDUrl) "
      << "WHERE Link.LinkResult='NotChecked' or Link.LinkResult='BadEncoded'";

   HtmysqlQueryResult ResultTmp;
   
   int NumLinks = 0;

   if (debug>0)
      output << endl << "Setting link results" << endl;

   if (Query (SQLStatement.str(), &ResultTmp) == -1)
      return -1;

   std::string strResult;
   MYSQL_ROW LinkRow;
   
   while ((LinkRow = (ResultTmp.GetNextRecord())))
   {
      unsigned int StatusCode;
      
      ++NumLinks;
      
      LinkTmp.Reset();  
      LinkTmp.SetIDUrlSrc((unsigned int) atoi(LinkRow[0]));
      LinkTmp.SetIDUrlDest((unsigned int) atoi(LinkRow[1]));
      LinkTmp.SetLinkResult(LinkRow[3]);

      if (LinkRow[2])
      {
         StatusCode=(unsigned int) atoi(LinkRow[2]);
         if (StatusCode == 200)
         {
            // Found - Let's check whether we had yet labeled it as 'BadEncoded' or not
            if (LinkTmp.GetLinkResult() != Link::Link_BadEncoded)
               LinkTmp.SetLinkResult(Link::Link_OK);  // No
         
         if (debug>2)
               output << "Link found: " << LinkTmp << endl;
         }
         else if (StatusCode >= 300 && StatusCode < 400)
         {
            // Redirected
            if (LinkTmp.GetLinkResult() != Link::Link_BadEncoded)
               LinkTmp.SetLinkResult(Link::Link_Redirected);
         
         if (debug>2)
               output << "Link redirected: " << LinkTmp << endl;
         }
         else if (StatusCode == 401)
         {
            // Not autorized
            if (LinkTmp.GetLinkResult() != Link::Link_BadEncoded)
               LinkTmp.SetLinkResult(Link::Link_NotAuthorized);
         
         if (debug>2)
               output << "Link not authorized: " << LinkTmp << endl;
         }
         else
         {
            // Broken
            LinkTmp.SetLinkResult(Link::Link_Broken);
         
         if (debug>2)
               output << "Link broken: " << LinkTmp << endl;
         }
      }
      else
      {
         if (LinkTmp.GetLinkResult() != Link::Link_BadEncoded)
            LinkTmp.SetLinkResult(Link::Link_NotRetrieved); // Not Retrieved
      }
      
      LinkTmp.RetrieveLinkResult(strResult);

      std::ostringstream SQLUpdateStatement;
      SQLUpdateStatement << "Update Link SET LinkResult = '" << strResult << "'"
         << " WHERE IDUrlSrc = " << LinkTmp.GetIDUrlSrc()
         << " AND IDUrlDest = " << LinkTmp.GetIDUrlDest()
         << " AND LinkResult = '" << LinkRow[3] << "'";

      if(Query(SQLUpdateStatement.str()) == -1) // A database error occured
         return -1;

   }

   ResultTmp.Free();      
   if (debug>0)
      output << "Links checked: " << NumLinks << endl;

   return 0;
   
}


///////
   //    Show the broken links summary
///////

int HtmysqlDB::ShowBrokenLinks(ostream& output)
{

   // Show broken links

   std::ostringstream SQLStatement;

   SQLStatement << "SELECT "
      << "Link.IDUrlSrc as IDUrlSrc, "
      << "Link.IDUrlDest as IDUrlDest, "
      << "Url.Url as UrlSrc, "
      << "Schedule.Url as UrlDest, "
      << "HtmlStatement.Statement, "
      << "Link.LinkResult as LinkResult "
      << "FROM Url, Schedule, Link "
      << "LEFT JOIN HtmlStatement "
      << "ON HtmlStatement.IDUrl = Link.IDUrlSrc "
      << "AND HtmlStatement.TagPosition = Link.TagPosition "
      << "WHERE (Link.LinkResult = 'Broken' OR Link.LinkResult = 'BadEncoded')"
      << "AND Schedule.IDUrl = Link.IDUrlDest "
      << "AND Url.IDUrl = Link.IDUrlSrc "
      << "ORDER BY UrlSrc, UrlDest, Link.TagPosition, "
      << "Link.AttrPosition";

   HtmysqlQueryResult ResultTmp;
   
   int NumUrlsNotFound = 0;
   int NumBrokenLinks = 0;
   int OldIDSrc=0;
   int OldIDDest=0;

   output << endl << "Checking links"
          << endl << "==============" << endl;

   if (Query (SQLStatement.str(), &ResultTmp) == -1)
      return -1;

   MYSQL_ROW BrokenLinks;
   
   while ((BrokenLinks = (ResultTmp.GetNextRecord())))
   {

      ++NumBrokenLinks;
      
      if (OldIDDest != atoi(BrokenLinks[1]))
      {

         // New URL not found
         ++NumUrlsNotFound;
         output << endl;
         if (!strncmp("BadEncoded", BrokenLinks[5], 10))
            output << "Bad encoded link: ";
         else
            output << "URL Not found: ";

         output << (char *) BrokenLinks[3] << endl;

         OldIDDest = atoi(BrokenLinks[1]); // New assignment
         OldIDSrc = 0;
         
      }

      if (OldIDSrc != atoi(BrokenLinks[0]))
      {
         output << "|- Referenced by: " << (char *) BrokenLinks[2] << endl;
         OldIDSrc = atoi(BrokenLinks[0]); // New assignment
      }

      if ((char *) BrokenLinks[4])
      {
         output << "| |- Tag        : <" << (char *) BrokenLinks[4]
            << ">" << endl;
      }

   }

   ResultTmp.Free();

   if (NumUrlsNotFound)
      output << endl;
   
   output << "Urls not Found: " << NumUrlsNotFound << endl;
   output << "Broken Links  : " << NumBrokenLinks << endl;
      
   return 0;
   
}


///////
   //    Show the status codes retrieved
///////

int HtmysqlDB::ShowStatusCode(ostream &output)
{

   // Show the status codes retrieved

   std::string SQLStatement="SELECT StatusCode, ReasonPhrase, count(*), ConnStatus FROM Url GROUP by StatusCode, ReasonPhrase, ConnStatus ORDER BY StatusCode, ReasonPhrase, ConnStatus";
   
   HtmysqlQueryResult ResultTmp;
   
   int NumStatus = 0;
   int StatusTmp;
   bool warning = false;

   output << endl << "Checking Status Codes returned"
          << endl << "==============================" << endl;

   if (Query (SQLStatement, &ResultTmp) == -1)
      return -1;

   MYSQL_ROW StatusCodes;
   
   while ((StatusCodes = (ResultTmp.GetNextRecord())))
   {

      StatusTmp = atoi(StatusCodes[0]);
      
      if (!StatusTmp)
      {
         // Abnormal status code
         warning = true;  // StatusCode = 0. No Header

         output << "Connection result: " << StatusCodes[3]
            << " Occurrences: " << setw(6) << atoi(StatusCodes[2]);

         output << " <Warning>";

         output << endl;
      
      }
      else
      {
         ++NumStatus;

         // Not abnomarl status code
         output << "Status Code: " << setw(3) << StatusTmp
            << " Occurrences: " << setw(6) << atoi(StatusCodes[2]);

         if (strlen((char *) StatusCodes[1]))
            output << " (" << (char *) StatusCodes[1] << ")";

         output << endl;

      }

   }

   ResultTmp.Free();
   
   if (warning)
   {

      output << "Warning!!! Some connection didn't go pretty good."
         << endl;

   }
      
   output << "Status codes encountered: " << NumStatus << endl;

   return 0;
   
}


int HtmysqlDB::ShowContentTypesPerServer(ostream &output)
{

   std::string SQLStatement="SELECT Server, Port, ContentType, count(*) from Url, Server WHERE Url.StatusCode = 200 AND Url.IDServer = Server.IDServer GROUP BY Server, Port, ContentType ORDER BY Server, Port";
   
   HtmysqlQueryResult ResultTmp;
   
   int NumServer = 0;
   int TotNumTot = 0;
   int TotNumImg = 0;
   int TotNumHTML = 0;
   int NumTot = 0;
   int NumImg = 0;
   int NumHTML = 0;
   std::string OldServer;
   std::string NewServer;
   int OldPort = 0;
   int NewPort = 0;

   output << endl << "Checking Content-Types successfully returned"
          << endl << "============================================" << endl;


   if (Query (SQLStatement, &ResultTmp) == -1)
      return -1;

   MYSQL_ROW StatusCodes;
   
   while ((StatusCodes = (ResultTmp.GetNextRecord())))
   {
      NewServer = (char *)StatusCodes[0];
      NewPort = atoi (StatusCodes[1]);
      
      if (OldServer != NewServer || OldPort != NewPort)
      {
         // Change of Server
         
         if (NumServer)
         {
            // Close a previous server
            
            output << "      Total Urls successfully seen: " << NumTot
               << " (HTML: " << NumHTML
               << " - images: " << NumImg
			   << " - other: " << NumTot-NumImg-NumHTML
               << ")" << endl;

			// Updates the total values
			TotNumTot += NumTot;
			TotNumImg += NumImg;
			TotNumHTML += NumHTML;
            
         }

         ++NumServer;
         output << endl << setw(4) << NumServer << ". " << NewServer
            << ":" << NewPort << endl;
         
         NumTot = 0;
         NumImg = 0;
         NumHTML = 0;
         
         OldServer = NewServer;
         OldPort = NewPort;
         
      }
      
      output << "      " << setw(32) << setiosflags( ios::left )
         << (char *)StatusCodes[2] << ":"
         << setw(6) << resetiosflags( ios::left )
         << atoi(StatusCodes[3]) << endl;

      NumTot += atoi(StatusCodes[3]);
      
      if (!strncmp("image/", (char *)StatusCodes[2], 6))
         NumImg += atoi(StatusCodes[3]);

      if (!strncmp("text/html", (char *)StatusCodes[2], 9))
         NumHTML += atoi(StatusCodes[3]);

   }

   ResultTmp.Free();    // Free the query results

   if (NumServer)
   {
      // Close a previous server
            
      output << "      Total Urls successfully seen: " << NumTot
         << " (HTML: " << NumHTML
         << " - images: " << NumImg
		 << " - other: " << NumTot-NumImg-NumHTML
         << ")" << endl;
            
      // Updates the total values
      TotNumTot += NumTot;
      TotNumImg += NumImg;
      TotNumHTML += NumHTML;
            
   }

   // Flush an empty line ans display the number of servers that have been
   // crawled by htcheck and info regarding the types globally
   output << endl << " Total Urls successfully seen: " << TotNumTot
      << " (HTML: " << TotNumHTML
      << " - images: " << TotNumImg
	  << " - other: " << TotNumTot-TotNumImg-TotNumHTML
      << ")" << endl;
   output << " Servers seen: " << NumServer << endl << endl;

   return 0;
   
}



///////
   //    Calculate the size to be added to URLs (links of the 'Direct' type)
   //    After executing a query, it updates SizeAdd field of the URL table
   //	 A value of bytes to be added to a URL (an HTML document for now)
   //	 depends on the attributes used to link to another URL. For example:
   //    images are called usually with <IMG src="URL A">. This is considered
   //    as a direct link and the size of URL A is being added to the SizeAdd
   //    field of the URL calling it. But this is added only once, even if
   //    inside the document it's called twice, 3 times, a hundred times.
   //    Indeed we suppose the user has a cache system on his computer.
   //    By adding a URL size with the SizeAdd field, we obtain an approximate
   //    URL weight.
///////

int HtmysqlDB::CalculateUrlSizeAdd(ostream &output)
{

   // Show the status codes retrieved

   std::string SQLStatement="SELECT distinct Link.IDUrlSrc, Link.IDUrlDest, Url.Size FROM Link, Url WHERE Link.LinkType='Direct' AND Link.IDUrlDest=Url.IDUrl AND Url.Size > 0 ORDER BY Link.IDUrlSrc";
   
   HtmysqlQueryResult ResultTmp;
   
   int NumUrls = 0;
   int NumDirectLinks = 0;
   int IDUrlSrc = 0;       // Current source IDUrl
   int IDPrev = 0;         // Previous IDUrl
   int SizeAdd = 0;        // Total size to be added to the source URL
   int CurrentSize = 0;    // Size of the Destination URL
   
   if (debug>0)
      output << endl << "Calculating added size of URL"
         << " (elements directly loaded with a URL)" << endl;

   if (Query (SQLStatement, &ResultTmp) == -1)
      return -1;

   MYSQL_ROW RecordTmp;
   
   while ((RecordTmp = (ResultTmp.GetNextRecord())))
   {

      ++NumDirectLinks;

      IDUrlSrc = atoi(RecordTmp[0]);
      
      if (IDUrlSrc != IDPrev && IDPrev != 0)
      {
         ///////
            //   Break
         ///////

         ++NumUrls;

         // Update

         if (debug>2)
            output << "Updating Url.SizeAdd - " << IDPrev << " - "
               << SizeAdd << " (" << NumUrls << ")" << endl;
         
         // Query execution for update (Url Table)
         std::ostringstream SQLStatementUpdate;
	
         SQLStatementUpdate << "Update Url Set SizeAdd = "
	     << SizeAdd << " Where IDUrl = " << IDPrev;
      
         if(Query(SQLStatementUpdate.str()) == -1) // A database error occured
            return -1;
         
         SizeAdd = 0;
         
      }
      
      CurrentSize = atoi(RecordTmp[2]);
      SizeAdd += CurrentSize;

      IDPrev = IDUrlSrc;   // Storing Previous Url ID

   }

   if (IDPrev != 0)
   {
      // Break
      // Update
      if (debug>2)
         output << "Updating Url.SizeAdd - " << IDPrev << " - "
            << SizeAdd << " (" << NumUrls << ")" << endl;

      // Query execution for update (Url Table)
      std::ostringstream SQLStatementUpdate;
      SQLStatementUpdate << "Update Url Set SizeAdd = "
	<< SizeAdd << " Where IDUrl = " << IDPrev;
      
      if(Query(SQLStatementUpdate.str()) == -1) // A database error occured
         return -1;
         
      SizeAdd = 0;
         
   }

   ResultTmp.Free();

   if (debug>0)
   {
      output << "Direct Links found: " << NumDirectLinks << endl;
      output << "Urls updated: " << NumUrls << endl;
   }

   return 0;
   
}


///////
   // Updates a scheduler entry's status
///////
int HtmysqlDB::UpdateStatus(const SchedulerEntry& entry)
{
    std::string status;
    std::ostringstream SQLStatement;

    entry.RetrieveStatus(status);

    SQLStatement << "Update Schedule Set Status = '" << status << "' "
        << "Where IDUrl = " << entry.GetIDSchedule();

    if(Query (SQLStatement.str()) == -1) // A database error occured
        return -1;

    return 0;
}


///////
   //    Set the list of available charsets
///////

void HtmysqlDB::LoadAvailableCharsets(const std::string& c)
{
    if (debug>0)
        cout << "Loading available charsets: ";
    StringList charsets(c.c_str(), " \t");

    for (int i = 0; i < charsets.Count(); i++)
    {
        if (debug>0)
            cout << charsets[i] << " ";

	AvailableCharsets.insert(charsets[i]);
    }

    if (debug>0)
        cout << endl;
}


#if 0

///////
   //    Look for a schedule entries, given a filter and a
   //     HtmysqlQuery Result object. Returns -1 if an error occurs
   //     else returns the number of records found.
///////

int HtmysqlDB::Search(_Server &filter, HtmysqlQueryResult &result)
{

   std::ostringstream SQLStatement;

   // SQL statement construction
   
   SQLStatement << "Select "
      << "IDServer"  << ", "     // Server identifier
      << "IPAddress" << ", "     // IP Address
      << "Port" << ", "
      << "HttpServer" << ", "
      << "HttpVersion" << ", "
      << "PersistentConnection" << ", "
      << "Requests"              // Number of requests
      << " from Server ";

   // Create the SQL 'Where' statement given a filter
   CreateFilter(SQLStatement, filter);

   // Execute and store the Query
   
   return Query(SQLStatement, &result);

}



///////
   //    Create a SQL filter string on a _Server Entry
   //    Incomplete ...
///////

void HtmysqlDB::CreateFilter(std::string &SQLStatement, _Server &filter)
{
   // Search for any filter to be applied to the query
   
   int flag = 0;

   // Applying Server Identifier
      
   if (filter.GetID() != 0)   // specified an ID
   {
      if (flag)   // already specified a filter
         SQLStatement << "And ";
      else
      {
         SQLStatement << "Where ";
         flag ++; // Add an occurrence to flag
      }

      // Write the sentence
      SQLStatement << "IDServer = " << filter.GetID() << " ";
   }

   // Applying Server name (host)
      
   if (filter.host() != 0)
   {
      if (flag)   // already specified a filter
         SQLStatement << "And ";
      else
      {
         SQLStatement << "Where ";
         flag ++; // Add an occurrence to flag
      }

      // Write the sentence
      SQLStatement << "Server = " << filter.host() << " ";
   }

   // Applying Server port (not done)
      
   // Applying Server Info
      
   if (filter.GetHttpServer() != 0)
   {
      if (flag)   // already specified a filter
         SQLStatement << "And ";
      else
      {
         SQLStatement << "Where ";
         flag ++; // Add an occurrence to flag
      }

      // Write the sentence
      SQLStatement << "HttpServer = " << filter.GetHttpServer() << " ";
   }

   // Applying Server Version
      
   if (filter.GetHttpVersion() != 0)
   {
      if (flag)   // already specified a filter
         SQLStatement << "And ";
      else
      {
         SQLStatement << "Where ";
         flag ++; // Add an occurrence to flag
      }

      // Write the sentence
      SQLStatement << "HttpVersion = " << filter.GetHttpVersion() << " ";
   }

}

#endif
