#! /usr/bin/perl -w
# $Id: update_blastdb.pl 419604 2013-11-26 22:49:06Z rafanovi $
# ===========================================================================
#
#                            PUBLIC DOMAIN NOTICE
#               National Center for Biotechnology Information
#
#  This software/database is a "United States Government Work" under the
#  terms of the United States Copyright Act.  It was written as part of
#  the author's official duties as a United States Government employee and
#  thus cannot be copyrighted.  This software/database is freely available
#  to the public for use. The National Library of Medicine and the U.S.
#  Government have not placed any restriction on its use or reproduction.
#
#  Although all reasonable efforts have been taken to ensure the accuracy
#  and reliability of the software and data, the NLM and the U.S.
#  Government do not and cannot warrant the performance or results that
#  may be obtained by using this software or data. The NLM and the U.S.
#  Government disclaim all warranties, express or implied, including
#  warranties of performance, merchantability or fitness for any particular
#  purpose.
#
#  Please cite the author in any work or product based on this material.
#
# ===========================================================================
#
# Author:  Christiam Camacho
#
# File Description:
#   Script to download the pre-formatted BLAST databases from the NCBI ftp
#   server.
#
# ===========================================================================

use strict;
use warnings;
use Net::FTP;
use Getopt::Long;
use Pod::Usage;
use File::stat;
use Digest::MD5;
use Archive::Tar;

use constant NCBI_FTP => "ftp.ncbi.nlm.nih.gov";
use constant BLAST_DB_DIR => "/blast/db";
use constant USER => "anonymous";
use constant PASSWORD => "anonymous";
use constant DEBUG => 0;
use constant MAX_DOWNLOAD_ATTEMPTS => 3;
use constant EXIT_FAILURE => 2;

# Process command line options
my $opt_verbose = 1;
my $opt_quiet = 0;
my $opt_force_download = 0;     
my $opt_help = 0;
my $opt_passive = 0;
my $opt_timeout = 120;
my $opt_showall = 0;
my $opt_show_version = 0;
my $opt_decompress = 0;
my $result = GetOptions("verbose+"      =>  \$opt_verbose,
                        "quiet"         =>  \$opt_quiet,
                        "force"         =>  \$opt_force_download,
                        "passive"       =>  \$opt_passive,
                        "timeout=i"     =>  \$opt_timeout,
                        "showall"       =>  \$opt_showall,
                        "version"       =>  \$opt_show_version,
                        "decompress"    =>  \$opt_decompress,
                        "help"          =>  \$opt_help);
$opt_verbose = 0 if $opt_quiet;
die "Failed to parse command line options\n" unless $result;
pod2usage({-exitval => 0, -verbose => 2}) if $opt_help;
pod2usage({-exitval => 0, -verbose => 2}) unless (scalar @ARGV or 
                                                  $opt_showall or
                                                  $opt_show_version);

my $exit_code = 0;
$|++;

# Connect and download files
my $ftp = &connect_to_ftp() unless ($opt_show_version);
if ($opt_show_version) {
    my $revision = '$Revision: 419604 $';
    $revision =~ s/\$Revision: | \$//g;
    print "$0 version $revision\n";
} elsif ($opt_showall) {
    print "$_\n" foreach (sort(&get_available_databases()));
} else {
    my @files = sort(&get_files_to_download());
    my @files2decompress;
    $exit_code = &download(\@files, \@files2decompress);
    if ($exit_code == 1) {
        foreach (@files2decompress) {
            $exit_code = &decompress($_);
            last if ($exit_code != 1);
        }
    }
}
$ftp->quit() unless ($opt_show_version);

exit($exit_code);

# Connects to NCBI ftp server
sub connect_to_ftp
{
    my %ftp_opts;
    $ftp_opts{'Passive'} = 1 if $opt_passive;
    $ftp_opts{'Timeout'} = $opt_timeout if ($opt_timeout >= 0);
    $ftp_opts{'Debug'}   = 1 if ($opt_verbose > 1);
    my $ftp = Net::FTP->new(NCBI_FTP, %ftp_opts)
        or die "Failed to connect to " . NCBI_FTP . ": $!\n";
    $ftp->login(USER, PASSWORD) 
        or die "Failed to login to " . NCBI_FTP . ": $!\n";
    $ftp->cwd(BLAST_DB_DIR);
    $ftp->binary();
    print "Connected to NCBI\n" if $opt_verbose;
    return $ftp;
}

# Gets the list of available databases on NCBI FTP site
sub get_available_databases
{
    my @blast_db_files = $ftp->ls();
    my @retval = ();

    foreach (@blast_db_files) {
        next unless (/\.tar\.gz$/);
        push @retval, &extract_db_name($_);
    }

    # Sort and eliminate adjacent duplicates
    @retval = sort @retval;
    my $prev = "not equal to $retval[0]";
    return grep($_ ne $prev && ($prev = $_, 1), @retval);
}

# Obtains the list of files to download
sub get_files_to_download
{
    my @blast_db_files = $ftp->ls();
    my @retval = ();

    if ($opt_verbose > 2) {
        print "Found the following files on ftp site:\n";
        print "$_\n" for (@blast_db_files);
    }

    for my $requested_db (@ARGV) {
        for my $file (@blast_db_files) {
            next unless ($file =~ /\.tar\.gz$/);    
            if ($file =~ /^$requested_db\..*/) {
                push @retval, $file;
            }
        }
    }

    if ($opt_verbose) {
        for my $requested_db (@ARGV) {
            unless (grep(/$requested_db/, @retval)) {
                print STDERR "$requested_db not found, skipping.\n" 
            }
        }
    }

    return @retval;
}

# Download the requested files only if their checksum files are missing or if
# these (or the archives) are newer in the FTP site. Returns 0 if no files were
# downloaded, 1 if at least one file was downloaded (so that this can be the
# application's exit code)
sub download($$)
{
    my @requested_dbs = @ARGV;
    my @files2download = @{$_[0]};
    my $files2decompress = $_[1];
    my $retval = 0;

    for my $file (@files2download) {

        my $attempts = 0;   # Download attempts for this file
        if ($opt_verbose and &is_multivolume_db($file) and $file =~ /\.00\./) {
            my $db_name = &extract_db_name($file);
            my $nvol = &get_num_volumes($db_name,print!cbi16Chnceultivolume &get_num_volume &ge =~ /\ $fileame($fileee        &get_num_$fi-e($hge$=~ /\ $fileame($fileee       D&gat_num_$ract_db_name($file);
            my $nvol = &get_num_volumes($db_name,ps2do_num_volumes(\ $filt_ranknum_voluone file)Rne file wasf at s2do_num_volum ($fileeernum_volumes( _ranknummultivolume_dxH;=|ddptlumeasis($idb_name($fp     tluolume_        q           `!        r           h!    H Di|ddptlume!ile was2  tlume!pd (so that this can be the
# application's exit code)
sub download($$)
{
    my @requested_dbs = @AFGV;
    my @files2download = @{$_[0]};
    mypsfiles2dtc"mfsest = $_[1];
    my $retval = les2d     
   t_num_        iles2       
   t_num_        ileeasis($i_dbceSetmfsest = $PR@ifiist1   " 
 Tag`          >0 l" 
 J      S   SHHw Hng fss    ng[g  >0 l"     = @{_num_  l" 
 $<+ v t_i S  %le wH Hngnload($$)
{
    meeefrrdm_volum& = @AFGV;
    mmilugrenxfowre $!:sisn$io_bci__d " 
 T file  illlllllllloorrrrrrrrr********$$wi)Hw Hngugreny $retva  `!  " 
 Tag`tion's exit co          prev &&

 $)
{
       PR@ipeES1_T);
            mytmeeyfrrdmEvolum&R= @AFGV7
    mmilugtogume!ile nnfrrdmEvwqt c mypsfiR= @AFGV7
 it co  rrrrrrr****a @AFGV7
 it co  :Evolum  A!1iles2dtc"mfsest =$meeyaeu 'me!ile nnfrramEvw!1xi**a R= @AFGV7
 itst !il %le wH    if (nnfrramil %le**a _files);
 oni   "lesl  o  ?    esl  o oni  iR= "lesl:"?    esl T wmi  iR= "lesl:"?    esl T wmi  iR"leAFGV7lesfi-eleslverbose) {
  wnload($$)
y fss   x\ $filiss   @AFGV7
  wmi  iR iRT wmi etval = les2ram!  " 
 Ta(@ARGV) {
    w Hngugreny  = $PRad($$)
yi)Hfiliss   t co  rrrrrrr*s   x\ $fl&R= @AFGV7
    meb_nrrrrt w Hngugreny  =. l = lese**a _fil
    g wmnrnrrrrt w Hnis co  rrrrrrr*s e**a _f             *gjp($_ l
   w Hnis,= lese*ame!ile nnsnra:is,4GG  esl 0  =ni  iR= "les  es4GG  esl on" namn's ead) sli  4GG             ame!ileo  =noa:is,4GG  esl 0  =ni                         ilisstion's exit cmil %(  es4GG  esl   me!ile nles  NGG/cy$zoa:is,4GGeyaeu        }
     /-eleslveu        }('getSeqAlignt A  /cytysI        XG  esl  Kle nles   X!        K           `!        L        lveu        }('getSeqAlignf A  /gytysI       XG  esl  Kle nlesi  X!   x   K           `!        L        lveu   ()  `}t 
         !        X                 Y      \    !       PZ is       ́   ()  `}t 
   lesiel     PLH   ́   ()  `}t 
   lesiel     PLH `}   ́   ()  `}t 
 
    ()  `}t H
 &.G=UB  ()8caih 
    aih()8caih  `}t   Kle nlesi `}t 
 
    () `uHx  Kle nrimngi  
`}t <veu        eld.G  `}t Kle nrimngi  
`  KlKle nglt  Kle nslegss/tracb   nrimngileo  =noa:eu      name="term"            `s/tracbrrt w/t     =noa:eu           Kle rnum_volumes( _ratu  iR"leAFGVleolume nglt  8oluos  esl PtermG  u  iR/t W   =no                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          