[BUG] - Active mode commands timeout

[rkimoakbioinformatics]

Description

ftp.ncbi.nih.gov uses active mode. list command after login times out.

Steps to reproduce

use suppaftp::FtpStream;

fn main() {
    let parent_url = "ftp.ncbi.nlm.nih.gov:21";
    let ftp_stream = FtpStream::connect(parent_url).unwrap();
    let mut ftp_stream = ftp_stream.active_mode(std::time::Duration::new(60, 0));
    ftp_stream.login("anonymous", "").unwrap();
    ftp_stream.cwd("/snp/latest_release/VCF").unwrap();
    let filenames_r = ftp_stream.list(None);
    println!("@ filenames r={:?}", filenames_r);
}

Expected behaviour

filenames_r being a Vec of filenames

Environment

• OS: MacOS on M3 Max
• Architecture: Apple Silicon
• Rust version: 1.75.0
• library version: 5.3.0
• Protocol used: No TLS
• Remote server version and name: N/A

Additional information

N/A

[veeso]

Mhm, I have tried with the suppaftp-cli binary and it connects and works fine in passive mode.

I have an active mode failure, when listing files:

 error: Invalid response: [500] 500 Illegal PORT command

Maybe your timeout is caused by the failure to bind on the local address?

Try to enable trace log and check if it prints some more information.

[rkimoakbioinformatics]

Thanks. In my test with suppaftp-cli, I get the following.

>suppaftp ftp.ncbi.nih.gov:21
>> login
Username:

Password:

OK
>> list
LIST error: Connection error: Operation timed out (os error 60)
>>

The following Python script works with the same host and the same local machine.

import ftplib

conn=ftplib.FTP(host="ftp.ncbi.nih.gov", user="anonymous", passwd="")
filenames = conn.nlst()
print(filenames)

Output

['bioproject', 'hmm', 'refseq', 'tpa', 'fa2htgs', 'pub', 'entrez', 'genbank', 'ncbi-asn1', 'gene', 'repository', 'cn3d', 'toolbox', 'sequin', 'blast', 'README.ftp', 'eqtl', 'seqc', 'sra', 'robots.txt', 'giab', 'nist-immsa', 'sky-cgh', 'tech-reports', 'epigenomics', 'pubmed', 'snp', 'pathogen', 'mmdb', 'SampleData', 'bigwig', 'favicon.ico', 'biosample', 'hapmap', '1000genomes', 'variation', 'geo', 'cgap', 'dbgap', 'diffexpIR-notebook', 'pubchem', 'asn1-converters', 'genomes', '1GB', '10GB', '100GB', '5GB', '50GB', 'osiris', 'ReferenceSamples', 'refsam', 'rapt', 'fufuter.html', 'comparative-genome-viewer']

Since the same host and the local machine were tested, it seems that cuppa-ftp and ftplib do something different?

[veeso]

at this point it times out even in passive mode, so the problem is probably unrelated to active mode.
Do other commands work? Like cwd, pwd...? Try also to retrieve a file

[rkimoakbioinformatics]

With cuppa-ftp, cwd and pwd worked. retr timed out, with and without active_mode.

• Code (without active_mode)

fn main() -> suppaftp::FtpResult<()> {
    let parent_url = "ftp.ncbi.nih.gov:21";
    let mut ftp_stream = suppaftp::FtpStream::connect(parent_url).unwrap();
    ftp_stream.login("anonymous", "").unwrap();
    ftp_stream.cwd("/snp/latest_release").unwrap();
    println!("Current directory is {}.", ftp_stream.pwd().unwrap());
    let mut buf: Vec<u8> = Vec::new();
    let r = ftp_stream.retr("release_notes.txt", |stream| {
        println!("stream={:#?}", stream.read_to_end(&mut buf));
        Ok(())
    }).map_err(|e| {
        e
    });
    println!("r={:#?}", r);
    let _ = ftp_stream.quit();
    Ok(())
}

• Result

Current directory is /snp/.redesign/latest_release.
r=Err(
    ConnectionError(
        Os {
            code: 60,
            kind: TimedOut,
            message: "Operation timed out",
        },
    ),
)

• Code (with active_mode)

fn main() -> suppaftp::FtpResult<()> {
    let parent_url = "ftp.ncbi.nih.gov:21";
    let ftp_stream = suppaftp::FtpStream::connect(parent_url).unwrap();
    let mut ftp_stream = ftp_stream.active_mode(std::time::Duration::new(60, 0));
    ftp_stream.login("anonymous", "").unwrap();
    ftp_stream.cwd("/snp/latest_release").unwrap();
    println!("Current directory is {}.", ftp_stream.pwd().unwrap());
    let mut buf: Vec<u8> = Vec::new();
    let r = ftp_stream.retr("release_notes.txt", |stream| {
        println!("stream={:#?}", stream.read_to_end(&mut buf));
        Ok(())
    }).map_err(|e| {
        e
    });
    println!("r={:#?}", r);
    let _ = ftp_stream.quit();
    Ok(())
}

• Result

Current directory is /snp/.redesign/latest_release.
r=Err(
    ConnectionError(
        Os {
            code: 60,
            kind: TimedOut,
            message: "Operation timed out",
        },
    ),
)

list got operation not permitted without active_mode, and timed out with active_mode.

• Code (without active_mode)

fn main() -> suppaftp::FtpResult<()> {
    let parent_url = "ftp.ncbi.nih.gov:21";
    let mut ftp_stream = suppaftp::FtpStream::connect(parent_url).unwrap();
    ftp_stream.login("anonymous", "").unwrap();
    ftp_stream.cwd("/snp/latest_release").unwrap();
    println!("Current directory is {}.", ftp_stream.pwd().unwrap());
    let r = ftp_stream.list(None);
    match r {
        Ok(_) => {
            println!("r={:#?}", r);
        },
        Err(suppaftp::FtpError::UnexpectedResponse(v)) => {
            let msg = std::str::from_utf8(&v.body).unwrap();
            println!("Error: {}", msg);
        },
        _ => {
            println!("r={:#?}", r);
        }
    }
    let _ = ftp_stream.quit();
    Ok(())
}

• Result

Current directory is /snp/.redesign/latest_release.
Error: 501 PASV: Operation not permitted

• Code (with active_mode)

fn main() -> suppaftp::FtpResult<()> {
    let parent_url = "ftp.ncbi.nih.gov:21";
    let ftp_stream = suppaftp::FtpStream::connect(parent_url).unwrap();
    let mut ftp_stream = ftp_stream.active_mode(std::time::Duration::new(60, 0));
    ftp_stream.login("anonymous", "").unwrap();
    ftp_stream.cwd("/snp/latest_release").unwrap();
    println!("Current directory is {}.", ftp_stream.pwd().unwrap());
    let r = ftp_stream.list(None);
    match r {
        Ok(_) => {
            println!("r={:#?}", r);
        },
        Err(suppaftp::FtpError::UnexpectedResponse(v)) => {
            let msg = std::str::from_utf8(&v.body).unwrap();
            println!("Error: {}", msg);
        },
        _ => {
            println!("r={:#?}", r);
        }
    }
    let _ = ftp_stream.quit();
    Ok(())
}

• Result

Current directory is /snp/.redesign/latest_release.
r=Err(
    ConnectionError(
        Os {
            code: 60,
            kind: TimedOut,
            message: "Operation timed out",
        },
    ),
)

A Python script with which cwd, pwd, list, and retr worked is below.

import ftplib

conn=ftplib.FTP(host="ftp.ncbi.nih.gov", user="anonymous", passwd="")
conn.cwd("/snp/latest_release")
print(f"pwd={conn.pwd()}")
filenames = conn.nlst()
print(f"files={filenames}")
lines = conn.retrlines("RETR release_notes.txt", store_retr_line)
print(f"lines={lines}")

This specific FTP server has a readme: https://ftp.ncbi.nlm.nih.gov/README.ftp.
I tested with another FTP site and list worked.

• Code

fn main() -> suppaftp::FtpResult<()> {
    let parent_url = "ftp.cs.brown.edu:21";
    let mut ftp_stream = suppaftp::FtpStream::connect(parent_url).unwrap();
    ftp_stream.login("anonymous", "").unwrap();
    ftp_stream.cwd("/").unwrap();
    println!("Current directory is {}.", ftp_stream.pwd().unwrap());
    let r = ftp_stream.list(None);
    match r {
        Ok(_) => {
            println!("r={:#?}", r);
        },
        Err(suppaftp::FtpError::UnexpectedResponse(v)) => {
            let msg = std::str::from_utf8(&v.body).unwrap();
            println!("Error: {}", msg);
        },
        _ => {
            println!("r={:#?}", r);
        }
    }
    let _ = ftp_stream.quit();
    Ok(())
}

• Result

Current directory is /.
r=Ok(
    [
        "drwxrwxrwt    2 0          0            11626752 Jan 25 09:54 incoming",
        "drwxr-xr-x   33 0          2                2323 Aug 13  2013 pub",
        "drwxr-xr-x   52 0          2                1087 Feb 10  2016 u",
    ],
)