Downloading Files With Selenium

Selenium in an indispensable tool but it cannot download files. It gives you control of the DOM but cannot interact with the native dialogs the browser coughs up when prompting for a file download.

The work-around is to use Apache HTTP Client to download the target file by loading it up with cookies from Selenium. First, navigate to the SIT and do whatever is necessary to establish a session, then grab the cookies from Selenium and push them into HTTP Client’s cookie store, and perform an HTTP GET to fetch the file.

public static final int TIMEOUT = 10000;

WebDriver driver;

// Do Selenium stuff...

ProductsPage products = this.home.navigateProducts();

// Set up the HTTP client to emulate a browser.
// Use cookies and follow 302 redirects
// Set a reasonable timeout
RequestConfig requestCfg = RequestConfig.custom()
.setConnectTimeout( TIMEOUT )
.setSocketTimeout( TIMEOUT )
.setConnectionRequestTimeout( TIMEOUT )
.build();

BasicCookieStore cookieStore = new BasicCookieStore();

CloseableHttpClient httpclient = HttpClients.custom()
.setDefaultCookieStore(cookieStore)
.setRedirectStrategy( new LaxRedirectStrategy() )
.setDefaultRequestConfig( requestCfg )
.build();

// Save state
HttpClientContext context = HttpClientContext.create();

// Duplicate cookies from Selenium in HTTP Client
Set seCookies = this.driver.manage().getCookies();

for( Cookie seCookie : seCookies ) {

System.out.println( "Converting cookie " + seCookie.getName() );

BasicClientCookie dupCookie =
new BasicClientCookie(seCookie.getName(), seCookie.getValue());

dupCookie.setDomain( seCookie.getDomain());
dupCookie.setPath( seCookie.getPath());
dupCookie.setSecure(seCookie.isSecure());
dupCookie.setExpiryDate( seCookie.getExpiry());

cookieStore.addCookie(dupCookie);
}

// Download the file
FileResponseHandler fileHandler = new FileResponseHandler( "out.csv" );

HttpGet httpget = new HttpGet( "https://sit/foobar/out.csv" );

File csv = httpclient.execute( httpget, f, context );

System.out.println( "Downloaded to " + csv.getAbsolutePath() );

The file response handler that writes the file from the HTTP response is described thusly:

public class FileResponseHandler implements ResponseHandler {

protected String filename;

public FileResponseHandler( String filename ) {

if( filename == null || filename.isEmpty() ) {
throw new IllegalArgumentException( "Filename is null or empty." );
}

this.filename = filename;
}

public File handleResponse( HttpResponse response ) throws IOException {

System.out.println( "File download response: " +
response.getStatusLine().getStatusCode() );

if( 200 != response.getStatusLine().getStatusCode() ) {
throw new IllegalStateException( "Bad response code for file fetch " +
response.getStatusLine().getStatusCode() );
}

File outFile = new File( this.filename );
FileOutputStream out = new FileOutputStream( outFile );

HttpEntity entity = response.getEntity();
BufferedInputStream in = new BufferedInputStream(entity.getContent());

byte[] buff = new byte[ 4096 ];
int read = 0;
int total = 0;
int oldTotal = 0;

while((read = in.read( buff )) != -1 ) {
out.write( buff,  0,  read );
total += read;

// Output every 1MB
if( total - oldTotal > 1024000) {
System.out.print( "." );
oldTotal = total;
}
}
System.out.println();
out.flush();
out.close();

System.out.println( "Wrote " + total + " bytes to " + this.filename );
return outFile;
}
}

A similar approach can be found here.