Http request with redirects runs into issues (reqwest)

help
1

I am new with Rust and ran into an issue creating a simple http request:

use reqwest::Client;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue, ACCEPT, ACCEPT_LANGUAGE, REFERER, USER_AGENT};
use std::error::Error;

#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {

    // Create custom headers
    let mut headers = HeaderMap::new();
    
    // Add standard headers
    headers.insert(USER_AGENT, HeaderValue::from_static(
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
    ));
    headers.insert(ACCEPT, HeaderValue::from_static(
        "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
    ));
    headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static(
        "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"
    ));
    headers.insert(REFERER, HeaderValue::from_static(
        "https://www.nvidia.com/"
    ));
    
    // Add non-standard headers
    headers.insert(HeaderName::from_static("sec-ch-ua"), HeaderValue::from_static(
        "\"Not(A:Brand\";v=\"99\", \"Brave\";v=\"133\", \"Chromium\";v=\"133\""
    ));
    headers.insert(HeaderName::from_static("sec-ch-ua-mobile"), HeaderValue::from_static("?0"));
    headers.insert(HeaderName::from_static("sec-ch-ua-platform"), HeaderValue::from_static("\"Windows\""));
    headers.insert(HeaderName::from_static("sec-fetch-dest"), HeaderValue::from_static("document"));
    headers.insert(HeaderName::from_static("sec-fetch-mode"), HeaderValue::from_static("navigate"));
    headers.insert(HeaderName::from_static("sec-fetch-site"), HeaderValue::from_static("same-origin"));
    headers.insert(HeaderName::from_static("sec-gpc"), HeaderValue::from_static("1"));
      
    // Create a client with default headers
    let client = Client::builder()
        .default_headers(headers)
        .timeout(std::time::Duration::from_secs(30))
        .build()?;

    // Target URL
    let url = "https://store.nvidia.com/de-de/geforce/store";
    
    // Make the request
    let response = client
        .get(url)
        .send()
        .await
        .map_err(|e| {
            eprintln!("Request failed: {}", e);
            e
        })?;
    
    // Print results
    println!("[INFO] Response: {} bytes", response.text().await?.len());
    
    Ok(())
}

This produces an error:

Request failed: error sending request for url (https://store.nvidia.com/de-de/geforce/store): connection closed before message completed
Error: reqwest::Error { kind: Request, url: Url { scheme: "https", cannot_be_a_base: false, username: "", password: None, host: Some(Domain("store.nvidia.com")), port: None, path: "/de-de/geforce/store", query: None, fragment: None }, source: hyper::Error(IncompleteMessage) }
error: process didn't exit successfully

The expected behavior would be 200 OK and then an automatic redirect to another domain at nvidia's website.

I am using the same headers to simulate browser behavior that I used for another test python script which is successful.

Do you see where I am making a mistake (as far as I understand the default is allowing 10 redirects so that at least should not be the issue). Thanks!

For context, here the successful response using python:

[INFO] Status Code: 200
[INFO] Final URL: https://marketplace.nvidia.com/de-de/consumer/
[INFO] Response: 60110 bytes

[INFO] Response Headers:
x-amz-id-2: LZqMXgNRLDRgnOFefyIfmX73nkKXHE2ZWVTdi+EduTsVyl23q6aFFDBS8rla6luUsMlebN1KsqwftplimlGM/IUXUUqWJCjg
x-amz-request-id: X7ZQ3906SD5NWXJN
Last-Modified: Fri, 28 Feb 2025 03:15:22 GMT
ETag: "c1be5aa5d38e742afe14fee1aa7ed004"
x-amz-server-side-encryption: AES256
x-amz-version-id: M3GyqtzvnTQA0W4h_5NK9KW2N5.VW6Vj
Accept-Ranges: bytes
Content-Type: text/html
X-Akamai-Transformed: 9 - 0 pmb=mTOE,2
Vary: Accept-Encoding
Content-Encoding: gzip
Expires: Mon, 03 Mar 2025 20:33:45 GMT
Cache-Control: max-age=0, no-cache, no-store
Pragma: no-cache
Date: Mon, 03 Mar 2025 20:33:45 GMT
Content-Length: 8291
Connection: keep-alive
Strict-Transport-Security: max-age=86400 ; includeSubDomains ; preload
Content-Security-Policy: frame-ancestors 'self' https://resources.nvidia.com;
x-queueit-connector: akamai
Set-Cookie: ak_bmsc=59D02B1079DAFAADEF18287246929698~000000000000000000000000000000~YAAQyh0QAiaMWlWVAQAA4vC2XRpTr4E0C8W539RPDhxnYyTSzRbeev5rRyp51Cs6+9yBTJoSIyN8CZMpDNmosVUyKH37GvbkVEOFZeNADMqfOJ2OiZv/GP5h34UZhviTgWbsLTElurvHJ919+hrcQaggORFI0h9oXlI7NVfeH8EdVd3kgT2E/VtR8FSbMhGC1gDwK6EZ/C6WgP7SpqzHUULoygKvwW42qvm7vUVp0RT4yI8mPFgn+aXB/SPczYgsEoDpBnIF+aOPCD6BZlJI8nrHiNvo2MUKe3pHmZB3GLwV1pwxCzE04vmiM0EEnzSrLA5SvZRz2vxN0GmH+dywSQ3krNxeKxKiHI8gigQDY00v8ZHJvu4DXsyIfhSU; Domain=.nvidia.com; Path=/; Expires=Mon, 03 Mar 2025 22:33:45 GMT; Max-Age=7200, bm_mi=6D106DAA0888DF360C662E7E4373CD73~YAAQyh0QAieMWlWVAQAA4vC2XRop9wHnf/Fz9tac3UDQSVMmIqvq7Eczqpa6HfSHSsL7v0nm4t53ihrTBeSXyEWSPBIy/KfqZTMHBA1LCYfK/TodIRj2Og2siujNwdfoToN3nCnMvpewWrRAA9q0JNQGl0kiN9ClUHDsSH919ZJn0Tlesp9wjluyfncZNQHQKNQSBjvIiwWrJQuj++vMeHMqgK/dRYfod9IPixzw5lywvvh8V+8q5MzwLGZIibvRpN6CcV1OfcXMm8zvNxlvQ08+JssgNiAqJPTfykCD4r7rTsY5s1w5IfomieKvetkkfKu1FcZuMcsk1qIRMQ==~1; Domain=.nvidia.com; Path=/; Expires=Mon, 03 Mar 2025 20:33:45 GMT; Max-Age=0; Secure

[INFO] Redirect History:
Redirect 1: 301 - https://store.nvidia.com/de-de/geforce/store
Redirect 2: 301 - https://store.nvidia.com/de-de/consumer
Redirect 3: 301 - https://store.nvidia.com/de-de/consumer/
Redirect 4: 301 - https://marketplace.nvidia.com/de-de/consumer

2

As an update, using attohttpc as an alternative seems to work:

use std::error::Error;

fn main() -> Result<(), Box<dyn Error>> {
    // Target URL
    let url = "https://store.nvidia.com/de-de/geforce/store";
    
    println!("[INFO] Sending request to {}", url);
    
    // Create a request with headers
    let response = attohttpc::get(url)
        .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
        .header("sec-fetch-dest", "document")
        .header("sec-fetch-mode", "navigate")
        .header("sec-fetch-site", "none")
        .timeout(std::time::Duration::from_secs(30))
        .follow_redirects(true)
        .send()?;
    
    // Print response details
    println!("[INFO] Status Code: {}", response.status());
    println!("[INFO] Final URL: {}", response.url());
    
    // Print response headers
    println!("\n[INFO] Response Headers:");
    for (name, value) in response.headers() {
        println!("  {}: {}", name, value.to_str().unwrap_or("Invalid UTF-8"));
    }
    
    // Get the response body
    let text = response.text()?;
    println!("[INFO] Response: {} bytes", text.len());
    
    Ok(())
}

Strange, would be insightful which setting I have to be explicit about with reqwest to achieve the same result.

3

From testing with curl, the host is behind Akamai and it requires HTTP/2. (HTTP/1.1 hangs until the connection times out).

Running your code, I get the timeout. reqwest is defaulting to HTTP/1.1! And I can't force HTTP/2 because I'm hitting this bug: Setting HTTP version is overridden by ALPN · Issue #2116 · seanmonstar/reqwest · GitHub

2 Likes