Wrong encoding of response

I am trying to get the data from http get and once I receive the data I want to log the same before returning a response to the user but somehow when I decode the response to byte it gets corrupted.

Sample line of code for decoding response
let (resp_parts, res_body) = backend_res.into_parts();
let body_str = hyper::body::to_bytes(res_body).await.unwrap();
let body_string = String::from_utf8(body_str.to_vec()).unwrap();

use std::borrow::Borrow;
use std::collections::HashMap;
use std::convert::Infallible;
use std::error::Error;
use std::hash::Hasher;
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime};

use hyper::{Body, Client, Request, Response, Server};
use hyper::{body::HttpBody as _, Uri};
use hyper::{Method, StatusCode};
use hyper::client::HttpConnector;
use hyper::server::conn::Http;
use hyper::service::{make_service_fn, service_fn};
use hyper_tls::HttpsConnector;
use lazy_static::lazy_static;
use xxhash_rust::xxh3::xxh3_64;
async fn logResponse(req: Request<Body>) -> Result<Response<Body>, Infallible> {
    let mut backend_req = Request::builder()
                    .method(req.method())
                    .uri(format!("https://blockstream.info/api/blocks/0"))
                    .version(req.version());
    for (header_name, header_value) in req.headers() {
        backend_req = backend_req.header(header_name, header_value);
    }
    let https = hyper_tls::HttpsConnector::new();
    let client = Client::builder().build::<_, hyper::Body>(https);
    let body = backend_req.body(Body::empty()).unwrap();

    let backend_res = client.request(body).await.unwrap();

    let mut res = Response::builder()
        .status(backend_res.status())
        .version(backend_res.version());

    for (header_name, header_value) in backend_res.headers() {
        res = res.header(header_name, header_value);
    }

    /*
    Comment from next line to Ok(response) and
    uncomment Ok(res.body(backend_res.into_body()).unwrap()) to get working code
    */
    let (resp_parts, res_body) = backend_res.into_parts();
    /*In below line response to byte array is giving wrong byte array*/
    let body_str = hyper::body::to_bytes(res_body).await.unwrap();
    let body_string = String::from_utf8(body_str.to_vec()).unwrap();
    println!("*************{}",body_string);
    let response : Response<Body> = Response::from_parts(resp_parts, body_string.clone().into());
    Ok(response)

    /*If you uncomment below code you can see correct result is given*/
     //Ok(res.body(backend_res.into_body()).unwrap())
}

#[tokio::main]
async fn main() {
    // Set up a new HTTP server
    let addr = ([0, 0, 0, 0], 3000).into();
    let make_svc = make_service_fn(|_conn| async {
        Ok::<_, Infallible>(service_fn(logResponse))
    });
    let server = Server::bind(&addr).serve(make_svc);

    // Start the server
    println!("Listening on http://{}", addr);
    if let Err(e) = server.await {
        eprintln!("server error: {}", e);
    }
}

Cargo.toml

[package]
name = "minigrep"
version = "0.1.0"
edition = "2021"

[rust]
debug = true
debuginfo-level = 2

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
hyper = "0.13.0-alpha.4"
hyper-tls = "0.4.1"
tokio = { version = "0.2.22", features = ["full"] }
pretty_env_logger = "0.3.1"
xxhash-rust = { version = "0.8.6", features = ["xxh3", "const_xxh3"] }
lazy_static = "1.4.0"

Can you give an example request that triggers the error? I tried with a simple curl -v http://localhost:3000 and the logged string is the same as the response body.

1 Like

Cross-post on SO:

Please link cross-posts in your question to avoid duplicated effort by the community.

1 Like

I got the error

thread 'tokio-runtime-worker' panicked at 'called Result::unwrap() on an Err value: FromUtf8Error { bytes: [31, 139, 8, 0, 0, 0, 0, 0, 4, 3, 85, 144, 203, 106, 196, 48, 12, 69, 255, 69, 235, 44, 108, 43, 145, 149, 252, 74, 41, 131, 236, 200, 141, 153, 76, 60, 228, 49, 211, 7, 253, 247, 58, 80, 10, 213, 78, 151, 35, 233, 160, 151, 47, 200, 35, 12, 96, 254, 202, 246, 35, 17, 247, 209, 112, 39, 106, 169, 99, 180, 218, 99, 155, 146, 39, 20, 109, 73, 156, 80, 180, 222, 5, 76, 54, 144, 17, 142, 234, 40, 65, 3, 147, 230, 183, 105, 135, 193, 52, 240, 208, 117, 203, 101, 129, 193, 54, 176, 231, 155, 110, 187, 220, 238, 181, 115, 104, 141, 161, 206, 116, 53, 126, 191, 196, 114, 44, 149, 175, 204, 150, 63, 21, 6, 199, 53, 127, 254, 110, 97, 75, 13, 220, 116, 189, 206, 122, 89, 75, 169, 28, 180, 210, 169, 213, 54, 136, 4, 238, 19, 10, 186, 206, 178, 48, 71, 180, 33, 178, 79, 100, 185, 74, 146, 71, 117, 49, 122, 47, 193, 89, 231, 131, 151, 52, 234, 40, 136, 161, 58, 222, 87, 125, 228, 114, 108, 97, 46, 241, 58, 201, 54, 193, 176, 28, 243, 124, 94, 26, 179, 44, 167, 235, 127, 205, 165, 44, 241, 84, 51, 140, 14, 235, 91, 176, 129, 144, 247, 13, 134, 150, 137, 76, 235, 251, 190, 129, 49, 167, 148, 227, 49, 239, 31, 117, 246, 251, 245, 7, 137, 32, 137, 109, 83, 1, 0, 0], error: Utf8Error { valid_up_to: 1, error_len: Some(1) } }', src/main.rs:47:60
stack backtrace:

The blockstream endpoint encodes its JSON response body with gzip to compress the body's size (see content-encoding header). The hyper client does not handle content-encoding automatically, so you must decompress the body by hand if you want to parse it into a valid UTF-8 string. Here is an example with your bytes and the flate2 crate:

use flate2::read::GzDecoder;

use std::io::Read;

fn main() {
    let bytes = [
        31, 139, 8, 0, 0, 0, 0, 0, 4, 3, 85, 144, 203, 106, 196, 48, 12, 69, 255, 69, 235, 44, 108,
        43, 145, 149, 252, 74, 41, 131, 236, 200, 141, 153, 76, 60, 228, 49, 211, 7, 253, 247, 58,
        80, 10, 213, 78, 151, 35, 233, 160, 151, 47, 200, 35, 12, 96, 254, 202, 246, 35, 17, 247,
        209, 112, 39, 106, 169, 99, 180, 218, 99, 155, 146, 39, 20, 109, 73, 156, 80, 180, 222, 5,
        76, 54, 144, 17, 142, 234, 40, 65, 3, 147, 230, 183, 105, 135, 193, 52, 240, 208, 117, 203,
        101, 129, 193, 54, 176, 231, 155, 110, 187, 220, 238, 181, 115, 104, 141, 161, 206, 116,
        53, 126, 191, 196, 114, 44, 149, 175, 204, 150, 63, 21, 6, 199, 53, 127, 254, 110, 97, 75,
        13, 220, 116, 189, 206, 122, 89, 75, 169, 28, 180, 210, 169, 213, 54, 136, 4, 238, 19, 10,
        186, 206, 178, 48, 71, 180, 33, 178, 79, 100, 185, 74, 146, 71, 117, 49, 122, 47, 193, 89,
        231, 131, 151, 52, 234, 40, 136, 161, 58, 222, 87, 125, 228, 114, 108, 97, 46, 241, 58,
        201, 54, 193, 176, 28, 243, 124, 94, 26, 179, 44, 167, 235, 127, 205, 165, 44, 241, 84, 51,
        140, 14, 235, 91, 176, 129, 144, 247, 13, 134, 150, 137, 76, 235, 251, 190, 129, 49, 167,
        148, 227, 49, 239, 31, 117, 246, 251, 245, 7, 137, 32, 137, 109, 83, 1, 0, 0,
    ];

    let mut gz = GzDecoder::new(&bytes[..]);
    let mut s = String::new();
    gz.read_to_string(&mut s).unwrap();
    
    println!("{s}");
}

Playground.

1 Like

@jofas thanks for ur feedback its really helpful

This topic was automatically closed 90 days after the last reply. We invite you to open a new topic if you have further questions or comments.