Unzipping file in temp dir

[mthelm]

I'm trying to write a function that downloads a zip file and then extracts its contents but it is failing.

pub async fn qcew(year: u32) -> Result<()> {
    let tmp_dir = Builder::new().prefix("qcew").tempdir()?;
    let target = format!("https://data.bls.gov/cew/data/files/{}/csv/{}_qtrly_singlefile.zip", year, year);
    let response = reqwest::get(target).await?;

    let mut dest = {
        let fname = response
            .url()
            .path_segments()
            .and_then(|segments| segments.last())
            .and_then(|name| if name.is_empty() { None } else { Some(name) })
            .unwrap_or("tmp.bin");

        println!("Downloading '{}' ...", fname);
        let fname = tmp_dir.path().join(fname);
        println!("will be located under: '{:?}'", fname);
        (File::create(&fname)?, fname)
    };
    let content =  response.text().await?;
    copy(&mut content.as_bytes(), &mut dest.0)?;
    Command::new("tar")
        .args(["-xf", dest.1.to_str().unwrap(), "-C", "C:\\Users\\mthel\\Rust\\place-based"])
        .spawn()
        .expect("tar command failed");
    Ok(())
}

I can manually use tar at the command line to unzip the file just fine so there is nothing wrong with the file. The error message that is displayed is:

Error: Access is denied. (os error 5)

Any ideas??

[H2CO3]

It seems like you are on Windows. As far as I remember, on Windows, files follow the RWLock pattern by default: you aren't allowed to open a file for reading if it's already open for writing, too. However, you do open the file for writing (File::create(&fname)), and then you instruct tar to extract it. That's probably why you get the permission error. Try dropping the file handle in order to relinquish write access before untarring.

By the way, are you sure the error comes from tar? You should first check which part of your code returns an error. It might as well be File::create() or copy().

[mthelm]

I tried using the zip crate rather than the tar utility on my Windows machine and I get the same access denied error. I tried dropping the file handle as you suggested and then opening it and passing that to a new ZipArchive:

pub async fn qcew(year: u32) -> Result<()> {
    let tmp_dir = Builder::new().prefix("qcew").tempdir()?;
    let target = format!("https://data.bls.gov/cew/data/files/{}/csv/{}_qtrly_singlefile.zip", year, year);
    let response = reqwest::get(target).await?;

    let mut dest = {
        let fname = response
            .url()
            .path_segments()
            .and_then(|segments| segments.last())
            .and_then(|name| if name.is_empty() { None } else { Some(name) })
            .unwrap_or("tmp.bin");

        println!("Downloading {}...", fname);
        let fname = tmp_dir.path().join(fname);
        println!("will be located under: {:?}", fname);
        (File::create(&fname)?, fname)
    };

    let content = response.text().await?;
    copy(&mut content.as_bytes(), &mut dest.0)?;
    drop(dest.0);
    let f = File::open(dest.1).unwrap();
    let mut zip = zip::ZipArchive::new(f)?;
    zip.extract("C:\\Users\\mthel\\Rust\\place-based")?;

    Ok(())
}

this produces the error: Error: failed to fill whole buffer.

[chrisd]

Tbh, it's a bit hard to remotely diagnose the error without knowing which line is causing it or at least having a fully runnable example (i.e. that has a main function and shows all imports).

[mthelm]

Sorry about that - this should be fully reproduceable, assuming you have the tar utility on your machine.

use anyhow::Result;
use tokio::io::copy;
use tokio::fs::File;
use std::process::Command;
use reqwest;
use tempfile::Builder;

#[tokio::main]
async fn main() -> Result<()> {
    let tmp_dir = Builder::new().prefix("qcew").tempdir()?;
    let target = format!("https://data.bls.gov/cew/data/files/{}/csv/{}_qtrly_singlefile.zip", 2020, 2020);
    let response = reqwest::get(target).await?;

    let mut dest = {
        let fname = response
            .url()
            .path_segments()
            .and_then(|segments| segments.last())
            .and_then(|name| if name.is_empty() { None } else { Some(name) })
            .unwrap_or("tmp.bin");

        println!("Downloading {}...", fname);
        let fname = tmp_dir.path().join(fname);
        println!("will be located under: {:?}", fname);
        (File::create(&fname).await?, fname)
    };

    let content = response.text().await?;
    copy(&mut content.as_bytes(), &mut dest.0).await?;
    drop(dest.0);
    Command::new("tar")
        .args(["-xf", &format!("{}", dest.1.to_str().unwrap())])
        .spawn()
        .expect("tar command failed");
    tmp_dir.close()?;
    Ok(())
}

The error this produces is tar: Error opening archive: Failed to open 'C:\Users\mthel\AppData\Local\Temp\qcewdl0uij\2020_qtrly_singlefile.zip'

[mthelm]

So I finally got this working:

pub fn qcew(year: u32) -> Result<()> {
    let tmp_dir = Builder::new().prefix("qcew").tempdir()?;
    let target = format!("https://data.bls.gov/cew/data/files/{}/csv/{}_qtrly_singlefile.zip", year, year);
    let client = reqwest::blocking::Client::builder()
        .timeout(None)
        .build()?;
    let response = client.get(target).send()?;

    let mut dest = {
        let fname = response
            .url()
            .path_segments()
            .and_then(|segments| segments.last())
            .and_then(|name| if name.is_empty() { None } else { Some(name) })
            .unwrap_or("tmp.bin");

        println!("Downloading {}...", fname);
        let fname = tmp_dir.path().join(fname);
        println!("will be located under: {:?}", fname);
        (File::create(&fname)?, fname)
    };
    
    let mut content = Cursor::new(response.bytes()?);
    copy(&mut content, &mut dest.0)?;
    let f = File::open(dest.1)?;
    let mut zip = zip::ZipArchive::new(f)?;
    zip.extract(&tmp_dir)?;
    tmp_dir.close()?;
    Ok(())
}

[chrisd]

So the problem was using text() instead of bytes?

[mthelm]

I think it was that and the get request was timing out because there is a 30 second default timeout on the reqwest client. The file I'm downloading is quite large and takes longer than 30 seconds.