How to Read A File Concurrently

In Golang I can use this but for Rust I can only think reading file to a Vec and using rayon for iterating on list in parallel is there any more effictent way?

As far as I can tell, this is equivalent to your go code.

use crossbeam::channel::{bounded, Receiver};
use std::fs::File;
use std::io::{BufRead, BufReader};

const CONCURRENCY: usize = 100;

fn main() -> std::io::Result<()> {
    let (send, recv) = bounded(1024);
    
    let mut join_handles = Vec::with_capacity(CONCURRENCY);
    
    for _ in 0..CONCURRENCY {
        let recv = recv.clone();
        let handle = std::thread::spawn(move || start_working(recv));
        join_handles.push(handle);
    }
    drop(recv);
    
    let file = File::open("/path/to/csv")?;
    for line in BufReader::new(file).lines() {
        send.send(line?).unwrap();
    }
    drop(send);
    
    for handle in join_handles {
        handle.join().unwrap();
    }
    
    Ok(())
}

fn start_working(recv: Receiver<String>) {
    while let Ok(line) = recv.recv() {
        println!("Got line: {}", line);
    }
}

playground

1 Like