I'm aiming at creating an equivalent of Python's Fiona package (usage example), which reads geospatial formats and allows iterating over records (features). Fiona supports formats that store features as separate records (GeoPackage = Sqlite, GeoJson, GeoJsonSeq, Shapefile, Flatgeobuf), not in a topological way (OSM XML/PBF, TopoJson).
Here's my attempt at code similar to Serde. To start, I tried implementing the 2 different formats: GDAL (GPKG, GeoJson, Shapefile) and FlatGeobuf, as they're quite different.
What's the API?
Ideally, I'd like to do something like
#[derive(FromGeoFormat)]
struct MyStruct {
name: String,
geometry: Point,
num: i64,
}
// I'd prefer it this idiomatic way
// (probably will require macros inside `read_geofile`)
for feature in my_crate::read_geofile::<MyStruct>>(&path) {
let feature = feature?; // iterator can break
// do something with feature
}
What's under the hood so far
- Note 1: I'm still unable to write macros, this is on a TODO list. RN, I'll write the foreseeable expanded code by hand.
- Note 2: geo crates have structs that reference one another. I can't open them in constructor and save. I figured out that at least 2 layers are necessary. I'd like to keep them to the minimum though, so that implementing another format weren't tough.
trait FormatDriver<'a, A>
where A: AutoStruct {
fn can_open(path: &str) -> bool;
fn from_path(path: &str) -> Result<Self, Box<dyn Error>>
where Self: Sized;
// create a reader (ideally this should look like for loop, but not right now)
type FeatureReader: FeatureReader + Iterator<Item=Result<A, Box<dyn Error>>>;
fn iter(&'a mut self) -> Result<Self::FeatureReader, Box<dyn Error>>;
}
trait FeatureReader {
// forward the reader 1 record
fn next_feature(&mut self) -> Result<bool, Box<dyn Error>>; // Ok(false) -> end loop
// accessors sort of like in Serde
fn get_field_i32(&self, field_name: &str) -> Result<Option<i32>, Box<dyn Error>>;
fn get_field_i64(&self, field_name: &str) -> Result<Option<i64>, Box<dyn Error>>;
fn get_field_point(&self, field_name: &str) -> Result<Option<Point>, Box<dyn Error>>;
}
// this should have some code to work with the drivers, like `from_driver` below
trait AutoStruct {
fn generate<F: FeatureReader>(reader: &F) -> Result<Self, Box<dyn Error>> where Self: Sized;
}
// FORMAT DRIVER 1: GPKG (via GDAL)
struct GpkgDriver<'a, A> {
fi: OwnedFeatureIterator,
_p: PhantomData<&'a A>
}
const PATH_REGEXP:&str = r"^(?P<file_path>(?:.*/)?(?P<file_name>(?:.*/)?(?P<file_own_name>.*)\.(?P<extension>gpkg)))(?::(?P<layer_name>[a-z0-9_-]+))?$";
impl<'a, A: AutoStruct> FormatDriver<'a, A> for GpkgDriver<'a, A> {
type FeatureReader = GpkgLayer<'a, A>;
fn can_open(path: &str) -> bool {
let re = Regex::new(PATH_REGEXP).unwrap();
re.is_match(&path)
}
fn from_path(path: &str) -> Result<Self, Box<dyn Error>> {
let dataset = Dataset::open(path)?;
// TODO: choose layer from path expression or return error if can't choose
let layer = dataset.into_layer(0)?;
let fi = layer.owned_features();
Ok(Self { fi, _p: PhantomData })
}
fn iter(&'a mut self) -> Result<GpkgLayer<'a, A>, Box<dyn Error>> {
let fii = self.fi.into_iter();
Ok(GpkgLayer { fii, fields: vec![], feature: None, _p: PhantomData })
}
}
struct GpkgLayer<'a, A> {
fii: &'a mut OwnedFeatureIterator,
fields: Vec<String>,
feature: Option<GdalFeature<'a>>,
_p: PhantomData<&'a A>
}
impl<'a, A> FeatureReader for GpkgLayer<'a, A> {
fn next_feature(&mut self) -> Result<bool, Box<dyn Error>> {
if let Some(f) = self.fii.next() {
self.feature.replace(f);
Ok(true)
}
else { Ok(false) }
}
fn get_field_i32(&self, field_name: &str) -> Result<Option<i32>, Box<dyn Error>> {
match match match &self.feature {
Some(f) => f.field(field_name)?,
None => panic!("no feature but reading field")
} {
Some(v) => v,
None => return Ok(None),
} {
FieldValue::IntegerValue(v) => Ok(Some(v.into())),
FieldValue::Integer64Value(v) => Ok(Some(v.try_into()?)),
_ => panic!("wrong format")
}
}
fn get_field_i64(&self, field_name: &str) -> Result<Option<i64>, Box<dyn Error>> {
match match match &self.feature {
Some(f) => f.field(field_name)?,
None => panic!("no feature but reading field")
} {
Some(v) => v,
None => return Ok(None),
} {
FieldValue::IntegerValue(v) => Ok(Some(v.into())),
FieldValue::Integer64Value(v) => Ok(Some(v.try_into()?)),
_ => panic!("wrong format")
}
}
fn get_field_point(&self, _field_name: &str) -> Result<Option<Point>, Box<dyn Error>> {
match match &self.feature {
Some(f) => Some(f.geometry().to_geo()?),
None => panic!("no feature read yet"),
_ => None::<Geometry> // TODO: this is just to fix the non-exhaustive patterns
} {
Some(Geometry::Point(g)) => Ok(Some(g)),
// just to fix the return types/exhaustiveness
None => Ok(None),
_ => panic!("what have I just got?")
}
}
}
impl<'a, A> Iterator for GpkgLayer<'a, A>
where A: AutoStruct{
type Item = Result<A, Box<dyn Error>>;
fn next(&mut self) -> Option<Self::Item> {
todo!()
}
}
// FORMAT DRIVER 2: FGB (FlatGeobuf)
// this format wants &File as input,
// so I must either a) open the file outside, or b) have 2 structs
struct FgbDriver<'a, A> {
fp: File,
features: Option<FgbReader<'a, File, FeaturesSelectedSeek>>,
_p: PhantomData<A>
}
impl<'a, A: AutoStruct> FormatDriver<'a, A> for FgbDriver<'a, A> {
type FeatureReader = FgbFeatureReader<'a, A>;
fn can_open(path: &str) -> bool {
path.ends_with(".fgb")
}
fn from_path(path: &str) -> Result<Self, Box<dyn Error>> {
let fp = File::open(path)?;
Ok(Self { fp, features: None, _p: PhantomData })
}
fn iter(&'a mut self) -> Result<Self::FeatureReader, Box<dyn Error>> {
let features_selected = FgbReader::open(&mut self.fp)?.select_all()?;
Ok(Self::FeatureReader { features_selected, _p: PhantomData })
}
}
struct FgbFeatureReader<'a, A> {
features_selected: FgbReader<'a, File, FeaturesSelectedSeek>,
_p: PhantomData<A>
}
impl<'a, A> FeatureReader for FgbFeatureReader<'a, A> {
fn next_feature(&mut self) -> Result<bool, Box<dyn Error>> {
// getters should use self.features_selected.get() to get current feature
Ok(self.features_selected.next()?.is_some())
}
fn get_field_i32(&self, field_name: &str) -> Result<Option<i32>, Box<dyn Error>> {
let ft = self.features_selected.cur_feature();
Ok(Some(ft.property::<i32>(field_name)?))
}
fn get_field_i64(&self, field_name: &str) -> Result<Option<i64>, Box<dyn Error>> {
let ft = self.features_selected.cur_feature();
Ok(Some(ft.property::<i64>(field_name)?))
}
fn get_field_point(&self, _field_name: &str) -> Result<Option<Point>, Box<dyn Error>> {
let ft = self.features_selected.cur_feature();
match ft.to_geo()? {
Geometry::Point(p) => Ok(Some(p)),
_ => panic!("wrong geometry type!")
}
}
}
impl<'a, A> Iterator for FgbFeatureReader<'a, A>
where A: AutoStruct {
type Item = Result<A, Box<dyn Error>>;
fn next(&mut self) -> Option<Self::Item> {
match self.next_feature() {
Ok(true) => Some(A::generate(self)),
_ => None,
}
}
}
#[derive(Debug)]
struct MyStruct {
x: i64,
geometry: Point
}
impl AutoStruct for MyStruct {
fn generate<F: FeatureReader>(reader: &F) -> Result<Self, Box<dyn Error>> {
Ok(Self {
x: reader.get_field_i64("x")?.unwrap(),
geometry: reader.get_field_point("geometry")?.unwrap()
})
}
}
// there'll be a function that will walk down the list of formats and check which one can open the file
// then call MyStruct::from_driver.
fn main() -> Result<(), Box<dyn Error>> {
let p = vec![
"places.gpkg:cities",
"places.gpkg",
"places",
"saontehusa.gpkg",
"sanhutens.gpkg:snoahtu:gosat",
"asoneht.fgb",
"aosnetuh"
];
for i in p.iter() {
if GpkgDriver::<'_, MyStruct>::can_open(i) { println!("Gpkg can open {:?}", i); }
if FgbDriver::<'_, MyStruct>::can_open(i) { println!("Fgb can open {:?}", i); }
}
let mut fd: FgbDriver<MyStruct> = FgbDriver::from_path("local.fgb")?;
for feature in fd.iter()? {
println!("{:?}", feature);
}
//let fdi = fd.iter()?;
//while fdi.next_feature()? {
//}
Ok(())
}