generic to_snf

Makes map tool work for external libs. It works if you have problems merge it and fix it later.

Reviewed-on: #42
Reviewed-by: Rhys Lloyd <quaternions@noreply@itzana.me>
Co-authored-by: itzaname <me@sliving.io>
Co-committed-by: itzaname <me@sliving.io>

Cargo.toml

@@ -37,7 +37,7 @@ unused_qualifications = "warn"
 unexpected_cfgs = "warn"
 
 [workspace.dependencies]
-glam = "0.31.0"
+glam = "0.32.0"
 
 # engine
 strafesnet_graphics = { path = "engine/graphics", registry = "strafesnet" }
@@ -49,8 +49,8 @@ strafesnet_settings = { path = "engine/settings", registry = "strafesnet" }
 fixed_wide = { version = "0.2.2", path = "lib/fixed_wide", registry = "strafesnet" }
 linear_ops = { version = "0.1.1", path = "lib/linear_ops", registry = "strafesnet" }
 ratio_ops = { version = "0.1.0", path = "lib/ratio_ops", registry = "strafesnet" }
-strafesnet_bsp_loader = { path = "lib/bsp_loader", registry = "strafesnet" }
-strafesnet_common = { version = "0.8.2", path = "lib/common", registry = "strafesnet" }
-strafesnet_deferred_loader = { version = "0.5.1", path = "lib/deferred_loader", registry = "strafesnet" }
-strafesnet_rbx_loader = { path = "lib/rbx_loader", registry = "strafesnet" }
+strafesnet_bsp_loader = { version = "0.4.0", path = "lib/bsp_loader", registry = "strafesnet" }
+strafesnet_common = { version = "0.8.6", path = "lib/common", registry = "strafesnet" }
+strafesnet_deferred_loader = { version = "0.6.0", path = "lib/deferred_loader", registry = "strafesnet" }
+strafesnet_rbx_loader = { version = "0.8.0", path = "lib/rbx_loader", registry = "strafesnet" }
 strafesnet_snf = { version = "0.3.2", path = "lib/snf", registry = "strafesnet" }

engine/graphics/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "strafesnet_graphics"
-version = "0.0.2"
+version = "0.0.4"
 edition = "2024"
 
 [dependencies]

engine/graphics/src/graphics.rs

@@ -5,10 +5,6 @@ use strafesnet_common::model::{self, ColorId, NormalId, PolygonIter, PositionId,
 use wgpu::{util::DeviceExt,AstcBlock,AstcChannel};
 use crate::model::{self as model_graphics,IndexedGraphicsMeshOwnedRenderConfig,IndexedGraphicsMeshOwnedRenderConfigId,GraphicsMeshOwnedRenderConfig,GraphicsModelColor4,GraphicsModelOwned,GraphicsVertex};
 
-pub fn required_limits()->wgpu::Limits{
-	wgpu::Limits::default()
-}
-
 struct Indices{
 	count:u32,
 	buf:wgpu::Buffer,
@@ -468,7 +464,7 @@ impl GraphicsState{
 		//.into_iter() the modeldata vec so entities can be /moved/ to models.entities
 		let mut model_count=0;
 		let mut instance_count=0;
-		let uniform_buffer_binding_size=required_limits().max_uniform_buffer_binding_size as usize;
+		let uniform_buffer_binding_size=crate::setup::required_limits().max_uniform_buffer_binding_size as usize;
 		let chunk_size=uniform_buffer_binding_size/MODEL_BUFFER_SIZE_BYTES;
 		self.models.reserve(models.len());
 		for model in models.into_iter(){

engine/graphics/src/setup.rs

@@ -5,6 +5,9 @@ fn optional_features()->wgpu::Features{
 fn required_features()->wgpu::Features{
 	wgpu::Features::TEXTURE_COMPRESSION_BC
 }
+pub fn required_limits()->wgpu::Limits{
+	wgpu::Limits::defaults()
+}
 fn required_downlevel_capabilities()->wgpu::DownlevelCapabilities{
 	wgpu::DownlevelCapabilities{
 		flags:wgpu::DownlevelFlags::empty(),
@@ -56,20 +59,17 @@ pub mod step3{
 				wgpu::DeviceType::Other|wgpu::DeviceType::Cpu=>1,
 			})?;
 
-		let adapter_info=adapter.get_info();
-		println!("Using {} ({:?})", adapter_info.name, adapter_info.backend);
-
 		Some(adapter)
 	}
 }
 
 pub mod step4{
-	pub async fn request_device(adapter:&wgpu::Adapter)->(wgpu::Device,wgpu::Queue){
+	pub async fn request_device(adapter:&wgpu::Adapter)->Result<(wgpu::Device,wgpu::Queue),wgpu::RequestDeviceError>{
 		let optional_features=super::optional_features();
 		let required_features=super::required_features();
 
 		// Make sure we use the texture resolution limits from the adapter, so we can support images the size of the surface.
-		let needed_limits=crate::graphics::required_limits().using_resolution(adapter.limits());
+		let needed_limits=super::required_limits().using_resolution(adapter.limits());
 
 		let (device, queue)=adapter
 			.request_device(
@@ -81,32 +81,38 @@ pub mod step4{
 					trace:wgpu::Trace::Off,
 					experimental_features:wgpu::ExperimentalFeatures::disabled(),
 				},
-			).await
-			.expect("Unable to find a suitable GPU adapter!");
+			).await?;
 
-		(
+		Ok((
 			device,
 			queue,
-		)
+		))
 	}
 }
 
 pub mod step5{
+	#[derive(Debug)]
+	pub struct ErrorSurfaceNotSupported;
+	impl std::fmt::Display for ErrorSurfaceNotSupported{
+		fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+			write!(f,"Surface isn't supported by the adapter.")
+		}
+	}
 	pub fn configure_surface(
 		adapter:&wgpu::Adapter,
 		device:&wgpu::Device,
 		surface:&wgpu::Surface<'_>,
 		(width,height):(u32,u32),
-	)->wgpu::SurfaceConfiguration{
+	)->Result<wgpu::SurfaceConfiguration,ErrorSurfaceNotSupported>{
 		let mut config=surface
 			.get_default_config(adapter, width, height)
-			.expect("Surface isn't supported by the adapter.");
+			.ok_or(ErrorSurfaceNotSupported)?;
 
 		let surface_view_format=config.format.add_srgb_suffix();
 		config.view_formats.push(surface_view_format);
 		config.present_mode=wgpu::PresentMode::AutoNoVsync;
 		surface.configure(device,&config);
 
-		config
+		Ok(config)
 	}
 }

lib/bsp_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "strafesnet_bsp_loader"
-version = "0.3.1"
+version = "0.4.0"
 edition = "2024"
 repository = "https://git.itzana.me/StrafesNET/strafe-project"
 license = "MIT OR Apache-2.0"

lib/bsp_loader/src/lib.rs

@@ -1,9 +1,9 @@
 use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLoader,RenderConfigDeferredLoader};
+use strafesnet_deferred_loader::loader::Loader;
 
 mod bsp;
 mod mesh;
 mod brush;
-pub mod loader;
 
 const VALVE_SCALE:f32=1.0/16.0;
 pub(crate) fn valve_transform_dist(d:f32)->strafesnet_common::integer::Planar64{
@@ -30,26 +30,25 @@ impl std::error::Error for ReadError{}
 
 
 #[derive(Debug)]
-pub enum LoadError{
-	Texture(loader::TextureError),
-	Mesh(loader::MeshError),
+pub enum LoadError<M,T>{
+	Mesh(M),
+	Texture(T),
 }
-impl std::fmt::Display for LoadError{
+impl<M,T> std::fmt::Display for LoadError<M,T>
+	where
+		M:std::fmt::Debug,
+		T:std::fmt::Debug,
+{
 	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
 		write!(f,"{self:?}")
 	}
 }
-impl std::error::Error for LoadError{}
-impl From<loader::TextureError> for LoadError{
-	fn from(value:loader::TextureError)->Self{
-		Self::Texture(value)
-	}
-}
-impl From<loader::MeshError> for LoadError{
-	fn from(value:loader::MeshError)->Self{
-		Self::Mesh(value)
-	}
-}
+impl<M,T> std::error::Error for LoadError<M,T>
+where
+		M:std::fmt::Debug,
+		T:std::fmt::Debug,
+{}
+
 pub struct Bsp{
 	bsp:vbsp::Bsp,
 	case_folded_file_names:std::collections::HashMap<String,String>,
@@ -84,7 +83,17 @@ impl Bsp{
 			None=>Ok(None),
 		}
 	}
-	pub fn to_snf(&self,failure_mode:LoadFailureMode,vpk_list:&[Vpk])->Result<strafesnet_common::map::CompleteMap,LoadError>{
+	pub fn to_snf<'dom,'mesh,'texture,M,T>(
+		&'dom self,
+		failure_mode:LoadFailureMode,
+		mut mesh_loader:M,
+		mut texture_loader:T,
+	)->Result<strafesnet_common::map::CompleteMap,LoadError<M::Error,T::Error>>
+		where
+			'dom:'mesh+'texture,
+			M:Loader<Resource=strafesnet_common::model::Mesh,Index<'mesh>=&'mesh str>+'mesh,
+			T:Loader<Resource=strafesnet_deferred_loader::texture::Texture,Index<'texture>=std::borrow::Cow<'texture,str>>+'texture,
+	{
 		let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
 		let mut mesh_deferred_loader=MeshDeferredLoader::new();
 
@@ -94,12 +103,10 @@ impl Bsp{
 			&mut mesh_deferred_loader,
 		);
 
-		let mut mesh_loader=loader::MeshLoader::new(loader::BspFinder{bsp:self,vpks:vpk_list},&mut texture_deferred_loader);
 		let prop_meshes=mesh_deferred_loader.into_meshes(&mut mesh_loader,failure_mode).map_err(LoadError::Mesh)?;
 
 		let map_step2=map_step1.add_prop_meshes(prop_meshes);
 
-		let mut texture_loader=loader::TextureLoader::new();
 		let render_configs=texture_deferred_loader.into_render_configs(&mut texture_loader,failure_mode).map_err(LoadError::Texture)?;
 
 		let map=map_step2.add_render_configs_and_textures(render_configs);

lib/common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "strafesnet_common"
-version = "0.8.2"
+version = "0.8.6"
 edition = "2024"
 repository = "https://git.itzana.me/StrafesNET/strafe-project"
 license = "MIT OR Apache-2.0"

lib/common/src/integer.rs

@@ -56,6 +56,14 @@ impl<T> Time<T>{
 	pub const fn coerce<U>(self)->Time<U>{
 		Time::raw(self.0)
 	}
+	#[inline]
+	pub const fn div_euclid(self,other:Self)->Self{
+		Time::raw(self.0.div_euclid(other.0))
+	}
+	#[inline]
+	pub const fn rem_euclid(self,other:Self)->Self{
+		Time::raw(self.0.rem_euclid(other.0))
+	}
 }
 impl<T> From<Planar64> for Time<T>{
 	#[inline]
@@ -149,10 +157,23 @@ impl_time_i64_rhs_operator!(Shr,shr);
 impl_time_i64_rhs_operator!(Shl,shl);
 impl<T> core::ops::Mul<Time<T>> for Planar64{
 	type Output=Ratio<Fixed<2,64>,Planar64>;
+	#[inline]
 	fn mul(self,rhs:Time<T>)->Self::Output{
 		Ratio::new(self*Fixed::raw(rhs.0),Planar64::raw(1_000_000_000))
 	}
 }
+impl<T> From<Time<T>> for f32{
+	#[inline]
+	fn from(value:Time<T>)->Self{
+		value.get() as f32/Time::<T>::ONE_SECOND.get() as f32
+	}
+}
+impl<T> From<Time<T>> for f64{
+	#[inline]
+	fn from(value:Time<T>)->Self{
+		value.get() as f64/Time::<T>::ONE_SECOND.get() as f64
+	}
+}
 #[cfg(test)]
 mod test_time{
 	use super::*;

lib/common/src/timer.rs

@@ -69,10 +69,8 @@ impl<In,Out> Scaled<In,Out>
 	const fn get_scale(&self)->Ratio64{
 		self.scale
 	}
-	fn set_scale(&mut self,time:Time<In>,new_scale:Ratio64){
-		let new_time=self.get_time(time);
+	const fn set_scale(&mut self,new_scale:Ratio64){
 		self.scale=new_scale;
-		self.set_time(time,new_time);
 	}
 }
 
@@ -132,7 +130,7 @@ pub struct TimerFixed<T:TimerState,P:PauseState>{
 	_paused:P,
 }
 
-//scaled timer methods are generic across PauseState
+//some scaled timer methods are generic across PauseState
 impl<P:PauseState,In,Out> TimerFixed<Scaled<In,Out>,P>
 	where Time<In>:Copy,
 {
@@ -147,8 +145,22 @@ impl<P:PauseState,In,Out> TimerFixed<Scaled<In,Out>,P>
 	pub const fn get_scale(&self)->Ratio64{
 		self.state.get_scale()
 	}
+}
+// setting the scale of an unpaused timer is different than a paused timer
+impl<In,Out> TimerFixed<Scaled<In,Out>,Unpaused>
+	where Time<In>:Copy,
+{
 	pub fn set_scale(&mut self,time:Time<In>,new_scale:Ratio64){
-		self.state.set_scale(time,new_scale)
+		let new_time=self.state.get_time(time);
+		self.state.set_scale(new_scale);
+		self.state.set_time(time,new_time);
+	}
+}
+impl<In,Out> TimerFixed<Scaled<In,Out>,Paused>
+	where Time<In>:Copy,
+{
+	pub fn set_scale(&mut self,new_scale:Ratio64){
+		self.state.set_scale(new_scale);
 	}
 }
 
@@ -305,7 +317,7 @@ impl<In,Out> Timer<Scaled<In,Out>>
 	}
 	pub fn set_scale(&mut self,time:Time<In>,new_scale:Ratio64){
 		match self{
-			Self::Paused(timer)=>timer.set_scale(time,new_scale),
+			Self::Paused(timer)=>timer.set_scale(new_scale),
 			Self::Unpaused(timer)=>timer.set_scale(time,new_scale),
 		}
 	}

lib/deferred_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "strafesnet_deferred_loader"
-version = "0.5.1"
+version = "0.6.0"
 edition = "2024"
 repository = "https://git.itzana.me/StrafesNET/strafe-project"
 license = "MIT OR Apache-2.0"

lib/rbx_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "strafesnet_rbx_loader"
-version = "0.7.0"
+version = "0.8.0"
 edition = "2024"
 repository = "https://git.itzana.me/StrafesNET/strafe-project"
 license = "MIT OR Apache-2.0"

lib/rbx_loader/src/lib.rs

@@ -1,17 +1,19 @@
-use std::io::Read;
 use rbx_dom_weak::WeakDom;
 use roblox_emulator::context::Context;
 use strafesnet_common::map::CompleteMap;
 use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLoader,RenderConfigDeferredLoader};
+use strafesnet_deferred_loader::loader::Loader;
+use strafesnet_deferred_loader::texture::Texture;
 
 pub use error::RecoverableErrors;
 pub use roblox_emulator::runner::Error as RunnerError;
 
+use crate::loader::{MeshIndex,MeshWithSize};
+
 mod rbx;
 mod mesh;
 mod error;
 mod union;
-pub mod loader;
 pub mod primitives;
 
 pub mod data{
@@ -30,11 +32,21 @@ pub struct Model{
 	dom:WeakDom,
 }
 impl Model{
-	fn new(dom:WeakDom)->Self{
+	pub fn new(dom:WeakDom)->Self{
 		Self{dom}
 	}
-	pub fn to_snf(&self,failure_mode:LoadFailureMode)->Result<(CompleteMap,RecoverableErrors),LoadError>{
-		to_snf(self,failure_mode)
+	pub fn to_snf<'dom,'mesh,'texture,M,T>(
+		&'dom self,
+		failure_mode:LoadFailureMode,
+		mesh_loader:M,
+		texture_loader:T,
+	)->Result<(CompleteMap,RecoverableErrors),LoadError<M::Error,T::Error>>
+		where
+			'dom:'mesh+'texture,
+			M:Loader<Resource=MeshWithSize,Index<'mesh>=MeshIndex<'mesh>>+'mesh,
+			T:Loader<Resource=Texture,Index<'texture>=&'texture str>+'texture,
+	{
+		to_snf(self.as_ref(),failure_mode,mesh_loader,texture_loader)
 	}
 }
 impl AsRef<WeakDom> for Model{
@@ -66,8 +78,18 @@ impl Place{
 		}
 		Ok(errors)
 	}
-	pub fn to_snf(&self,failure_mode:LoadFailureMode)->Result<(CompleteMap,RecoverableErrors),LoadError>{
-		to_snf(self,failure_mode)
+	pub fn to_snf<'dom,'mesh,'texture,M,T>(
+		&'dom self,
+		failure_mode:LoadFailureMode,
+		mesh_loader:M,
+		texture_loader:T,
+	)->Result<(CompleteMap,RecoverableErrors),LoadError<M::Error,T::Error>>
+		where
+			'dom:'mesh+'texture,
+			M:Loader<Resource=MeshWithSize,Index<'mesh>=MeshIndex<'mesh>>+'mesh,
+			T:Loader<Resource=Texture,Index<'texture>=&'texture str>+'texture,
+	{
+		to_snf(self.as_ref(),failure_mode,mesh_loader,texture_loader)
 	}
 }
 impl AsRef<WeakDom> for Place{
@@ -85,54 +107,36 @@ impl From<Model> for Place{
 }
 
 #[derive(Debug)]
-pub enum ReadError{
-	RbxBinary(rbx_binary::DecodeError),
-	RbxXml(rbx_xml::DecodeError),
-	Io(std::io::Error),
-	UnknownFileFormat,
+pub enum LoadError<M,T>{
+	Mesh(M),
+	Texture(T),
 }
-impl std::fmt::Display for ReadError{
+impl<M,T> std::fmt::Display for LoadError<M,T>
+	where
+		M:std::fmt::Debug,
+		T:std::fmt::Debug,
+{
 	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
 		write!(f,"{self:?}")
 	}
 }
-impl std::error::Error for ReadError{}
-
-pub fn read<R:Read>(input:R)->Result<Model,ReadError>{
-	let mut buf=std::io::BufReader::new(input);
-	let peek=std::io::BufRead::fill_buf(&mut buf).map_err(ReadError::Io)?;
-	match peek.get(0..8){
-		Some(b"<roblox!")=>rbx_binary::from_reader(buf).map(Model::new).map_err(ReadError::RbxBinary),
-		Some(b"<roblox ")=>rbx_xml::from_reader_default(buf).map(Model::new).map_err(ReadError::RbxXml),
-		_=>Err(ReadError::UnknownFileFormat),
-	}
-}
-
-#[derive(Debug)]
-pub enum LoadError{
-	Texture(loader::TextureError),
-	Mesh(loader::MeshError),
-}
-impl std::fmt::Display for LoadError{
-	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
-		write!(f,"{self:?}")
-	}
-}
-impl std::error::Error for LoadError{}
-impl From<loader::TextureError> for LoadError{
-	fn from(value:loader::TextureError)->Self{
-		Self::Texture(value)
-	}
-}
-impl From<loader::MeshError> for LoadError{
-	fn from(value:loader::MeshError)->Self{
-		Self::Mesh(value)
-	}
-}
-
-fn to_snf(dom:impl AsRef<WeakDom>,failure_mode:LoadFailureMode)->Result<(CompleteMap,RecoverableErrors),LoadError>{
-	let dom=dom.as_ref();
+impl<M,T> std::error::Error for LoadError<M,T>
+where
+		M:std::fmt::Debug,
+		T:std::fmt::Debug,
+{}
 
+fn to_snf<'dom,'mesh,'texture,M,T>(
+	dom:&'dom WeakDom,
+	failure_mode:LoadFailureMode,
+	mut mesh_loader:M,
+	mut texture_loader:T,
+)->Result<(CompleteMap,RecoverableErrors),LoadError<M::Error,T::Error>>
+	where
+		'dom:'mesh+'texture,
+		M:Loader<Resource=MeshWithSize,Index<'mesh>=MeshIndex<'mesh>>+'mesh,
+		T:Loader<Resource=Texture,Index<'texture>=&'texture str>+'texture,
+{
 	let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
 	let mut mesh_deferred_loader=MeshDeferredLoader::new();
 
@@ -142,12 +146,10 @@ fn to_snf(dom:impl AsRef<WeakDom>,failure_mode:LoadFailureMode)->Result<(Complet
 		&mut mesh_deferred_loader,
 	);
 
-	let mut mesh_loader=loader::MeshLoader::new();
 	let meshpart_meshes=mesh_deferred_loader.into_meshes(&mut mesh_loader,failure_mode).map_err(LoadError::Mesh)?;
 
 	let map_step2=map_step1.add_meshpart_meshes_and_calculate_attributes(meshpart_meshes);
 
-	let mut texture_loader=loader::TextureLoader::new();
 	let render_configs=texture_deferred_loader.into_render_configs(&mut texture_loader,failure_mode).map_err(LoadError::Texture)?;
 
 	Ok(map_step2.add_render_configs_and_textures(render_configs))

map-tool/Cargo.toml

@@ -1,15 +1,26 @@
 [package]
 name = "map-tool"
-version = "1.7.2"
+version = "2.0.0"
 edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+default = ["cli"]
+cli = ["dep:clap", "tokio/macros", "tokio/rt-multi-thread", "tokio/fs", "dep:futures"]
+
+[lib]
+name = "map_tool"
+
+[[bin]]
+name = "map-tool"
+required-features = ["cli"]
+
 [dependencies]
 anyhow = "1.0.75"
-clap = { version = "4.4.2", features = ["derive"] }
+clap = { version = "4.4.2", features = ["derive"], optional = true }
 flate2 = "1.0.27"
-futures = "0.3.31"
+futures = { version = "0.3.31", optional = true }
 image = "0.25.2"
 image_dds = "0.7.1"
 rbx_asset = { version = "0.5.0", registry = "strafesnet" }
@@ -23,7 +34,7 @@ strafesnet_deferred_loader.workspace = true
 strafesnet_rbx_loader.workspace = true
 strafesnet_snf.workspace = true
 thiserror = "2.0.11"
-tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread", "fs"] }
+tokio = { version = "1.43.0", features = ["time"] }
 vbsp = "0.9.1"
 vbsp-entities-css = "0.6.0"
 vmdl = "0.2.0"

map-tool/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod roblox;
+pub mod source;

map-tool/src/main.rs

@@ -1,23 +1,20 @@
-mod roblox;
-mod source;
-
 use clap::{Parser,Subcommand};
 use anyhow::Result as AResult;
 
 #[derive(Parser)]
-#[command(author, version, about, long_about = None)]
-#[command(propagate_version = true)]
-struct Cli {
+#[command(author,version,about,long_about=None)]
+#[command(propagate_version=true)]
+struct Cli{
 	#[command(subcommand)]
-	command: Commands,
+	command:Commands,
 }
 
 #[derive(Subcommand)]
 enum Commands{
 	#[command(flatten)]
-	Roblox(roblox::Commands),
+	Roblox(map_tool::roblox::Commands),
 	#[command(flatten)]
-	Source(source::Commands),
+	Source(map_tool::source::Commands),
 }
 
 #[tokio::main]

map-tool/src/roblox.rs

@@ -1,501 +0,0 @@
-use std::path::{Path,PathBuf};
-use std::io::{Cursor,Read,Seek};
-use std::collections::HashSet;
-use clap::{Args,Subcommand};
-use anyhow::Result as AResult;
-use rbx_dom_weak::Instance;
-use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
-use rbxassetid::RobloxAssetId;
-use tokio::io::AsyncReadExt;
-
-// disallow non-static lifetimes
-fn static_ustr(s:&'static str)->rbx_dom_weak::Ustr{
-	rbx_dom_weak::ustr(s)
-}
-
-const DOWNLOAD_LIMIT:usize=16;
-
-#[derive(Subcommand)]
-pub enum Commands{
-	RobloxToSNF(RobloxToSNFSubcommand),
-	DownloadAssets(DownloadAssetsSubcommand),
-}
-
-#[derive(Args)]
-pub struct RobloxToSNFSubcommand {
-	#[arg(long)]
-	output_folder:PathBuf,
-	#[arg(required=true)]
-	input_files:Vec<PathBuf>,
-}
-#[derive(Args)]
-pub struct DownloadAssetsSubcommand{
-	#[arg(required=true)]
-	roblox_files:Vec<PathBuf>,
-	#[arg(long,group="cookie",required=true)]
-	cookie_literal:Option<String>,
-	#[arg(long,group="cookie",required=true)]
-	cookie_envvar:Option<String>,
-	#[arg(long,group="cookie",required=true)]
-	cookie_file:Option<PathBuf>,
-}
-
-impl Commands{
-	pub async fn run(self)->AResult<()>{
-		match self{
-			Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder).await,
-			Commands::DownloadAssets(subcommand)=>download_assets(
-				subcommand.roblox_files,
-				cookie_from_args(
-					subcommand.cookie_literal,
-					subcommand.cookie_envvar,
-					subcommand.cookie_file,
-				).await?,
-			).await,
-		}
-	}
-}
-
-async fn cookie_from_args(literal:Option<String>,environment:Option<String>,file:Option<PathBuf>)->AResult<rbx_asset::cookie::Cookie>{
-	let cookie=match (literal,environment,file){
-		(Some(cookie_literal),None,None)=>cookie_literal,
-		(None,Some(cookie_environment),None)=>std::env::var(cookie_environment)?,
-		(None,None,Some(cookie_file))=>tokio::fs::read_to_string(cookie_file).await?,
-		_=>Err(anyhow::Error::msg("Illegal cookie argument triple"))?,
-	};
-	Ok(rbx_asset::cookie::Cookie::new(cookie))
-}
-
-#[expect(dead_code)]
-#[derive(Debug)]
-enum LoadDomError{
-	IO(std::io::Error),
-	Binary(rbx_binary::DecodeError),
-	Xml(rbx_xml::DecodeError),
-	UnknownFormat,
-}
-fn load_dom<R:Read+Seek>(mut input:R)->Result<rbx_dom_weak::WeakDom,LoadDomError>{
-	let mut first_8=[0u8;8];
-	input.read_exact(&mut first_8).map_err(LoadDomError::IO)?;
-	input.rewind().map_err(LoadDomError::IO)?;
-	match &first_8{
-		b"<roblox!"=>rbx_binary::from_reader(input).map_err(LoadDomError::Binary),
-		b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(LoadDomError::Xml),
-		_=>Err(LoadDomError::UnknownFormat),
-	}
-}
-
-/* The ones I'm interested in:
-Beam.Texture
-Decal.Texture
-FileMesh.MeshId
-FileMesh.TextureId
-MaterialVariant.ColorMap
-MaterialVariant.MetalnessMap
-MaterialVariant.NormalMap
-MaterialVariant.RoughnessMap
-MeshPart.MeshId
-MeshPart.TextureID
-ParticleEmitter.Texture
-Sky.MoonTextureId
-Sky.SkyboxBk
-Sky.SkyboxDn
-Sky.SkyboxFt
-Sky.SkyboxLf
-Sky.SkyboxRt
-Sky.SkyboxUp
-Sky.SunTextureId
-SurfaceAppearance.ColorMap
-SurfaceAppearance.MetalnessMap
-SurfaceAppearance.NormalMap
-SurfaceAppearance.RoughnessMap
-SurfaceAppearance.TexturePack
-*/
-/* These properties now use Content
-BaseWrap.CageMeshContent
-Decal.TextureContent
-ImageButton.ImageContent
-ImageLabel.ImageContent
-MeshPart.MeshContent
-MeshPart.TextureContent
-SurfaceAppearance.ColorMapContent
-SurfaceAppearance.MetalnessMapContent
-SurfaceAppearance.NormalMapContent
-SurfaceAppearance.RoughnessMapContent
-WrapLayer.ReferenceMeshContent
-*/
-
-fn accumulate_content(content_list:&mut HashSet<RobloxAssetId>,object:&Instance,property:&'static str){
-	let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(&static_ustr(property))else{
-		println!("property={} does not exist for class={}",property,object.class.as_str());
-		return;
-	};
-	let rbx_dom_weak::types::ContentType::Uri(uri)=content.value()else{
-		println!("ContentType is not Uri");
-		return;
-	};
-	let Ok(asset_id)=uri.parse()else{
-		println!("Content failed to parse into AssetID: {:?}",content);
-		return;
-	};
-	content_list.insert(asset_id);
-}
-fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instance,property:&'static str){
-	let Some(rbx_dom_weak::types::Variant::ContentId(content))=object.properties.get(&static_ustr(property))else{
-		println!("property={} does not exist for class={}",property,object.class.as_str());
-		return;
-	};
-	let Ok(asset_id)=content.as_str().parse()else{
-		println!("Content failed to parse into AssetID: {:?}",content);
-		return;
-	};
-	content_list.insert(asset_id);
-}
-async fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
-	let mut file=tokio::fs::File::open(path).await?;
-	let mut data=Vec::new();
-	file.read_to_end(&mut data).await?;
-	Ok(Cursor::new(data))
-}
-#[derive(Default)]
-struct UniqueAssets{
-	meshes:HashSet<RobloxAssetId>,
-	unions:HashSet<RobloxAssetId>,
-	textures:HashSet<RobloxAssetId>,
-}
-impl UniqueAssets{
-	fn collect(&mut self,object:&Instance){
-		match object.class.as_str(){
-			"Beam"=>accumulate_content_id(&mut self.textures,object,"Texture"),
-			"Decal"=>accumulate_content(&mut self.textures,object,"TextureContent"),
-			"Texture"=>accumulate_content(&mut self.textures,object,"TextureContent"),
-			"FileMesh"=>accumulate_content_id(&mut self.textures,object,"TextureId"),
-			"MeshPart"=>{
-				accumulate_content(&mut self.textures,object,"TextureContent");
-				accumulate_content(&mut self.meshes,object,"MeshContent");
-			},
-			"SpecialMesh"=>accumulate_content_id(&mut self.meshes,object,"MeshId"),
-			"ParticleEmitter"=>accumulate_content_id(&mut self.textures,object,"Texture"),
-			"Sky"=>{
-				accumulate_content_id(&mut self.textures,object,"MoonTextureId");
-				accumulate_content_id(&mut self.textures,object,"SkyboxBk");
-				accumulate_content_id(&mut self.textures,object,"SkyboxDn");
-				accumulate_content_id(&mut self.textures,object,"SkyboxFt");
-				accumulate_content_id(&mut self.textures,object,"SkyboxLf");
-				accumulate_content_id(&mut self.textures,object,"SkyboxRt");
-				accumulate_content_id(&mut self.textures,object,"SkyboxUp");
-				accumulate_content_id(&mut self.textures,object,"SunTextureId");
-			},
-			"UnionOperation"=>accumulate_content_id(&mut self.unions,object,"AssetId"),
-			_=>(),
-		}
-	}
-}
-
-#[expect(dead_code)]
-#[derive(Debug)]
-enum UniqueAssetError{
-	IO(std::io::Error),
-	LoadDom(LoadDomError),
-}
-async fn unique_assets(path:&Path)->Result<UniqueAssets,UniqueAssetError>{
-	// read entire file
-	let mut assets=UniqueAssets::default();
-	let data=read_entire_file(path).await.map_err(UniqueAssetError::IO)?;
-	let dom=load_dom(data).map_err(UniqueAssetError::LoadDom)?;
-	for object in dom.into_raw().1.into_values(){
-		assets.collect(&object);
-	}
-	Ok(assets)
-}
-enum DownloadType{
-	Texture(RobloxAssetId),
-	Mesh(RobloxAssetId),
-	Union(RobloxAssetId),
-}
-impl DownloadType{
-	fn path(&self)->PathBuf{
-		match self{
-			DownloadType::Texture(RobloxAssetId(asset_id))=>format!("downloaded_textures/{asset_id}").into(),
-			DownloadType::Mesh(RobloxAssetId(asset_id))=>format!("meshes/{asset_id}").into(),
-			DownloadType::Union(RobloxAssetId(asset_id))=>format!("unions/{asset_id}").into(),
-		}
-	}
-	fn asset_id(&self)->u64{
-		match self{
-			&DownloadType::Texture(RobloxAssetId(asset_id))=>asset_id,
-			&DownloadType::Mesh(RobloxAssetId(asset_id))=>asset_id,
-			&DownloadType::Union(RobloxAssetId(asset_id))=>asset_id,
-		}
-	}
-}
-enum DownloadResult{
-	Cached(PathBuf),
-	Data(Vec<u8>),
-	Failed,
-}
-#[derive(Default,Debug)]
-struct Stats{
-	total_assets:u32,
-	cached_assets:u32,
-	downloaded_assets:u32,
-	failed_downloads:u32,
-	timed_out_downloads:u32,
-}
-async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::Context,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
-	stats.total_assets+=1;
-	// check if file exists on disk
-	let path=download_instruction.path();
-	if tokio::fs::try_exists(path.as_path()).await?{
-		stats.cached_assets+=1;
-		return Ok(DownloadResult::Cached(path));
-	}
-	let asset_id=download_instruction.asset_id();
-	// if not, download file
-	let mut retry=0;
-	const BACKOFF_MUL:f32=1.3956124250860895286;//exp(1/3)
-	let mut backoff=1000f32;
-	loop{
-		let asset_result=context.get_asset(rbx_asset::cookie::GetAssetRequest{
-			asset_id,
-			version:None,
-		}).await;
-		match asset_result{
-			Ok(asset_result)=>{
-				stats.downloaded_assets+=1;
-				let data=asset_result.to_vec()?;
-				tokio::fs::write(path,&data).await?;
-				break Ok(DownloadResult::Data(data));
-			},
-			Err(rbx_asset::cookie::GetError::Response(rbx_asset::types::ResponseError::Details{status_code,url_and_body}))=>{
-				if status_code.as_u16()==429{
-					if retry==12{
-						println!("Giving up asset download {asset_id}");
-						stats.timed_out_downloads+=1;
-						break Ok(DownloadResult::Failed);
-					}
-					println!("Hit roblox rate limit, waiting {:.0}ms...",backoff);
-					tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
-					backoff*=BACKOFF_MUL;
-					retry+=1;
-				}else{
-					stats.failed_downloads+=1;
-					println!("weird status_code error: status_code={status_code} url={} body={}",url_and_body.url,url_and_body.body);
-					break Ok(DownloadResult::Failed);
-				}
-			},
-			Err(e)=>{
-				stats.failed_downloads+=1;
-				println!("sadly error: {e}");
-				break Ok(DownloadResult::Failed);
-			},
-		}
-	}
-}
-#[derive(Debug,thiserror::Error)]
-enum ConvertTextureError{
-	#[error("Io error {0:?}")]
-	Io(#[from]std::io::Error),
-	#[error("Image error {0:?}")]
-	Image(#[from]image::ImageError),
-	#[error("DDS create error {0:?}")]
-	DDS(#[from]image_dds::CreateDdsError),
-	#[error("DDS write error {0:?}")]
-	DDSWrite(#[from]image_dds::ddsfile::Error),
-}
-async fn convert_texture(RobloxAssetId(asset_id):RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
-	let data=match download_result{
-		DownloadResult::Cached(path)=>tokio::fs::read(path).await?,
-		DownloadResult::Data(data)=>data,
-		DownloadResult::Failed=>return Ok(()),
-	};
-	// image::ImageFormat::Png
-	// image::ImageFormat::Jpeg
-	let image=image::load_from_memory(&data)?.to_rgba8();
-
-	// pick format
-	let format=if image.width()%4!=0||image.height()%4!=0{
-		image_dds::ImageFormat::Rgba8UnormSrgb
-	}else{
-		image_dds::ImageFormat::BC7RgbaUnormSrgb
-	};
-
-	//this fails if the image dimensions are not a multiple of 4
-	let dds=image_dds::dds_from_image(
-		&image,
-		format,
-		image_dds::Quality::Slow,
-		image_dds::Mipmaps::GeneratedAutomatic,
-	)?;
-
-	let file_name=format!("textures/{asset_id}.dds");
-	let mut file=std::fs::File::create(file_name)?;
-	dds.write(&mut file)?;
-	Ok(())
-}
-async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
-	tokio::try_join!(
-		tokio::fs::create_dir_all("downloaded_textures"),
-		tokio::fs::create_dir_all("textures"),
-		tokio::fs::create_dir_all("meshes"),
-		tokio::fs::create_dir_all("unions"),
-	)?;
-	// use mpsc
-	let thread_limit=std::thread::available_parallelism()?.get();
-	let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
-	let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
-	// map decode dispatcher
-	// read files multithreaded
-	// produce UniqueAssetsResult per file
-	tokio::spawn(async move{
-		// move send so it gets dropped when all maps have been decoded
-		// closing the channel
-		let mut it=paths.into_iter();
-		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
-		SEM.add_permits(thread_limit);
-		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
-			let send=send_assets.clone();
-			tokio::spawn(async move{
-				let result=unique_assets(path.as_path()).await;
-				_=send.send(result).await;
-				drop(permit);
-			});
-		}
-	});
-	// download manager
-	// insert into global unique assets guy
-	// add to download queue if the asset is globally unique and does not already exist on disk
-	let mut stats=Stats::default();
-	let context=rbx_asset::cookie::Context::new(cookie);
-	let mut globally_unique_assets=UniqueAssets::default();
-	// pop a job = retry_queue.pop_front() or ingest(recv.recv().await)
-	// SLOW MODE:
-	// acquire all permits
-	// drop all permits
-	// pop one job
-	// if it succeeds go into fast mode
-	// FAST MODE:
-	// acquire one permit
-	// pop a job
-	let download_thread=tokio::spawn(async move{
-		while let Some(result)=recv_assets.recv().await{
-			let unique_assets=match result{
-				Ok(unique_assets)=>unique_assets,
-				Err(e)=>{
-					println!("error: {e:?}");
-					continue;
-				},
-			};
-			for texture_id in unique_assets.textures{
-				if globally_unique_assets.textures.insert(texture_id){
-					let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
-					send_texture.send((texture_id,data)).await?;
-				}
-			}
-			for mesh_id in unique_assets.meshes{
-				if globally_unique_assets.meshes.insert(mesh_id){
-					download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
-				}
-			}
-			for union_id in unique_assets.unions{
-				if globally_unique_assets.unions.insert(union_id){
-					download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
-				}
-			}
-		}
-		dbg!(stats);
-		Ok::<(),anyhow::Error>(())
-	});
-	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
-	SEM.add_permits(thread_limit);
-	while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){
-		tokio::spawn(async move{
-			let result=convert_texture(asset_id,download_result).await;
-			drop(permit);
-			result.unwrap();
-		});
-	}
-	download_thread.await??;
-	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
-	Ok(())
-}
-
-#[derive(Debug)]
-#[expect(dead_code)]
-enum ConvertError{
-	IO(std::io::Error),
-	SNFMap(strafesnet_snf::map::Error),
-	RobloxRead(strafesnet_rbx_loader::ReadError),
-	RobloxLoad(strafesnet_rbx_loader::LoadError),
-}
-impl std::fmt::Display for ConvertError{
-	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
-		write!(f,"{self:?}")
-	}
-}
-impl std::error::Error for ConvertError{}
-
-struct Errors{
-	script_errors:Vec<strafesnet_rbx_loader::RunnerError>,
-	convert_errors:strafesnet_rbx_loader::RecoverableErrors,
-}
-
-fn convert_to_snf(path:&Path,output_folder:PathBuf)->Result<Errors,ConvertError>{
-	let entire_file=std::fs::read(path).map_err(ConvertError::IO)?;
-
-	let model=strafesnet_rbx_loader::read(
-		entire_file.as_slice()
-	).map_err(ConvertError::RobloxRead)?;
-
-	let mut place=strafesnet_rbx_loader::Place::from(model);
-	let script_errors=place.run_scripts().unwrap_or_else(|e|vec![e]);
-
-	let (map,convert_errors)=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?;
-
-	let mut dest=output_folder;
-	dest.push(path.file_stem().unwrap());
-	dest.set_extension("snfm");
-	let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
-
-	strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
-
-	Ok(Errors{
-		script_errors,
-		convert_errors,
-	})
-}
-
-async fn roblox_to_snf(paths:Vec<PathBuf>,output_folder:PathBuf)->AResult<()>{
-	let start=std::time::Instant::now();
-
-	let thread_limit=std::thread::available_parallelism()?.get();
-	let mut it=paths.into_iter();
-	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
-	// This is wrong! Calling roblox_to_snf multiple times keeps adding permits
-	SEM.add_permits(thread_limit);
-
-	while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
-		let output_folder=output_folder.clone();
-		tokio::task::spawn_blocking(move||{
-			let result=convert_to_snf(path.as_path(),output_folder);
-			drop(permit);
-			match result{
-				Ok(errors)=>{
-					for error in errors.script_errors{
-						println!("Script error: {error}");
-					}
-					let error_count=errors.convert_errors.count();
-					if error_count!=0{
-						println!("Error count: {error_count}");
-						println!("Errors: {}",errors.convert_errors);
-					}
-				},
-				Err(e)=>println!("Convert error: {e:?}"),
-			}
-		});
-	}
-	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
-
-	println!("elapsed={:?}", start.elapsed());
-	Ok(())
-}

map-tool/src/roblox/cli.rs

@@ -0,0 +1,287 @@
+use std::path::PathBuf;
+use anyhow::Result as AResult;
+use rbxassetid::RobloxAssetId;
+
+use super::{convert_to_snf,convert_texture_to_dds,get_unique_assets_from_file};
+use super::{ConvertTextureError,UniqueAssets};
+
+const DOWNLOAD_LIMIT:usize=16;
+
+#[derive(clap::Subcommand)]
+pub enum Commands{
+	RobloxToSNF(RobloxToSNFSubcommand),
+	DownloadAssets(DownloadAssetsSubcommand),
+}
+
+#[derive(clap::Args)]
+pub struct RobloxToSNFSubcommand{
+	#[arg(long)]
+	output_folder:PathBuf,
+	#[arg(required=true)]
+	input_files:Vec<PathBuf>,
+}
+#[derive(clap::Args)]
+pub struct DownloadAssetsSubcommand{
+	#[arg(required=true)]
+	roblox_files:Vec<PathBuf>,
+	#[arg(long,group="cookie",required=true)]
+	cookie_literal:Option<String>,
+	#[arg(long,group="cookie",required=true)]
+	cookie_envvar:Option<String>,
+	#[arg(long,group="cookie",required=true)]
+	cookie_file:Option<PathBuf>,
+}
+
+impl Commands{
+	pub async fn run(self)->AResult<()>{
+		match self{
+			Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder).await,
+			Commands::DownloadAssets(subcommand)=>download_assets(
+				subcommand.roblox_files,
+				cookie_from_args(
+					subcommand.cookie_literal,
+					subcommand.cookie_envvar,
+					subcommand.cookie_file,
+				).await?,
+			).await,
+		}
+	}
+}
+
+async fn cookie_from_args(literal:Option<String>,environment:Option<String>,file:Option<PathBuf>)->AResult<rbx_asset::cookie::Cookie>{
+	let cookie=match (literal,environment,file){
+		(Some(cookie_literal),None,None)=>cookie_literal,
+		(None,Some(cookie_environment),None)=>std::env::var(cookie_environment)?,
+		(None,None,Some(cookie_file))=>tokio::fs::read_to_string(cookie_file).await?,
+		_=>Err(anyhow::Error::msg("Illegal cookie argument triple"))?,
+	};
+	Ok(rbx_asset::cookie::Cookie::new(cookie))
+}
+
+enum DownloadType{
+	Texture(RobloxAssetId),
+	Mesh(RobloxAssetId),
+	Union(RobloxAssetId),
+}
+impl DownloadType{
+	fn path(&self)->PathBuf{
+		match self{
+			DownloadType::Texture(RobloxAssetId(asset_id))=>format!("downloaded_textures/{asset_id}").into(),
+			DownloadType::Mesh(RobloxAssetId(asset_id))=>format!("meshes/{asset_id}").into(),
+			DownloadType::Union(RobloxAssetId(asset_id))=>format!("unions/{asset_id}").into(),
+		}
+	}
+	fn asset_id(&self)->u64{
+		match self{
+			&DownloadType::Texture(RobloxAssetId(asset_id))=>asset_id,
+			&DownloadType::Mesh(RobloxAssetId(asset_id))=>asset_id,
+			&DownloadType::Union(RobloxAssetId(asset_id))=>asset_id,
+		}
+	}
+}
+enum DownloadResult{
+	Cached(PathBuf),
+	Data(Vec<u8>),
+	Failed,
+}
+#[derive(Default,Debug)]
+struct Stats{
+	total_assets:u32,
+	cached_assets:u32,
+	downloaded_assets:u32,
+	failed_downloads:u32,
+	timed_out_downloads:u32,
+}
+async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::Context,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
+	stats.total_assets+=1;
+	// check if file exists on disk
+	let path=download_instruction.path();
+	if tokio::fs::try_exists(path.as_path()).await?{
+		stats.cached_assets+=1;
+		return Ok(DownloadResult::Cached(path));
+	}
+	let asset_id=download_instruction.asset_id();
+	// if not, download file
+	let mut retry=0;
+	let mut backoff=1000f32;
+	loop{
+		let asset_result=context.get_asset(rbx_asset::cookie::GetAssetRequest{
+			asset_id,
+			version:None,
+		}).await;
+		match asset_result{
+			Ok(asset_result)=>{
+				stats.downloaded_assets+=1;
+				let data=asset_result.to_vec()?;
+				tokio::fs::write(path,&data).await?;
+				break Ok(DownloadResult::Data(data));
+			},
+			Err(rbx_asset::cookie::GetError::Response(rbx_asset::types::ResponseError::Details{status_code,url_and_body}))=>{
+				if status_code.as_u16()==429{
+					if retry==12{
+						println!("Giving up asset download {asset_id}");
+						stats.timed_out_downloads+=1;
+						break Ok(DownloadResult::Failed);
+					}
+					println!("Hit roblox rate limit, waiting {:.0}ms...",backoff);
+					tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
+					backoff*=super::BACKOFF_MUL;
+					retry+=1;
+				}else{
+					stats.failed_downloads+=1;
+					println!("weird status_code error: status_code={status_code} url={} body={}",url_and_body.url,url_and_body.body);
+					break Ok(DownloadResult::Failed);
+				}
+			},
+			Err(e)=>{
+				stats.failed_downloads+=1;
+				println!("sadly error: {e}");
+				break Ok(DownloadResult::Failed);
+			},
+		}
+	}
+}
+
+async fn cli_convert_texture(RobloxAssetId(asset_id):RobloxAssetId,download_result:DownloadResult)->Result<(),CliConvertTextureError>{
+	let data=match download_result{
+		DownloadResult::Cached(path)=>tokio::fs::read(path).await?,
+		DownloadResult::Data(data)=>data,
+		DownloadResult::Failed=>return Ok(()),
+	};
+	let dds=convert_texture_to_dds(&data)?;
+
+	let file_name=format!("textures/{asset_id}.dds");
+	tokio::fs::write(file_name,&dds).await?;
+	Ok(())
+}
+#[derive(Debug,thiserror::Error)]
+enum CliConvertTextureError{
+	#[error("Io error {0:?}")]
+	Io(#[from]std::io::Error),
+	#[error("ConvertTexture error {0:?}")]
+	ConvertTexture(#[from]ConvertTextureError),
+}
+
+async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
+	tokio::try_join!(
+		tokio::fs::create_dir_all("downloaded_textures"),
+		tokio::fs::create_dir_all("textures"),
+		tokio::fs::create_dir_all("meshes"),
+		tokio::fs::create_dir_all("unions"),
+	)?;
+	// use mpsc
+	let thread_limit=std::thread::available_parallelism()?.get();
+	let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
+	let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+	// map decode dispatcher
+	// read files multithreaded
+	// produce UniqueAssetsResult per file
+	tokio::spawn(async move{
+		// move send so it gets dropped when all maps have been decoded
+		// closing the channel
+		let mut it=paths.into_iter();
+		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+		SEM.add_permits(thread_limit);
+		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+			let send=send_assets.clone();
+			tokio::spawn(async move{
+				let result=match tokio::fs::read(&path).await{
+					Ok(data)=>get_unique_assets_from_file(&data).map_err(|e|format!("{e:?}")),
+					Err(e)=>Err(format!("{e:?}")),
+				};
+				_=send.send(result).await;
+				drop(permit);
+			});
+		}
+	});
+	// download manager
+	let mut stats=Stats::default();
+	let context=rbx_asset::cookie::Context::new(cookie);
+	let mut globally_unique_assets=UniqueAssets::default();
+	let download_thread=tokio::spawn(async move{
+		while let Some(result)=recv_assets.recv().await{
+			let unique_assets=match result{
+				Ok(unique_assets)=>unique_assets,
+				Err(e)=>{
+					println!("error: {e:?}");
+					continue;
+				},
+			};
+			for texture_id in unique_assets.textures{
+				if globally_unique_assets.textures.insert(texture_id){
+					let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
+					send_texture.send((texture_id,data)).await?;
+				}
+			}
+			for mesh_id in unique_assets.meshes{
+				if globally_unique_assets.meshes.insert(mesh_id){
+					download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
+				}
+			}
+			for union_id in unique_assets.unions{
+				if globally_unique_assets.unions.insert(union_id){
+					download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
+				}
+			}
+		}
+		dbg!(stats);
+		Ok::<(),anyhow::Error>(())
+	});
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+	while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){
+		tokio::spawn(async move{
+			let result=cli_convert_texture(asset_id,download_result).await;
+			drop(permit);
+			result.unwrap();
+		});
+	}
+	download_thread.await??;
+	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
+	Ok(())
+}
+
+async fn roblox_to_snf(paths:Vec<PathBuf>,output_folder:PathBuf)->AResult<()>{
+	let start=std::time::Instant::now();
+
+	let thread_limit=std::thread::available_parallelism()?.get();
+	let mut it=paths.into_iter();
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	// This is wrong! Calling roblox_to_snf multiple times keeps adding permits
+	SEM.add_permits(thread_limit);
+
+	while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+		let output_folder=output_folder.clone();
+		tokio::task::spawn_blocking(move||{
+			let result=std::fs::read(&path).and_then(|data|{
+				super::load_dom(&data).map_err(|e|std::io::Error::other(e))
+				.and_then(|dom|convert_to_snf(dom).map_err(|e|std::io::Error::other(e)))
+				.and_then(|output|{
+					let mut dest=output_folder;
+					dest.push(path.file_stem().unwrap());
+					dest.set_extension("snfm");
+					std::fs::write(dest,&output.snf)?;
+					Ok(output)
+				})
+			});
+			drop(permit);
+			match result{
+				Ok(output)=>{
+					for error in output.script_errors{
+						println!("Script error: {error}");
+					}
+					let error_count=output.convert_errors.count();
+					if error_count!=0{
+						println!("Error count: {error_count}");
+						println!("Errors: {}",output.convert_errors);
+					}
+				},
+				Err(e)=>println!("Convert error: {e:?}"),
+			}
+		});
+	}
+	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
+
+	println!("elapsed={:?}", start.elapsed());
+	Ok(())
+}

map-tool/src/roblox/mod.rs

@@ -0,0 +1,248 @@
+#[cfg(feature="cli")]
+mod cli;
+#[cfg(feature="cli")]
+pub use cli::Commands;
+#[cfg(feature="cli")]
+mod loader;
+
+use std::io::{Cursor,Read,Seek};
+use std::collections::HashSet;
+use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
+pub use rbxassetid::RobloxAssetId;
+use rbx_dom_weak::Instance;
+
+// disallow non-static lifetimes
+fn static_ustr(s:&'static str)->rbx_dom_weak::Ustr{
+	rbx_dom_weak::ustr(s)
+}
+
+#[derive(Debug)]
+pub enum LoadDomError{
+	IO(std::io::Error),
+	Binary(rbx_binary::DecodeError),
+	Xml(rbx_xml::DecodeError),
+	UnknownFormat,
+}
+impl std::fmt::Display for LoadDomError{
+	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+		write!(f,"{self:?}")
+	}
+}
+impl std::error::Error for LoadDomError{}
+pub fn load_dom(data:&[u8])->Result<rbx_dom_weak::WeakDom,LoadDomError>{
+	let mut input=Cursor::new(data);
+	let mut first_8=[0u8;8];
+	input.read_exact(&mut first_8).map_err(LoadDomError::IO)?;
+	input.rewind().map_err(LoadDomError::IO)?;
+	match &first_8{
+		b"<roblox!"=>rbx_binary::from_reader(input).map_err(LoadDomError::Binary),
+		b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(LoadDomError::Xml),
+		_=>Err(LoadDomError::UnknownFormat),
+	}
+}
+
+fn accumulate_content(content_list:&mut HashSet<RobloxAssetId>,object:&Instance,property:&'static str){
+	let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(&static_ustr(property))else{
+		println!("property={} does not exist for class={}",property,object.class.as_str());
+		return;
+	};
+	let rbx_dom_weak::types::ContentType::Uri(uri)=content.value()else{
+		println!("ContentType is not Uri");
+		return;
+	};
+	let Ok(asset_id)=uri.parse()else{
+		println!("Content failed to parse into AssetID: {:?}",content);
+		return;
+	};
+	content_list.insert(asset_id);
+}
+fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instance,property:&'static str){
+	let Some(rbx_dom_weak::types::Variant::ContentId(content))=object.properties.get(&static_ustr(property))else{
+		println!("property={} does not exist for class={}",property,object.class.as_str());
+		return;
+	};
+	let Ok(asset_id)=content.as_str().parse()else{
+		println!("Content failed to parse into AssetID: {:?}",content);
+		return;
+	};
+	content_list.insert(asset_id);
+}
+
+#[derive(Default)]
+pub struct UniqueAssets{
+	pub meshes:HashSet<RobloxAssetId>,
+	pub unions:HashSet<RobloxAssetId>,
+	pub textures:HashSet<RobloxAssetId>,
+}
+impl UniqueAssets{
+	fn collect(&mut self,object:&Instance){
+		match object.class.as_str(){
+			"Beam"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+			"Decal"=>accumulate_content(&mut self.textures,object,"TextureContent"),
+			"Texture"=>accumulate_content(&mut self.textures,object,"TextureContent"),
+			"FileMesh"=>accumulate_content_id(&mut self.textures,object,"TextureId"),
+			"MeshPart"=>{
+				accumulate_content(&mut self.textures,object,"TextureContent");
+				accumulate_content(&mut self.meshes,object,"MeshContent");
+			},
+			"SpecialMesh"=>accumulate_content_id(&mut self.meshes,object,"MeshId"),
+			"ParticleEmitter"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+			"Sky"=>{
+				accumulate_content_id(&mut self.textures,object,"MoonTextureId");
+				accumulate_content_id(&mut self.textures,object,"SkyboxBk");
+				accumulate_content_id(&mut self.textures,object,"SkyboxDn");
+				accumulate_content_id(&mut self.textures,object,"SkyboxFt");
+				accumulate_content_id(&mut self.textures,object,"SkyboxLf");
+				accumulate_content_id(&mut self.textures,object,"SkyboxRt");
+				accumulate_content_id(&mut self.textures,object,"SkyboxUp");
+				accumulate_content_id(&mut self.textures,object,"SunTextureId");
+			},
+			"UnionOperation"=>accumulate_content_id(&mut self.unions,object,"AssetId"),
+			_=>(),
+		}
+	}
+}
+
+pub fn get_unique_assets(dom:rbx_dom_weak::WeakDom)->UniqueAssets{
+	let mut assets=UniqueAssets::default();
+	for object in dom.into_raw().1.into_values(){
+		assets.collect(&object);
+	}
+	assets
+}
+
+pub fn get_unique_assets_from_file(data:&[u8])->Result<UniqueAssets,UniqueAssetError>{
+	let dom=load_dom(data).map_err(UniqueAssetError::LoadDom)?;
+	Ok(get_unique_assets(dom))
+}
+
+#[derive(Debug)]
+pub enum UniqueAssetError{
+	LoadDom(LoadDomError),
+}
+
+#[derive(Debug)]
+pub enum ConvertError{
+	IO(std::io::Error),
+	SNFMap(strafesnet_snf::map::Error),
+	RobloxLoad(strafesnet_rbx_loader::LoadError),
+}
+impl std::fmt::Display for ConvertError{
+	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+		write!(f,"{self:?}")
+	}
+}
+impl std::error::Error for ConvertError{}
+
+pub struct ConvertOutput{
+	pub snf:Vec<u8>,
+	pub script_errors:Vec<strafesnet_rbx_loader::RunnerError>,
+	pub convert_errors:strafesnet_rbx_loader::RecoverableErrors,
+}
+
+pub fn convert_to_snf(dom:rbx_dom_weak::WeakDom)->Result<ConvertOutput,ConvertError>{
+	let model=strafesnet_rbx_loader::Model::new(dom);
+
+	let mut place=strafesnet_rbx_loader::Place::from(model);
+	let script_errors=place.run_scripts().unwrap_or_else(|e|vec![e]);
+
+	let (map,convert_errors)=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?;
+
+	let mut snf_buf=Vec::new();
+	strafesnet_snf::map::write_map(Cursor::new(&mut snf_buf),map).map_err(ConvertError::SNFMap)?;
+
+	Ok(ConvertOutput{
+		snf:snf_buf,
+		script_errors,
+		convert_errors,
+	})
+}
+
+#[derive(Debug,thiserror::Error)]
+pub enum ConvertTextureError{
+	#[error("Image error {0:?}")]
+	Image(#[from]image::ImageError),
+	#[error("DDS create error {0:?}")]
+	DDS(#[from]image_dds::CreateDdsError),
+	#[error("DDS write error {0:?}")]
+	DDSWrite(#[from]image_dds::ddsfile::Error),
+}
+pub fn convert_texture_to_dds(data:&[u8])->Result<Vec<u8>,ConvertTextureError>{
+	let image=image::load_from_memory(data)?.to_rgba8();
+
+	let format=if image.width()%4!=0||image.height()%4!=0{
+		image_dds::ImageFormat::Rgba8UnormSrgb
+	}else{
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
+	};
+
+	let dds=image_dds::dds_from_image(
+		&image,
+		format,
+		image_dds::Quality::Slow,
+		image_dds::Mipmaps::GeneratedAutomatic,
+	)?;
+
+	let mut buf=Vec::new();
+	dds.write(&mut Cursor::new(&mut buf))?;
+	Ok(buf)
+}
+
+#[derive(Debug)]
+pub enum DownloadAssetError{
+	Get(rbx_asset::cookie::GetError),
+	IO(std::io::Error),
+}
+impl std::fmt::Display for DownloadAssetError{
+	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+		write!(f,"{self:?}")
+	}
+}
+impl std::error::Error for DownloadAssetError{}
+
+pub async fn download_asset(context:&rbx_asset::cookie::Context,asset_id:u64)->Result<Vec<u8>,DownloadAssetError>{
+	let data=context.get_asset(rbx_asset::cookie::GetAssetRequest{
+		asset_id,
+		version:None,
+	}).await.map_err(DownloadAssetError::Get)?;
+	Ok(data.to_vec().map_err(DownloadAssetError::IO)?)
+}
+
+pub(crate) const BACKOFF_MUL:f32=1.3956124250860895286;//exp(1/3)
+const RETRY_LIMIT:u32=12;
+
+pub enum DownloadRetryResult{
+	Ok(Vec<u8>),
+	TimedOut,
+}
+
+pub async fn download_asset_retry(context:&rbx_asset::cookie::Context,asset_id:u64)->Result<DownloadRetryResult,DownloadAssetError>{
+	let mut retry=0u32;
+	let mut backoff=1000f32;
+	loop{
+		match context.get_asset(rbx_asset::cookie::GetAssetRequest{
+			asset_id,
+			version:None,
+		}).await{
+			Ok(data)=>{
+				let bytes=data.to_vec().map_err(DownloadAssetError::IO)?;
+				break Ok(DownloadRetryResult::Ok(bytes));
+			},
+			Err(rbx_asset::cookie::GetError::Response(rbx_asset::types::ResponseError::Details{status_code,url_and_body}))=>{
+				if status_code.as_u16()==429{
+					if retry==RETRY_LIMIT{
+						break Ok(DownloadRetryResult::TimedOut);
+					}
+					tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
+					backoff*=BACKOFF_MUL;
+					retry+=1;
+				}else{
+					break Err(DownloadAssetError::Get(rbx_asset::cookie::GetError::Response(rbx_asset::types::ResponseError::Details{status_code,url_and_body})));
+				}
+			},
+			Err(e)=>{
+				break Err(DownloadAssetError::Get(e));
+			},
+		}
+	}
+}

map-tool/src/source.rs

@@ -1,514 +0,0 @@
-use std::path::{Path,PathBuf};
-use std::borrow::Cow;
-use clap::{Args,Subcommand};
-use anyhow::Result as AResult;
-use futures::StreamExt;
-use strafesnet_bsp_loader::loader::BspFinder;
-use strafesnet_deferred_loader::loader::Loader;
-use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLoader,RenderConfigDeferredLoader};
-use vbsp_entities_css::Entity;
-
-#[derive(Subcommand)]
-pub enum Commands{
-	SourceToSNF(SourceToSNFSubcommand),
-	ExtractTextures(ExtractTexturesSubcommand),
-	VPKContents(VPKContentsSubcommand),
-	BSPContents(BSPContentsSubcommand),
-}
-
-#[derive(Args)]
-pub struct SourceToSNFSubcommand {
-	#[arg(long)]
-	output_folder:PathBuf,
-	#[arg(required=true)]
-	input_files:Vec<PathBuf>,
-	#[arg(long)]
-	vpks:Vec<PathBuf>,
-}
-#[derive(Args)]
-pub struct ExtractTexturesSubcommand{
-	#[arg(required=true)]
-	bsp_files:Vec<PathBuf>,
-	#[arg(long)]
-	vpks:Vec<PathBuf>,
-}
-#[derive(Args)]
-pub struct VPKContentsSubcommand {
-	#[arg(long)]
-	input_file:PathBuf,
-}
-#[derive(Args)]
-pub struct BSPContentsSubcommand {
-	#[arg(long)]
-	input_file:PathBuf,
-}
-
-impl Commands{
-	pub async fn run(self)->AResult<()>{
-		match self{
-			Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder,subcommand.vpks).await,
-			Commands::ExtractTextures(subcommand)=>extract_textures(subcommand.bsp_files,subcommand.vpks).await,
-			Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
-			Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
-		}
-	}
-}
-
-
-enum VMTContent{
-	VMT(String),
-	VTF(String),
-	Patch(vmt_parser::material::PatchMaterial),
-	Unsupported,//don't want to deal with whatever vmt variant
-	Unresolved,//could not locate a texture because of vmt content
-}
-impl VMTContent{
-	fn vtf(opt:Option<String>)->Self{
-		match opt{
-			Some(s)=>Self::VTF(s),
-			None=>Self::Unresolved,
-		}
-	}
-}
-
-fn get_some_texture(material:vmt_parser::material::Material)->VMTContent{
-	//just grab some texture from somewhere for now
-	match material{
-		vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none
-		vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
-		vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture),
-		vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture),
-		vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture),
-		vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)),
-		vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),//recursive
-		vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture),
-		vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::Refract(mat)=>VMTContent::vtf(mat.base_texture),
-		vmt_parser::material::Material::Modulate(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::DecalModulate(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported,
-		vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat),
-		_=>unreachable!(),
-	}
-}
-
-#[derive(Debug,thiserror::Error)]
-enum GetVMTError{
-	#[error("Bsp error {0:?}")]
-	Bsp(#[from]vbsp::BspError),
-	#[error("Utf8 error {0:?}")]
-	Utf8(#[from]std::str::Utf8Error),
-	#[error("Vdf error {0:?}")]
-	Vdf(#[from]vmt_parser::VdfError),
-	#[error("Vmt not found")]
-	NotFound,
-}
-
-fn get_vmt(finder:BspFinder,search_name:&str)->Result<vmt_parser::material::Material,GetVMTError>{
-	let vmt_data=finder.find(search_name)?.ok_or(GetVMTError::NotFound)?;
-	//decode vmt and then write
-	let vmt_str=core::str::from_utf8(&vmt_data)?;
-	let material=vmt_parser::from_str(vmt_str)?;
-	//println!("vmt material={:?}",material);
-	Ok(material)
-}
-
-#[derive(Debug,thiserror::Error)]
-enum LoadVMTError{
-	#[error("Bsp error {0:?}")]
-	Bsp(#[from]vbsp::BspError),
-	#[error("GetVMT error {0:?}")]
-	GetVMT(#[from]GetVMTError),
-	#[error("FromUtf8 error {0:?}")]
-	FromUtf8(#[from]std::string::FromUtf8Error),
-	#[error("Vdf error {0:?}")]
-	Vdf(#[from]vmt_parser::VdfError),
-	#[error("Vmt unsupported")]
-	Unsupported,
-	#[error("Vmt unresolved")]
-	Unresolved,
-	#[error("Vmt not found")]
-	NotFound,
-}
-fn recursive_vmt_loader<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,material:vmt_parser::material::Material)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
-	where
-		'bsp:'a,
-		'vpk:'a,
-{
-	match get_some_texture(material){
-		VMTContent::VMT(mut s)=>{
-			s.make_ascii_lowercase();
-			recursive_vmt_loader(finder,get_vmt(finder,&s)?)
-		},
-		VMTContent::VTF(s)=>{
-			let mut texture_file_name=PathBuf::from("materials");
-			texture_file_name.push(s);
-			texture_file_name.set_extension("vtf");
-			texture_file_name.as_mut_os_str().make_ascii_lowercase();
-			Ok(finder.find(texture_file_name.to_str().unwrap())?)
-		},
-		VMTContent::Patch(mat)=>recursive_vmt_loader(finder,
-			mat.resolve(|search_name|{
-				let name_lowercase=search_name.to_lowercase();
-				match finder.find(&name_lowercase)?{
-					Some(bytes)=>Ok(String::from_utf8(bytes.into_owned())?),
-					None=>Err(LoadVMTError::NotFound),
-				}
-			})?
-		),
-		VMTContent::Unsupported=>Err(LoadVMTError::Unsupported),
-		VMTContent::Unresolved=>Err(LoadVMTError::Unresolved),
-	}
-}
-fn load_texture<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,texture_name:&str)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
-	where
-		'bsp:'a,
-		'vpk:'a,
-{
-	let mut texture_file_name=PathBuf::from("materials");
-	//lower case
-	texture_file_name.push(texture_name);
-	texture_file_name.as_mut_os_str().make_ascii_lowercase();
-	//remove stem and search for both vtf and vmt files
-	let stem=texture_file_name.file_stem().unwrap().to_owned();
-	texture_file_name.pop();
-	texture_file_name.push(stem);
-	if let Some(stuff)=finder.find(texture_file_name.to_str().unwrap())?{
-		return Ok(Some(stuff));
-	}
-
-	// search for both vmt,vtf
-	let mut texture_file_name_vmt=texture_file_name.clone();
-	texture_file_name_vmt.set_extension("vmt");
-
-	let get_vmt_result=get_vmt(finder,texture_file_name_vmt.to_str().unwrap());
-	match get_vmt_result{
-		Ok(material)=>{
-			let vmt_result=recursive_vmt_loader(finder,material);
-			match vmt_result{
-				Ok(Some(stuff))=>return Ok(Some(stuff)),
-				Ok(None)
-				|Err(LoadVMTError::NotFound)=>(),
-				|Err(LoadVMTError::GetVMT(GetVMTError::NotFound))=>(),
-				Err(e)=>return Err(e),
-			}
-		}
-		|Err(GetVMTError::NotFound)=>(),
-		Err(e)=>Err(e)?,
-	}
-
-	// try looking for vtf
-	let mut texture_file_name_vtf=texture_file_name.clone();
-	texture_file_name_vtf.set_extension("vtf");
-
-	let get_vtf_result=get_vmt(finder,texture_file_name_vtf.to_str().unwrap());
-	match get_vtf_result{
-		Ok(material)=>{
-			let vtf_result=recursive_vmt_loader(finder,material);
-			match vtf_result{
-				Ok(Some(stuff))=>return Ok(Some(stuff)),
-				Ok(None)
-				|Err(LoadVMTError::NotFound)=>(),
-				|Err(LoadVMTError::GetVMT(GetVMTError::NotFound))=>(),
-				Err(e)=>return Err(e),
-			}
-		}
-		|Err(GetVMTError::NotFound)=>(),
-		Err(e)=>Err(e)?,
-	}
-
-	Ok(None)
-}
-#[derive(Debug,thiserror::Error)]
-enum ExtractTextureError{
-	#[error("Io error {0:?}")]
-	Io(#[from]std::io::Error),
-	#[error("Bsp error {0:?}")]
-	Bsp(#[from]vbsp::BspError),
-	#[error("MeshLoad error {0:?}")]
-	MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError),
-	#[error("Load VMT error {0:?}")]
-	LoadVMT(#[from]LoadVMTError),
-}
-async fn gimme_them_textures(path:&Path,vpk_list:&[strafesnet_bsp_loader::Vpk],send_texture:tokio::sync::mpsc::Sender<(Vec<u8>,String)>)->Result<(),ExtractTextureError>{
-	let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?;
-	let loader_bsp=strafesnet_bsp_loader::Bsp::new(bsp);
-	let bsp=loader_bsp.as_ref();
-
-	let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
-	for texture in bsp.textures(){
-		texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name())));
-	}
-
-	let mut mesh_deferred_loader=MeshDeferredLoader::new();
-	for name in &bsp.static_props.dict.name{
-		mesh_deferred_loader.acquire_mesh_id(name.as_str());
-	}
-
-	for raw_ent in &bsp.entities{
-		let model=match raw_ent.parse(){
-			Ok(Entity::Cycler(brush))=>brush.model,
-			Ok(Entity::EnvSprite(brush))=>brush.model,
-			Ok(Entity::FuncBreakable(brush))=>brush.model,
-			Ok(Entity::FuncBrush(brush))=>brush.model,
-			Ok(Entity::FuncButton(brush))=>brush.model,
-			Ok(Entity::FuncDoor(brush))=>brush.model,
-			Ok(Entity::FuncDoorRotating(brush))=>brush.model,
-			Ok(Entity::FuncIllusionary(brush))=>brush.model,
-			Ok(Entity::FuncMonitor(brush))=>brush.model,
-			Ok(Entity::FuncMovelinear(brush))=>brush.model,
-			Ok(Entity::FuncPhysbox(brush))=>brush.model,
-			Ok(Entity::FuncPhysboxMultiplayer(brush))=>brush.model,
-			Ok(Entity::FuncRotButton(brush))=>brush.model,
-			Ok(Entity::FuncRotating(brush))=>brush.model,
-			Ok(Entity::FuncTracktrain(brush))=>brush.model,
-			Ok(Entity::FuncTrain(brush))=>brush.model,
-			Ok(Entity::FuncWall(brush))=>brush.model,
-			Ok(Entity::FuncWallToggle(brush))=>brush.model,
-			Ok(Entity::FuncWaterAnalog(brush))=>brush.model,
-			Ok(Entity::PropDoorRotating(brush))=>brush.model,
-			Ok(Entity::PropDynamic(brush))=>brush.model,
-			Ok(Entity::PropDynamicOverride(brush))=>brush.model,
-			Ok(Entity::PropPhysics(brush))=>brush.model,
-			Ok(Entity::PropPhysicsMultiplayer(brush))=>brush.model,
-			Ok(Entity::PropPhysicsOverride(brush))=>brush.model,
-			Ok(Entity::PropRagdoll(brush))=>brush.model,
-			Ok(Entity::TriggerGravity(brush))=>brush.model,
-			Ok(Entity::TriggerHurt(brush))=>brush.model,
-			Ok(Entity::TriggerLook(brush))=>brush.model,
-			Ok(Entity::TriggerMultiple(brush))=>brush.model.unwrap_or_default(),
-			Ok(Entity::TriggerOnce(brush))=>brush.model,
-			Ok(Entity::TriggerProximity(brush))=>brush.model,
-			Ok(Entity::TriggerPush(brush))=>brush.model,
-			Ok(Entity::TriggerSoundscape(brush))=>brush.model,
-			Ok(Entity::TriggerTeleport(brush))=>brush.model.unwrap_or_default(),
-			Ok(Entity::TriggerVphysicsMotion(brush))=>brush.model,
-			Ok(Entity::TriggerWind(brush))=>brush.model,
-			_=>continue,
-		};
-		match model.chars().next(){
-			Some('*')=>(),
-			_=>{
-				mesh_deferred_loader.acquire_mesh_id(model);
-			},
-		}
-	}
-
-	let finder=BspFinder{
-		bsp:&loader_bsp,
-		vpks:vpk_list
-	};
-
-	let mut mesh_loader=strafesnet_bsp_loader::loader::ModelLoader::new(finder);
-	// load models and collect requested textures
-	for model_path in mesh_deferred_loader.into_indices(){
-		let model:vmdl::Model=match mesh_loader.load(model_path){
-			Ok(model)=>model,
-			Err(e)=>{
-				println!("Model={model_path} Load model error: {e}");
-				continue;
-			},
-		};
-		for texture in model.textures(){
-			for search_path in &texture.search_paths{
-				let mut path=PathBuf::from(search_path.as_str());
-				path.push(texture.name.as_str());
-				let path=path.to_str().unwrap().to_owned();
-				texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path)));
-			}
-		}
-	}
-
-	for texture_path in texture_deferred_loader.into_indices(){
-		match load_texture(finder,&texture_path){
-			Ok(Some(texture))=>send_texture.send(
-				(texture.into_owned(),texture_path.into_owned())
-			).await.unwrap(),
-			Ok(None)=>(),
-			Err(e)=>println!("Texture={texture_path} Load error: {e}"),
-		}
-	}
-
-	Ok(())
-}
-
-
-#[derive(Debug,thiserror::Error)]
-enum ConvertTextureError{
-	#[error("Bsp error {0:?}")]
-	Bsp(#[from]vbsp::BspError),
-	#[error("Vtf error {0:?}")]
-	Vtf(#[from]vtf::Error),
-	#[error("DDS create error {0:?}")]
-	DDS(#[from]image_dds::CreateDdsError),
-	#[error("DDS write error {0:?}")]
-	DDSWrite(#[from]image_dds::ddsfile::Error),
-	#[error("Io error {0:?}")]
-	Io(#[from]std::io::Error),
-}
-
-async fn convert_texture(texture:Vec<u8>,write_file_name:impl AsRef<Path>)->Result<(),ConvertTextureError>{
-	let image=vtf::from_bytes(&texture)?.highres_image.decode(0)?.to_rgba8();
-
-	let format=if image.width()%4!=0||image.height()%4!=0{
-		image_dds::ImageFormat::Rgba8UnormSrgb
-	}else{
-		image_dds::ImageFormat::BC7RgbaUnormSrgb
-	};
-	//this fails if the image dimensions are not a multiple of 4
-	let dds = image_dds::dds_from_image(
-		&image,
-		format,
-		image_dds::Quality::Slow,
-		image_dds::Mipmaps::GeneratedAutomatic,
-	)?;
-
-	//write dds
-	let mut dest=PathBuf::from("textures");
-	dest.push(write_file_name);
-	dest.set_extension("dds");
-	std::fs::create_dir_all(dest.parent().unwrap())?;
-	let mut writer=std::io::BufWriter::new(std::fs::File::create(dest)?);
-	dds.write(&mut writer)?;
-
-	Ok(())
-}
-
-async fn read_vpks(vpk_paths:Vec<PathBuf>,thread_limit:usize)->Vec<strafesnet_bsp_loader::Vpk>{
-	futures::stream::iter(vpk_paths).map(|vpk_path|async{
-		// idk why it doesn't want to pass out the errors but this is fatal anyways
-		tokio::task::spawn_blocking(move||Ok::<_,vpk::Error>(strafesnet_bsp_loader::Vpk::new(vpk::VPK::read(&vpk_path)?))).await.unwrap().unwrap()
-	})
-	.buffer_unordered(thread_limit)
-	.collect().await
-}
-
-async fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
-	tokio::try_join!(
-		tokio::fs::create_dir_all("extracted_textures"),
-		tokio::fs::create_dir_all("textures"),
-		tokio::fs::create_dir_all("meshes"),
-	)?;
-	let thread_limit=std::thread::available_parallelism()?.get();
-
-	// load vpk list and leak for static lifetime
-	let vpk_list:&[strafesnet_bsp_loader::Vpk]=read_vpks(vpk_paths,thread_limit).await.leak();
-
-	let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
-	let mut it=paths.into_iter();
-	let extract_thread=tokio::spawn(async move{
-		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
-		SEM.add_permits(thread_limit);
-		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
-			let send=send_texture.clone();
-			tokio::spawn(async move{
-				let result=gimme_them_textures(&path,vpk_list,send).await;
-				drop(permit);
-				match result{
-					Ok(())=>(),
-					Err(e)=>println!("Map={path:?} Decode error: {e:?}"),
-				}
-			});
-		}
-	});
-
-	// convert images
-	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
-	SEM.add_permits(thread_limit);
-	while let (Ok(permit),Some((data,dest)))=(SEM.acquire().await,recv_texture.recv().await){
-		// TODO: dedup dest?
-		tokio::spawn(async move{
-			let result=convert_texture(data,dest).await;
-			drop(permit);
-			match result{
-				Ok(())=>(),
-				Err(e)=>println!("Convert error: {e:?}"),
-			}
-		});
-	}
-	extract_thread.await?;
-	_=SEM.acquire_many(thread_limit as u32).await?;
-	Ok(())
-}
-
-fn vpk_contents(vpk_path:PathBuf)->AResult<()>{
-	let vpk_index=vpk::VPK::read(&vpk_path)?;
-	for (label,entry) in vpk_index.tree.into_iter(){
-		println!("vpk label={} entry={:?}",label,entry);
-	}
-	Ok(())
-}
-
-fn bsp_contents(path:PathBuf)->AResult<()>{
-	let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
-	for file_name in bsp.pack.into_zip().into_inner().unwrap().file_names(){
-		println!("file_name={:?}",file_name);
-	}
-	Ok(())
-}
-
-#[derive(Debug)]
-#[expect(dead_code)]
-enum ConvertError{
-	IO(std::io::Error),
-	SNFMap(strafesnet_snf::map::Error),
-	BspRead(strafesnet_bsp_loader::ReadError),
-	BspLoad(strafesnet_bsp_loader::LoadError),
-}
-impl std::fmt::Display for ConvertError{
-	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
-		write!(f,"{self:?}")
-	}
-}
-impl std::error::Error for ConvertError{}
-
-async fn convert_to_snf(path:&Path,vpk_list:&[strafesnet_bsp_loader::Vpk],output_folder:PathBuf)->AResult<()>{
-	let entire_file=tokio::fs::read(path).await?;
-
-	let bsp=strafesnet_bsp_loader::read(
-		std::io::Cursor::new(entire_file)
-	).map_err(ConvertError::BspRead)?;
-
-	let map=bsp.to_snf(LoadFailureMode::DefaultToNone,vpk_list).map_err(ConvertError::BspLoad)?;
-
-	let mut dest=output_folder;
-	dest.push(path.file_stem().unwrap());
-	dest.set_extension("snfm");
-	let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
-
-	strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
-
-	Ok(())
-}
-async fn source_to_snf(paths:Vec<PathBuf>,output_folder:PathBuf,vpk_paths:Vec<PathBuf>)->AResult<()>{
-	let start=std::time::Instant::now();
-
-	let thread_limit=std::thread::available_parallelism()?.get();
-
-	// load vpk list and leak for static lifetime
-	let vpk_list:&[strafesnet_bsp_loader::Vpk]=read_vpks(vpk_paths,thread_limit).await.leak();
-
-	let mut it=paths.into_iter();
-	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
-	SEM.add_permits(thread_limit);
-
-	while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
-		let output_folder=output_folder.clone();
-		tokio::spawn(async move{
-			let result=convert_to_snf(path.as_path(),vpk_list,output_folder).await;
-			drop(permit);
-			match result{
-				Ok(())=>(),
-				Err(e)=>println!("Convert error: {e:?}"),
-			}
-		});
-	}
-	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
-
-	println!("elapsed={:?}", start.elapsed());
-	Ok(())
-}

map-tool/src/source/cli.rs

@@ -0,0 +1,300 @@
+use std::path::PathBuf;
+use std::borrow::Cow;
+use strafesnet_deferred_loader::loader::Loader;
+use strafesnet_deferred_loader::deferred_loader::{MeshDeferredLoader,RenderConfigDeferredLoader};
+use vbsp_entities_css::Entity;
+
+use anyhow::Result as AResult;
+use futures::StreamExt;
+
+use super::{convert_texture_to_dds,convert_to_snf,load_texture};
+use super::{BspFinder,ConvertTextureError,LoadVMTError};
+
+#[derive(clap::Subcommand)]
+pub enum Commands{
+	SourceToSNF(SourceToSNFSubcommand),
+	ExtractTextures(ExtractTexturesSubcommand),
+	VPKContents(VPKContentsSubcommand),
+	BSPContents(BSPContentsSubcommand),
+}
+
+#[derive(clap::Args)]
+pub struct SourceToSNFSubcommand{
+	#[arg(long)]
+	output_folder:PathBuf,
+	#[arg(required=true)]
+	input_files:Vec<PathBuf>,
+	#[arg(long)]
+	vpks:Vec<PathBuf>,
+}
+#[derive(clap::Args)]
+pub struct ExtractTexturesSubcommand{
+	#[arg(required=true)]
+	bsp_files:Vec<PathBuf>,
+	#[arg(long)]
+	vpks:Vec<PathBuf>,
+}
+#[derive(clap::Args)]
+pub struct VPKContentsSubcommand{
+	#[arg(long)]
+	input_file:PathBuf,
+}
+#[derive(clap::Args)]
+pub struct BSPContentsSubcommand{
+	#[arg(long)]
+	input_file:PathBuf,
+}
+
+impl Commands{
+	pub async fn run(self)->AResult<()>{
+		match self{
+			Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder,subcommand.vpks).await,
+			Commands::ExtractTextures(subcommand)=>extract_textures(subcommand.bsp_files,subcommand.vpks).await,
+			Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
+			Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
+		}
+	}
+}
+
+#[derive(Debug,thiserror::Error)]
+enum ExtractTextureError{
+	#[error("Io error {0:?}")]
+	Io(#[from]std::io::Error),
+	#[error("Bsp error {0:?}")]
+	Bsp(#[from]vbsp::BspError),
+	#[error("MeshLoad error {0:?}")]
+	MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError),
+	#[error("Load VMT error {0:?}")]
+	LoadVMT(#[from]LoadVMTError),
+}
+async fn gimme_them_textures(path:&std::path::Path,vpk_list:&[strafesnet_bsp_loader::Vpk],send_texture:tokio::sync::mpsc::Sender<(Vec<u8>,String)>)->Result<(),ExtractTextureError>{
+	let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?;
+	let loader_bsp=strafesnet_bsp_loader::Bsp::new(bsp);
+	let bsp=loader_bsp.as_ref();
+
+	let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
+	for texture in bsp.textures(){
+		texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name())));
+	}
+
+	let mut mesh_deferred_loader=MeshDeferredLoader::new();
+	for name in &bsp.static_props.dict.name{
+		mesh_deferred_loader.acquire_mesh_id(name.as_str());
+	}
+
+	for raw_ent in &bsp.entities{
+		let model=match raw_ent.parse(){
+			Ok(Entity::Cycler(brush))=>brush.model,
+			Ok(Entity::EnvSprite(brush))=>brush.model,
+			Ok(Entity::FuncBreakable(brush))=>brush.model,
+			Ok(Entity::FuncBrush(brush))=>brush.model,
+			Ok(Entity::FuncButton(brush))=>brush.model,
+			Ok(Entity::FuncDoor(brush))=>brush.model,
+			Ok(Entity::FuncDoorRotating(brush))=>brush.model,
+			Ok(Entity::FuncIllusionary(brush))=>brush.model,
+			Ok(Entity::FuncMonitor(brush))=>brush.model,
+			Ok(Entity::FuncMovelinear(brush))=>brush.model,
+			Ok(Entity::FuncPhysbox(brush))=>brush.model,
+			Ok(Entity::FuncPhysboxMultiplayer(brush))=>brush.model,
+			Ok(Entity::FuncRotButton(brush))=>brush.model,
+			Ok(Entity::FuncRotating(brush))=>brush.model,
+			Ok(Entity::FuncTracktrain(brush))=>brush.model,
+			Ok(Entity::FuncTrain(brush))=>brush.model,
+			Ok(Entity::FuncWall(brush))=>brush.model,
+			Ok(Entity::FuncWallToggle(brush))=>brush.model,
+			Ok(Entity::FuncWaterAnalog(brush))=>brush.model,
+			Ok(Entity::PropDoorRotating(brush))=>brush.model,
+			Ok(Entity::PropDynamic(brush))=>brush.model,
+			Ok(Entity::PropDynamicOverride(brush))=>brush.model,
+			Ok(Entity::PropPhysics(brush))=>brush.model,
+			Ok(Entity::PropPhysicsMultiplayer(brush))=>brush.model,
+			Ok(Entity::PropPhysicsOverride(brush))=>brush.model,
+			Ok(Entity::PropRagdoll(brush))=>brush.model,
+			Ok(Entity::TriggerGravity(brush))=>brush.model,
+			Ok(Entity::TriggerHurt(brush))=>brush.model,
+			Ok(Entity::TriggerLook(brush))=>brush.model,
+			Ok(Entity::TriggerMultiple(brush))=>brush.model.unwrap_or_default(),
+			Ok(Entity::TriggerOnce(brush))=>brush.model,
+			Ok(Entity::TriggerProximity(brush))=>brush.model,
+			Ok(Entity::TriggerPush(brush))=>brush.model,
+			Ok(Entity::TriggerSoundscape(brush))=>brush.model,
+			Ok(Entity::TriggerTeleport(brush))=>brush.model.unwrap_or_default(),
+			Ok(Entity::TriggerVphysicsMotion(brush))=>brush.model,
+			Ok(Entity::TriggerWind(brush))=>brush.model,
+			_=>continue,
+		};
+		match model.chars().next(){
+			Some('*')=>(),
+			_=>{
+				mesh_deferred_loader.acquire_mesh_id(model);
+			},
+		}
+	}
+
+	let finder=BspFinder{
+		bsp:&loader_bsp,
+		vpks:vpk_list
+	};
+
+	let mut mesh_loader=strafesnet_bsp_loader::loader::ModelLoader::new(finder);
+	// load models and collect requested textures
+	for model_path in mesh_deferred_loader.into_indices(){
+		let model:vmdl::Model=match mesh_loader.load(model_path){
+			Ok(model)=>model,
+			Err(e)=>{
+				println!("Model={model_path} Load model error: {e}");
+				continue;
+			},
+		};
+		for texture in model.textures(){
+			for search_path in &texture.search_paths{
+				let mut path=PathBuf::from(search_path.as_str());
+				path.push(texture.name.as_str());
+				let path=path.to_str().unwrap().to_owned();
+				texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path)));
+			}
+		}
+	}
+
+	for texture_path in texture_deferred_loader.into_indices(){
+		match load_texture(finder,&texture_path){
+			Ok(Some(texture))=>send_texture.send(
+				(texture.into_owned(),texture_path.into_owned())
+			).await.unwrap(),
+			Ok(None)=>(),
+			Err(e)=>println!("Texture={texture_path} Load error: {e}"),
+		}
+	}
+
+	Ok(())
+}
+
+#[derive(Debug,thiserror::Error)]
+enum CliConvertTextureError{
+	#[error("Io error {0:?}")]
+	Io(#[from]std::io::Error),
+	#[error("ConvertTexture error {0:?}")]
+	ConvertTexture(#[from]ConvertTextureError),
+}
+
+async fn cli_convert_texture(texture:Vec<u8>,write_file_name:impl AsRef<std::path::Path>)->Result<(),CliConvertTextureError>{
+	let dds=convert_texture_to_dds(&texture)?;
+
+	let mut dest=PathBuf::from("textures");
+	dest.push(write_file_name);
+	dest.set_extension("dds");
+	std::fs::create_dir_all(dest.parent().unwrap())?;
+	std::fs::write(&dest,&dds)?;
+
+	Ok(())
+}
+
+async fn async_read_vpks(vpk_paths:Vec<PathBuf>,thread_limit:usize)->Vec<strafesnet_bsp_loader::Vpk>{
+	futures::stream::iter(vpk_paths).map(|vpk_path|async{
+		tokio::task::spawn_blocking(move||Ok::<_,vpk::Error>(strafesnet_bsp_loader::Vpk::new(vpk::VPK::read(&vpk_path)?))).await.unwrap().unwrap()
+	})
+	.buffer_unordered(thread_limit)
+	.collect().await
+}
+
+async fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
+	tokio::try_join!(
+		tokio::fs::create_dir_all("extracted_textures"),
+		tokio::fs::create_dir_all("textures"),
+		tokio::fs::create_dir_all("meshes"),
+	)?;
+	let thread_limit=std::thread::available_parallelism()?.get();
+
+	// load vpk list and leak for static lifetime
+	let vpk_list:&[strafesnet_bsp_loader::Vpk]=async_read_vpks(vpk_paths,thread_limit).await.leak();
+
+	let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+	let mut it=paths.into_iter();
+	let extract_thread=tokio::spawn(async move{
+		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+		SEM.add_permits(thread_limit);
+		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+			let send=send_texture.clone();
+			tokio::spawn(async move{
+				let result=gimme_them_textures(&path,vpk_list,send).await;
+				drop(permit);
+				match result{
+					Ok(())=>(),
+					Err(e)=>println!("Map={path:?} Decode error: {e:?}"),
+				}
+			});
+		}
+	});
+
+	// convert images
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+	while let (Ok(permit),Some((data,dest)))=(SEM.acquire().await,recv_texture.recv().await){
+		// TODO: dedup dest?
+		tokio::spawn(async move{
+			let result=cli_convert_texture(data,dest).await;
+			drop(permit);
+			match result{
+				Ok(())=>(),
+				Err(e)=>println!("Convert error: {e:?}"),
+			}
+		});
+	}
+	extract_thread.await?;
+	_=SEM.acquire_many(thread_limit as u32).await?;
+	Ok(())
+}
+
+fn vpk_contents(vpk_path:PathBuf)->AResult<()>{
+	let vpk_index=vpk::VPK::read(&vpk_path)?;
+	for (label,entry) in vpk_index.tree.into_iter(){
+		println!("vpk label={} entry={:?}",label,entry);
+	}
+	Ok(())
+}
+
+fn bsp_contents(path:PathBuf)->AResult<()>{
+	let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
+	for file_name in bsp.pack.into_zip().into_inner().unwrap().file_names(){
+		println!("file_name={:?}",file_name);
+	}
+	Ok(())
+}
+
+async fn source_to_snf(paths:Vec<PathBuf>,output_folder:PathBuf,vpk_paths:Vec<PathBuf>)->AResult<()>{
+	let start=std::time::Instant::now();
+
+	let thread_limit=std::thread::available_parallelism()?.get();
+
+	// load vpk list and leak for static lifetime
+	let vpk_list:&[strafesnet_bsp_loader::Vpk]=async_read_vpks(vpk_paths,thread_limit).await.leak();
+
+	let mut it=paths.into_iter();
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+
+	while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+		let output_folder=output_folder.clone();
+		tokio::spawn(async move{
+			let result=match tokio::fs::read(&path).await{
+				Ok(data)=>convert_to_snf(&data,vpk_list).map_err(|e|anyhow::anyhow!("{e:?}")).and_then(|snf_buf|{
+					let mut dest=output_folder;
+					dest.push(path.file_stem().unwrap());
+					dest.set_extension("snfm");
+					std::fs::write(dest,&snf_buf)?;
+					Ok(())
+				}),
+				Err(e)=>Err(e.into()),
+			};
+			drop(permit);
+			match result{
+				Ok(())=>(),
+				Err(e)=>println!("Convert error: {e:?}"),
+			}
+		});
+	}
+	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
+
+	println!("elapsed={:?}", start.elapsed());
+	Ok(())
+}

map-tool/src/source/mod.rs

@@ -0,0 +1,246 @@
+#[cfg(feature="cli")]
+mod cli;
+#[cfg(feature="cli")]
+pub use cli::Commands;
+#[cfg(feature="cli")]
+mod loader;
+
+use std::path::PathBuf;
+use std::borrow::Cow;
+use std::io::Cursor;
+use strafesnet_bsp_loader::loader::BspFinder;
+use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
+
+enum VMTContent{
+	VMT(String),
+	VTF(String),
+	Patch(vmt_parser::material::PatchMaterial),
+	Unsupported,//don't want to deal with whatever vmt variant
+	Unresolved,//could not locate a texture because of vmt content
+}
+impl VMTContent{
+	fn vtf(opt:Option<String>)->Self{
+		match opt{
+			Some(s)=>Self::VTF(s),
+			None=>Self::Unresolved,
+		}
+	}
+}
+
+fn get_some_texture(material:vmt_parser::material::Material)->VMTContent{
+	//just grab some texture from somewhere for now
+	match material{
+		vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none
+		vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
+		vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture),
+		vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture),
+		vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture),
+		vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)),
+		vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),//recursive
+		vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture),
+		vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::Refract(mat)=>VMTContent::vtf(mat.base_texture),
+		vmt_parser::material::Material::Modulate(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::DecalModulate(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported,
+		vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat),
+		_=>unreachable!(),
+	}
+}
+
+#[derive(Debug,thiserror::Error)]
+pub enum GetVMTError{
+	#[error("Bsp error {0:?}")]
+	Bsp(#[from]vbsp::BspError),
+	#[error("Utf8 error {0:?}")]
+	Utf8(#[from]std::str::Utf8Error),
+	#[error("Vdf error {0:?}")]
+	Vdf(#[from]vmt_parser::VdfError),
+	#[error("Vmt not found")]
+	NotFound,
+}
+
+fn get_vmt(finder:BspFinder,search_name:&str)->Result<vmt_parser::material::Material,GetVMTError>{
+	let vmt_data=finder.find(search_name)?.ok_or(GetVMTError::NotFound)?;
+	//decode vmt and then write
+	let vmt_str=core::str::from_utf8(&vmt_data)?;
+	let material=vmt_parser::from_str(vmt_str)?;
+	Ok(material)
+}
+
+#[derive(Debug,thiserror::Error)]
+pub enum LoadVMTError{
+	#[error("Bsp error {0:?}")]
+	Bsp(#[from]vbsp::BspError),
+	#[error("GetVMT error {0:?}")]
+	GetVMT(#[from]GetVMTError),
+	#[error("FromUtf8 error {0:?}")]
+	FromUtf8(#[from]std::string::FromUtf8Error),
+	#[error("Vdf error {0:?}")]
+	Vdf(#[from]vmt_parser::VdfError),
+	#[error("Vmt unsupported")]
+	Unsupported,
+	#[error("Vmt unresolved")]
+	Unresolved,
+	#[error("Vmt not found")]
+	NotFound,
+}
+fn recursive_vmt_loader<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,material:vmt_parser::material::Material)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
+	where
+		'bsp:'a,
+		'vpk:'a,
+{
+	match get_some_texture(material){
+		VMTContent::VMT(mut s)=>{
+			s.make_ascii_lowercase();
+			recursive_vmt_loader(finder,get_vmt(finder,&s)?)
+		},
+		VMTContent::VTF(s)=>{
+			let mut texture_file_name=PathBuf::from("materials");
+			texture_file_name.push(s);
+			texture_file_name.set_extension("vtf");
+			texture_file_name.as_mut_os_str().make_ascii_lowercase();
+			Ok(finder.find(texture_file_name.to_str().unwrap())?)
+		},
+		VMTContent::Patch(mat)=>recursive_vmt_loader(finder,
+			mat.resolve(|search_name|{
+				let name_lowercase=search_name.to_lowercase();
+				match finder.find(&name_lowercase)?{
+					Some(bytes)=>Ok(String::from_utf8(bytes.into_owned())?),
+					None=>Err(LoadVMTError::NotFound),
+				}
+			})?
+		),
+		VMTContent::Unsupported=>Err(LoadVMTError::Unsupported),
+		VMTContent::Unresolved=>Err(LoadVMTError::Unresolved),
+	}
+}
+pub fn load_texture<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,texture_name:&str)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
+	where
+		'bsp:'a,
+		'vpk:'a,
+{
+	let mut texture_file_name=PathBuf::from("materials");
+	//lower case
+	texture_file_name.push(texture_name);
+	texture_file_name.as_mut_os_str().make_ascii_lowercase();
+	//remove stem and search for both vtf and vmt files
+	let stem=texture_file_name.file_stem().unwrap().to_owned();
+	texture_file_name.pop();
+	texture_file_name.push(stem);
+	if let Some(stuff)=finder.find(texture_file_name.to_str().unwrap())?{
+		return Ok(Some(stuff));
+	}
+
+	// search for both vmt,vtf
+	let mut texture_file_name_vmt=texture_file_name.clone();
+	texture_file_name_vmt.set_extension("vmt");
+
+	let get_vmt_result=get_vmt(finder,texture_file_name_vmt.to_str().unwrap());
+	match get_vmt_result{
+		Ok(material)=>{
+			let vmt_result=recursive_vmt_loader(finder,material);
+			match vmt_result{
+				Ok(Some(stuff))=>return Ok(Some(stuff)),
+				Ok(None)
+				|Err(LoadVMTError::NotFound)=>(),
+				|Err(LoadVMTError::GetVMT(GetVMTError::NotFound))=>(),
+				Err(e)=>return Err(e),
+			}
+		}
+		|Err(GetVMTError::NotFound)=>(),
+		Err(e)=>Err(e)?,
+	}
+
+	// try looking for vtf
+	let mut texture_file_name_vtf=texture_file_name.clone();
+	texture_file_name_vtf.set_extension("vtf");
+
+	let get_vtf_result=get_vmt(finder,texture_file_name_vtf.to_str().unwrap());
+	match get_vtf_result{
+		Ok(material)=>{
+			let vtf_result=recursive_vmt_loader(finder,material);
+			match vtf_result{
+				Ok(Some(stuff))=>return Ok(Some(stuff)),
+				Ok(None)
+				|Err(LoadVMTError::NotFound)=>(),
+				|Err(LoadVMTError::GetVMT(GetVMTError::NotFound))=>(),
+				Err(e)=>return Err(e),
+			}
+		}
+		|Err(GetVMTError::NotFound)=>(),
+		Err(e)=>Err(e)?,
+	}
+
+	Ok(None)
+}
+
+#[derive(Debug)]
+pub enum ConvertError{
+	IO(std::io::Error),
+	SNFMap(strafesnet_snf::map::Error),
+	BspRead(strafesnet_bsp_loader::ReadError),
+	BspLoad(strafesnet_bsp_loader::LoadError),
+}
+impl std::fmt::Display for ConvertError{
+	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+		write!(f,"{self:?}")
+	}
+}
+impl std::error::Error for ConvertError{}
+
+pub fn convert_to_snf(bsp_data:&[u8],vpk_list:&[strafesnet_bsp_loader::Vpk])->Result<Vec<u8>,ConvertError>{
+	let bsp=strafesnet_bsp_loader::read(
+		Cursor::new(bsp_data)
+	).map_err(ConvertError::BspRead)?;
+
+	let mut mesh_loader=loader::MeshLoader::new(loader::BspFinder{bsp:self,vpks:vpk_list},&mut texture_deferred_loader);
+	let mut texture_loader=loader::TextureLoader::new();
+	let map=bsp.to_snf(LoadFailureMode::DefaultToNone,mesh_loader,texture_loader).map_err(ConvertError::BspLoad)?;
+
+	let mut snf_buf=Vec::new();
+	strafesnet_snf::map::write_map(Cursor::new(&mut snf_buf),map).map_err(ConvertError::SNFMap)?;
+
+	Ok(snf_buf)
+}
+
+#[derive(Debug,thiserror::Error)]
+pub enum ConvertTextureError{
+	#[error("Vtf error {0:?}")]
+	Vtf(#[from]vtf::Error),
+	#[error("DDS create error {0:?}")]
+	DDS(#[from]image_dds::CreateDdsError),
+	#[error("DDS write error {0:?}")]
+	DDSWrite(#[from]image_dds::ddsfile::Error),
+}
+
+pub fn convert_texture_to_dds(vtf_data:&[u8])->Result<Vec<u8>,ConvertTextureError>{
+	let vtf_vec=vtf_data.to_vec();
+	let image=vtf::from_bytes(&vtf_vec)?.highres_image.decode(0)?.to_rgba8();
+
+	let format=if image.width()%4!=0||image.height()%4!=0{
+		image_dds::ImageFormat::Rgba8UnormSrgb
+	}else{
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
+	};
+	let dds=image_dds::dds_from_image(
+		&image,
+		format,
+		image_dds::Quality::Slow,
+		image_dds::Mipmaps::GeneratedAutomatic,
+	)?;
+
+	let mut buf=Vec::new();
+	dds.write(&mut Cursor::new(&mut buf))?;
+	Ok(buf)
+}
+
+pub fn read_vpks(vpk_paths:&[PathBuf])->Result<Vec<strafesnet_bsp_loader::Vpk>,vpk::Error>{
+	vpk_paths.iter().map(|vpk_path|{
+		Ok(strafesnet_bsp_loader::Vpk::new(vpk::VPK::read(vpk_path)?))
+	}).collect()
+}

strafe-client/src/setup.rs

@@ -19,10 +19,13 @@ pub async fn setup_and_start(title:&str){
 
 	let adapter=setup::step3::pick_adapter(&instance,&surface).await.expect("No suitable GPU adapters found on the system!");
 
-	let (device,queue)=setup::step4::request_device(&adapter).await;
+	let adapter_info=adapter.get_info();
+	println!("Using {} ({:?})", adapter_info.name, adapter_info.backend);
+
+	let (device,queue)=setup::step4::request_device(&adapter).await.unwrap();
 
 	let size=window.inner_size();
-	let config=setup::step5::configure_surface(&adapter,&device,&surface,(size.width,size.height));
+	let config=setup::step5::configure_surface(&adapter,&device,&surface,(size.width,size.height)).unwrap();
 
 	//dedicated thread to ping request redraw back and resize the window doesn't seem logical