Support library usage

Cargo.toml

@@ -1,25 +1,42 @@
 [package]
 name = "map-tool"
-version = "1.6.0"
+version = "2.0.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[lib]
+name = "map_tool"
+path = "src/lib.rs"
+
+[[bin]]
+name = "map-tool"
+path = "src/main.rs"
+required-features = ["cli"]
+
+[features]
+cli = ["dep:clap"]
+
 [dependencies]
 anyhow = "1.0.75"
-clap = { version = "4.4.2", features = ["derive"] }
+clap = { version = "4.4.2", features = ["derive"], optional = true }
 flate2 = "1.0.27"
+futures = "0.3.31"
 image = "0.25.2"
-image_dds = "0.6.0"
+image_dds = "0.7.1"
 lazy-regex = "3.1.0"
+rbx_asset = { version = "0.2.5", registry = "strafesnet" }
 rbx_binary = { version = "0.7.4", registry = "strafesnet" }
 rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
 rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
 rbx_xml = { version = "0.13.3", registry = "strafesnet" }
-strafesnet_bsp_loader = { version = "0.2.1", registry = "strafesnet" }
+rbxassetid = { version = "0.1.0", registry = "strafesnet" }
-strafesnet_deferred_loader = { version = "0.4.0", features = ["legacy"], registry = "strafesnet" }
+strafesnet_bsp_loader = { version = "0.3.0", registry = "strafesnet" }
-strafesnet_rbx_loader = { version = "0.5.1", registry = "strafesnet" }
+strafesnet_deferred_loader = { version = "0.5.0", registry = "strafesnet" }
-strafesnet_snf = { version = "0.2.0", registry = "strafesnet" }
+strafesnet_rbx_loader = { version = "0.6.0", registry = "strafesnet" }
+strafesnet_snf = { version = "0.3.0", registry = "strafesnet" }
+thiserror = "2.0.11"
+tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread", "fs"] }
 vbsp = "0.6.0"
 vmdl = "0.2.0"
 vmt-parser = "0.2.0"

src/common.rs

@@ -1,75 +0,0 @@
-use std::path::PathBuf;
-use std::io::{Read,Seek};
-use anyhow::Result as AResult;
-
-fn load_image<R:Read+Seek+std::io::BufRead>(input:&mut R)->AResult<image::DynamicImage>{
-	let mut fourcc=[0u8;4];
-	input.read_exact(&mut fourcc)?;
-	input.rewind()?;
-	match &fourcc{
-		b"\x89PNG"=>Ok(image::load(input,image::ImageFormat::Png)?),
-		b"\xFF\xD8\xFF\xE0"=>Ok(image::load(input,image::ImageFormat::Jpeg)?),//JFIF
-		b"<rob"=>Err(anyhow::Error::msg("Roblox xml garbage is not supported yet")),
-		other=>Err(anyhow::Error::msg(format!("Unknown texture format {:?}",other))),
-	}
-}
-
-fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
-	let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?);
-
-	let image=load_image(&mut input)?.to_rgba8();//this sets a=255, arcane is actually supposed to look like that
-
-	let format=if image.width()%4!=0||image.height()%4!=0{
-		image_dds::ImageFormat::Rgba8UnormSrgb
-	}else{
-		image_dds::ImageFormat::BC7RgbaUnormSrgb
-	};
-	//this fails if the image dimensions are not a multiple of 4
-	let dds = image_dds::dds_from_image(
-		&image,
-		format,
-		image_dds::Quality::Slow,
-		image_dds::Mipmaps::GeneratedAutomatic,
-	)?;
-
-	//write dds
-	let mut dest=PathBuf::from("textures");
-	dest.push(file_thing.file_name());
-	dest.set_extension("dds");
-	let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
-	dds.write(&mut writer)?;
-
-	//move file to processed
-	let mut dest=PathBuf::from("textures/processed");
-	dest.push(file_thing.file_name());
-	std::fs::rename(file_thing.path(), dest)?;
-	Ok(())
-}
-pub fn convert_textures() -> AResult<()>{
-	std::fs::create_dir_all("textures/unprocessed")?;
-	std::fs::create_dir_all("textures/processed")?;
-	let start = std::time::Instant::now();
-	let mut threads=Vec::new();
-	for entry in std::fs::read_dir("textures/unprocessed")? {
-		let file_thing=entry?;
-		threads.push(std::thread::spawn(move ||{
-			let file_name=format!("{:?}",file_thing);
-			let result=convert(file_thing);
-			if let Err(e)=result{
-				println!("error processing file:{:?} error message:{:?}",file_name,e);
-			}
-		}));
-	}
-	let mut i=0;
-	let n_threads=threads.len();
-	for thread in threads{
-		i+=1;
-		if let Err(e)=thread.join(){
-			println!("thread error: {:?}",e);
-		}else{
-			println!("{}/{}",i,n_threads);
-		}
-	}
-	println!("{:?}", start.elapsed());
-	Ok(())
-}

src/lib.rs

@@ -0,0 +1,2 @@
+pub mod roblox;
+pub mod source;

src/main.rs

@@ -1,7 +1,3 @@
-mod common;
-mod roblox;
-mod source;
-
 use clap::{Parser,Subcommand};
 use anyhow::Result as AResult;
 
@@ -16,17 +12,16 @@ struct Cli {
 #[derive(Subcommand)]
 enum Commands{
 	#[command(flatten)]
-	Roblox(roblox::Commands),
+	Roblox(map_tool::roblox::Commands),
 	#[command(flatten)]
-	Source(source::Commands),
+	Source(map_tool::source::Commands),
-	ConvertTextures,
 }
 
-fn main() -> AResult<()> {
+#[tokio::main]
+async fn main()->AResult<()>{
 	let cli=Cli::parse();
 	match cli.command{
-		Commands::Roblox(commands)=>commands.run(),
+		Commands::Roblox(commands)=>commands.run().await,
-		Commands::Source(commands)=>commands.run(),
+		Commands::Source(commands)=>commands.run().await,
-		Commands::ConvertTextures=>common::convert_textures(),
 	}
 }

src/roblox.rs

@@ -1,100 +1,154 @@
-use std::path::{Path,PathBuf};
 use std::io::{Cursor,Read,Seek};
 use std::collections::HashSet;
-use clap::{Args,Subcommand};
-use anyhow::Result as AResult;
 use rbx_dom_weak::Instance;
-use strafesnet_deferred_loader::rbxassetid::RobloxAssetId;
+use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
+use rbxassetid::RobloxAssetId;
 
-#[derive(Subcommand)]
+// === Public library API ===
-pub enum Commands{
+
-	RobloxToSNF(RobloxToSNFSubcommand),
+/// Unique asset IDs referenced by a Roblox place/model file.
-	DownloadAssets(DownloadAssetsSubcommand),
+#[derive(Default)]
+pub struct UniqueAssets{
+	pub meshes:HashSet<RobloxAssetId>,
+	pub unions:HashSet<RobloxAssetId>,
+	pub textures:HashSet<RobloxAssetId>,
 }
 
-#[derive(Args)]
+#[derive(Debug,thiserror::Error)]
-pub struct RobloxToSNFSubcommand {
+pub enum LoadDomError{
-	#[arg(long)]
+	#[error("IO error {0:?}")]
-	output_folder:PathBuf,
+	IO(#[from]std::io::Error),
-	#[arg(required=true)]
+	#[error("Binary decode error {0:?}")]
-	input_files:Vec<PathBuf>,
+	Binary(rbx_binary::DecodeError),
-}
+	#[error("XML decode error {0:?}")]
-#[derive(Args)]
+	Xml(rbx_xml::DecodeError),
-pub struct DownloadAssetsSubcommand{
+	#[error("Unknown file format")]
-	#[arg(long,required=true)]
+	UnknownFormat,
-	roblox_files:Vec<PathBuf>
 }
 
-impl Commands{
+/// Parse a Roblox file (binary or XML) from bytes into a WeakDom.
-	pub fn run(self)->AResult<()>{
+pub fn load_dom(data:&[u8])->Result<rbx_dom_weak::WeakDom,LoadDomError>{
-		match self{
+	load_dom_reader(Cursor::new(data))
-			Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
-			Commands::DownloadAssets(subcommand)=>download_assets(subcommand.roblox_files),
-		}
-	}
 }
 
-fn load_dom<R:Read+Seek>(mut input:R)->AResult<rbx_dom_weak::WeakDom>{
+fn load_dom_reader<R:Read+Seek>(mut input:R)->Result<rbx_dom_weak::WeakDom,LoadDomError>{
 	let mut first_8=[0u8;8];
 	input.read_exact(&mut first_8)?;
 	input.rewind()?;
 	match &first_8{
-		b"<roblox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg),
+		b"<roblox!"=>rbx_binary::from_reader(input).map_err(LoadDomError::Binary),
-		b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg),
+		b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(LoadDomError::Xml),
-		_=>Err(anyhow::Error::msg("unsupported file type")),
+		_=>Err(LoadDomError::UnknownFormat),
 	}
 }
 
-/* The ones I'm interested in:
+/// Scan a parsed DOM and return all unique asset IDs (meshes, textures, unions).
-Beam.Texture
+pub fn get_unique_assets(dom:rbx_dom_weak::WeakDom)->UniqueAssets{
-Decal.Texture
+	let mut assets=UniqueAssets::default();
-FileMesh.MeshId
+	for object in dom.into_raw().1.into_values(){
-FileMesh.TextureId
+		assets.collect(&object);
-MaterialVariant.ColorMap
+	}
-MaterialVariant.MetalnessMap
+	assets
-MaterialVariant.NormalMap
+}
-MaterialVariant.RoughnessMap
+
-MeshPart.MeshId
+/// Scan a Roblox file (bytes) and return all unique asset IDs.
-MeshPart.TextureID
+pub fn get_unique_assets_from_file(data:&[u8])->Result<UniqueAssets,LoadDomError>{
-ParticleEmitter.Texture
+	let dom=load_dom(data)?;
-Sky.MoonTextureId
+	Ok(get_unique_assets(dom))
-Sky.SkyboxBk
+}
-Sky.SkyboxDn
+
-Sky.SkyboxFt
+#[derive(Debug,thiserror::Error)]
-Sky.SkyboxLf
+pub enum ConvertTextureError{
-Sky.SkyboxRt
+	#[error("Image error {0:?}")]
-Sky.SkyboxUp
+	Image(#[from]image::ImageError),
-Sky.SunTextureId
+	#[error("DDS create error {0:?}")]
-SurfaceAppearance.ColorMap
+	DDS(#[from]image_dds::CreateDdsError),
-SurfaceAppearance.MetalnessMap
+	#[error("DDS write error {0:?}")]
-SurfaceAppearance.NormalMap
+	DDSWrite(#[from]image_dds::ddsfile::Error),
-SurfaceAppearance.RoughnessMap
+}
-SurfaceAppearance.TexturePack
+
-*/
+/// Convert image bytes (PNG, JPEG, etc.) into DDS texture bytes.
-fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instance,property:&str){
+pub fn convert_texture_to_dds(image_data:&[u8])->Result<Vec<u8>,ConvertTextureError>{
-	if let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(property){
+	let image=image::load_from_memory(image_data)?.to_rgba8();
-		let url:&str=content.as_ref();
+
-		if let Ok(asset_id)=url.parse(){
+	let format=if image.width()%4!=0||image.height()%4!=0{
-			content_list.insert(asset_id);
+		image_dds::ImageFormat::Rgba8UnormSrgb
 	}else{
-			println!("Content failed to parse into AssetID: {:?}",content);
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
+	};
+
+	let dds=image_dds::dds_from_image(
+		&image,
+		format,
+		image_dds::Quality::Slow,
+		image_dds::Mipmaps::GeneratedAutomatic,
+	)?;
+
+	let mut buf=Vec::new();
+	dds.write(&mut Cursor::new(&mut buf))?;
+	Ok(buf)
 }
+
+#[derive(Debug,thiserror::Error)]
+pub enum ConvertError{
+	#[error("IO error {0:?}")]
+	IO(#[from]std::io::Error),
+	#[error("SNF map error {0:?}")]
+	SNFMap(strafesnet_snf::map::Error),
+	#[error("Roblox read error {0:?}")]
+	RobloxRead(strafesnet_rbx_loader::ReadError),
+	#[error("Roblox load error {0:?}")]
+	RobloxLoad(strafesnet_rbx_loader::LoadError),
+}
+
+/// Convert a Roblox place/model file (bytes) to SNF map format (bytes).
+pub fn convert_to_snf(data:&[u8])->Result<Vec<u8>,ConvertError>{
+	let model=strafesnet_rbx_loader::read(
+		Cursor::new(data)
+	).map_err(ConvertError::RobloxRead)?;
+
+	let mut place=model.into_place();
+	place.run_scripts();
+
+	let map=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?;
+
+	let mut buf=Vec::new();
+	strafesnet_snf::map::write_map(Cursor::new(&mut buf),map).map_err(ConvertError::SNFMap)?;
+	Ok(buf)
+}
+
+/// Download a single asset from Roblox by ID. Returns raw asset bytes.
+pub async fn download_asset(context:&rbx_asset::cookie::CookieContext,asset_id:u64)->Result<Vec<u8>,rbx_asset::cookie::GetError>{
+	context.get_asset(rbx_asset::cookie::GetAssetRequest{
+		asset_id,
+		version:None,
+	}).await
+}
+
+/// Download a single asset with retry and exponential backoff for rate limiting.
+/// Returns None if all retries are exhausted or a non-rate-limit error occurs.
+pub async fn download_asset_retry(context:&rbx_asset::cookie::CookieContext,asset_id:u64)->Option<Vec<u8>>{
+	const BACKOFF_MUL:f32=1.3956124250860895286;
+	let mut backoff=1000f32;
+	for _ in 0..12{
+		match download_asset(context,asset_id).await{
+			Ok(data)=>return Some(data),
+			Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{
+				if scwuab.status_code.as_u16()==429{
+					tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
+					backoff*=BACKOFF_MUL;
 				}else{
-		println!("property={} does not exist for class={}",object.class.as_str(),property);
+					return None;
+				}
+			},
+			Err(_)=>return None,
 		}
 	}
-fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
+	None
-	let mut file=std::fs::File::open(path)?;
-	let mut data=Vec::new();
-	file.read_to_end(&mut data)?;
-	Ok(Cursor::new(data))
-}
-#[derive(Default)]
-struct UniqueAssets{
-	meshes:HashSet<RobloxAssetId>,
-	unions:HashSet<RobloxAssetId>,
-	textures:HashSet<RobloxAssetId>,
 }
+
+// === Private helpers ===
+
 impl UniqueAssets{
 	fn collect(&mut self,object:&Instance){
 		match object.class.as_str(){
@@ -123,118 +177,281 @@ impl UniqueAssets{
 		}
 	}
 }
-fn unique_assets(path:&Path)->AResult<UniqueAssets>{
+
-	// read entire file
+fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instance,property:&str){
-	let mut assets=UniqueAssets::default();
+	if let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(property){
-	let data=read_entire_file(path)?;
+		let url:&str=content.as_ref();
-	let dom=load_dom(data)?;
+		if let Ok(asset_id)=url.parse(){
-	for object in dom.into_raw().1.into_values(){
+			content_list.insert(asset_id);
-		assets.collect(&object);
+		}else{
+			println!("Content failed to parse into AssetID: {:?}",content);
 		}
-	Ok(assets)
+	}else{
+		println!("property={} does not exist for class={}",property,object.class.as_str());
 	}
-struct UniqueAssetsResult{
-	path:std::path::PathBuf,
-	result:AResult<UniqueAssets>,
 }
-fn do_thread(path:std::path::PathBuf,send:std::sync::mpsc::Sender<UniqueAssetsResult>){
+
-	std::thread::spawn(move ||{
+// === CLI ===
-		let result=unique_assets(path.as_path());
+
-		send.send(UniqueAssetsResult{
+#[cfg(feature="cli")]
-			path,
+mod cli{
-			result,
+	use super::*;
-		}).unwrap();
+	use std::path::{Path,PathBuf};
+	use clap::{Args,Subcommand};
+	use anyhow::Result as AResult;
+	use tokio::io::AsyncReadExt;
+
+	const DOWNLOAD_LIMIT:usize=16;
+
+	#[derive(Subcommand)]
+	pub enum Commands{
+		RobloxToSNF(RobloxToSNFSubcommand),
+		DownloadAssets(DownloadAssetsSubcommand),
+	}
+
+	#[derive(Args)]
+	pub struct RobloxToSNFSubcommand {
+		#[arg(long)]
+		output_folder:PathBuf,
+		#[arg(required=true)]
+		input_files:Vec<PathBuf>,
+	}
+	#[derive(Args)]
+	pub struct DownloadAssetsSubcommand{
+		#[arg(required=true)]
+		roblox_files:Vec<PathBuf>,
+	}
+
+	impl Commands{
+		pub async fn run(self)->AResult<()>{
+			match self{
+				Commands::RobloxToSNF(subcommand)=>cli_roblox_to_snf(subcommand.input_files,subcommand.output_folder).await,
+				Commands::DownloadAssets(subcommand)=>cli_download_assets(
+					subcommand.roblox_files,
+					rbx_asset::cookie::Cookie::new("".to_string()),
+				).await,
+			}
+		}
+	}
+
+	async fn read_entire_file(path:impl AsRef<Path>)->Result<Vec<u8>,std::io::Error>{
+		let mut file=tokio::fs::File::open(path).await?;
+		let mut data=Vec::new();
+		file.read_to_end(&mut data).await?;
+		Ok(data)
+	}
+
+	enum DownloadType{
+		Texture(RobloxAssetId),
+		Mesh(RobloxAssetId),
+		Union(RobloxAssetId),
+	}
+	impl DownloadType{
+		fn path(&self)->PathBuf{
+			match self{
+				DownloadType::Texture(asset_id)=>format!("downloaded_textures/{}",asset_id.0.to_string()).into(),
+				DownloadType::Mesh(asset_id)=>format!("meshes/{}",asset_id.0.to_string()).into(),
+				DownloadType::Union(asset_id)=>format!("unions/{}",asset_id.0.to_string()).into(),
+			}
+		}
+		fn asset_id(&self)->u64{
+			match self{
+				DownloadType::Texture(asset_id)=>asset_id.0,
+				DownloadType::Mesh(asset_id)=>asset_id.0,
+				DownloadType::Union(asset_id)=>asset_id.0,
+			}
+		}
+	}
+	enum DownloadResult{
+		Cached(PathBuf),
+		Data(Vec<u8>),
+		Failed,
+	}
+	#[derive(Default,Debug)]
+	struct Stats{
+		total_assets:u32,
+		cached_assets:u32,
+		downloaded_assets:u32,
+		failed_downloads:u32,
+		timed_out_downloads:u32,
+	}
+	async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
+		stats.total_assets+=1;
+		let path=download_instruction.path();
+		if tokio::fs::try_exists(path.as_path()).await?{
+			stats.cached_assets+=1;
+			return Ok(DownloadResult::Cached(path));
+		}
+		let asset_id=download_instruction.asset_id();
+		let mut retry=0;
+		const BACKOFF_MUL:f32=1.3956124250860895286;
+		let mut backoff=1000f32;
+		loop{
+			let asset_result=context.get_asset(rbx_asset::cookie::GetAssetRequest{
+				asset_id,
+				version:None,
+			}).await;
+			match asset_result{
+				Ok(asset_result)=>{
+					stats.downloaded_assets+=1;
+					tokio::fs::write(path,&asset_result).await?;
+					break Ok(DownloadResult::Data(asset_result));
+				},
+				Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{
+					if scwuab.status_code.as_u16()==429{
+						if retry==12{
+							println!("Giving up asset download {asset_id}");
+							stats.timed_out_downloads+=1;
+							break Ok(DownloadResult::Failed);
+						}
+						println!("Hit roblox rate limit, waiting {:.0}ms...",backoff);
+						tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
+						backoff*=BACKOFF_MUL;
+						retry+=1;
+					}else{
+						stats.failed_downloads+=1;
+						println!("weird scuwab error: {scwuab:?}");
+						break Ok(DownloadResult::Failed);
+					}
+				},
+				Err(e)=>{
+					stats.failed_downloads+=1;
+					println!("sadly error: {e}");
+					break Ok(DownloadResult::Failed);
+				},
+			}
+		}
+	}
+
+	async fn cli_convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),CliConvertTextureError>{
+		let data=match download_result{
+			DownloadResult::Cached(path)=>tokio::fs::read(path).await?,
+			DownloadResult::Data(data)=>data,
+			DownloadResult::Failed=>return Ok(()),
+		};
+		let dds_data=convert_texture_to_dds(&data)?;
+		let file_name=format!("textures/{}.dds",asset_id.0);
+		tokio::fs::write(file_name,dds_data).await?;
+		Ok(())
+	}
+
+	#[derive(Debug,thiserror::Error)]
+	enum CliConvertTextureError{
+		#[error("IO error {0:?}")]
+		Io(#[from]std::io::Error),
+		#[error("Convert texture error {0:?}")]
+		Convert(#[from]ConvertTextureError),
+	}
+
+	async fn cli_download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
+		tokio::try_join!(
+			tokio::fs::create_dir_all("downloaded_textures"),
+			tokio::fs::create_dir_all("textures"),
+			tokio::fs::create_dir_all("meshes"),
+			tokio::fs::create_dir_all("unions"),
+		)?;
+		let thread_limit=std::thread::available_parallelism()?.get();
+		let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
+		let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+		tokio::spawn(async move{
+			let mut it=paths.into_iter();
+			static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+			SEM.add_permits(thread_limit);
+			while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+				let send=send_assets.clone();
+				tokio::spawn(async move{
+					let data=read_entire_file(path.as_path()).await;
+					let result=data.map_err(LoadDomError::from).and_then(|d|{
+						let dom=load_dom(&d)?;
+						Ok(get_unique_assets(dom))
+					});
+					_=send.send(result).await;
+					drop(permit);
 				});
 			}
-fn download_assets(paths:Vec<PathBuf>)->AResult<()>{
+		});
+		let mut stats=Stats::default();
+		let context=rbx_asset::cookie::CookieContext::new(cookie);
+		let mut globally_unique_assets=UniqueAssets::default();
+		let download_thread=tokio::spawn(async move{
+			while let Some(result)=recv_assets.recv().await{
+				let unique_assets=match result{
+					Ok(unique_assets)=>unique_assets,
+					Err(e)=>{
+						println!("error: {e:?}");
+						continue;
+					},
+				};
+				for texture_id in unique_assets.textures{
+					if globally_unique_assets.textures.insert(texture_id){
+						let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
+						send_texture.send((texture_id,data)).await?;
+					}
+				}
+				for mesh_id in unique_assets.meshes{
+					if globally_unique_assets.meshes.insert(mesh_id){
+						download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
+					}
+				}
+				for union_id in unique_assets.unions{
+					if globally_unique_assets.unions.insert(union_id){
+						download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
+					}
+				}
+			}
+			dbg!(stats);
+			Ok::<(),anyhow::Error>(())
+		});
+		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+		SEM.add_permits(thread_limit);
+		while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){
+			tokio::spawn(async move{
+				let result=cli_convert_texture(asset_id,download_result).await;
+				drop(permit);
+				result.unwrap();
+			});
+		}
+		download_thread.await??;
+		_=SEM.acquire_many(thread_limit as u32).await.unwrap();
 		Ok(())
 	}
 
-#[derive(Debug)]
+	async fn cli_roblox_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
-#[allow(dead_code)]
-enum ConvertError{
-	IO(std::io::Error),
-	SNFMap(strafesnet_snf::map::Error),
-	RbxLoader(strafesnet_rbx_loader::ReadError),
-}
-impl std::fmt::Display for ConvertError{
-	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
-		write!(f,"{self:?}")
-	}
-}
-impl std::error::Error for ConvertError{}
-
-type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
-
-fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
-	let n_paths=pathlist.len();
 		let start=std::time::Instant::now();
-	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
+
-	let mut i=0;
+		let thread_limit=std::thread::available_parallelism()?.get();
-	let mut join_thread=|thread:MapThread|{
+		let mut it=paths.into_iter();
-		i+=1;
+		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
-		if let Err(e)=thread.join(){
+		SEM.add_permits(thread_limit);
-			println!("thread error: {:?}",e);
+
-		}else{
+		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
-			println!("{}/{}",i,n_paths);
-		}
-	};
-	for path in pathlist{
-		if 32<=threads.len(){
-			join_thread(threads.pop_front().unwrap());
-		}
 			let output_folder=output_folder.clone();
-		threads.push_back(std::thread::spawn(move ||{
+			tokio::spawn(async move{
-			let model=strafesnet_rbx_loader::read(
+				let result=cli_convert_to_snf(path.as_path(),output_folder).await;
-				std::fs::File::open(path.as_path())
+				drop(permit);
-				.map_err(ConvertError::IO)?
+				match result{
-			).map_err(ConvertError::RbxLoader)?;
+					Ok(())=>(),
+					Err(e)=>println!("Convert error: {e:?}"),
+				}
+			});
+		}
+		_=SEM.acquire_many(thread_limit as u32).await.unwrap();
 
-			let mut place=model.into_place();
+		println!("elapsed={:?}", start.elapsed());
-			place.run_scripts();
+		Ok(())
+	}
 
-			let mut loader=strafesnet_deferred_loader::roblox_legacy();
+	async fn cli_convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
+		let entire_file=tokio::fs::read(path).await?;
+		let snf_data=convert_to_snf(&entire_file)?;
 
-			let (texture_loader,mesh_loader)=loader.get_inner_mut();
+		let mut dest=output_folder;
-
-			let map_step1=strafesnet_rbx_loader::convert(
-				&place,
-				|name|texture_loader.acquire_render_config_id(name),
-				|name|mesh_loader.acquire_mesh_id(name),
-			);
-
-			let meshpart_meshes=mesh_loader.load_meshes().map_err(ConvertError::IO)?;
-
-			let map_step2=map_step1.add_meshpart_meshes_and_calculate_attributes(
-				meshpart_meshes.into_iter().map(|(mesh_id,loader_model)|
-					(mesh_id,strafesnet_rbx_loader::data::RobloxMeshBytes::new(loader_model.get()))
-				)
-			);
-
-			let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
-
-			let map=map_step2.add_render_configs_and_textures(
-				render_configs.into_iter(),
-				textures.into_iter().map(|(texture_id,texture)|
-					(texture_id,match texture{
-						strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
-					})
-				)
-			);
-
-			let mut dest=output_folder.clone();
 		dest.push(path.file_stem().unwrap());
 		dest.set_extension("snfm");
-			let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
+		tokio::fs::write(dest,snf_data).await?;
 
-			strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
-			Ok(())
-		}));
-	}
-
-	for thread in threads{
-		join_thread(thread);
-	}
-	println!("{:?}", start.elapsed());
 		Ok(())
 	}
+}
+#[cfg(feature="cli")]
+pub use cli::Commands;

src/source.rs

@@ -1,6 +1,209 @@
-use std::path::PathBuf;
+use std::io::Cursor;
+use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
+
+// === Public library API ===
+
+#[derive(Debug,thiserror::Error)]
+pub enum ConvertTextureError{
+	#[error("Vtf error {0:?}")]
+	Vtf(#[from]vtf::Error),
+	#[error("DDS create error {0:?}")]
+	DDS(#[from]image_dds::CreateDdsError),
+	#[error("DDS write error {0:?}")]
+	DDSWrite(#[from]image_dds::ddsfile::Error),
+}
+
+/// Convert VTF texture bytes to DDS texture bytes.
+pub fn convert_texture_to_dds(vtf_data:&[u8])->Result<Vec<u8>,ConvertTextureError>{
+	let vtf_data=vtf_data.to_vec();
+	let image=vtf::from_bytes(&vtf_data)?.highres_image.decode(0)?.to_rgba8();
+
+	let format=if image.width()%4!=0||image.height()%4!=0{
+		image_dds::ImageFormat::Rgba8UnormSrgb
+	}else{
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
+	};
+
+	let dds=image_dds::dds_from_image(
+		&image,
+		format,
+		image_dds::Quality::Slow,
+		image_dds::Mipmaps::GeneratedAutomatic,
+	)?;
+
+	let mut buf=Vec::new();
+	dds.write(&mut Cursor::new(&mut buf))?;
+	Ok(buf)
+}
+
+#[derive(Debug,thiserror::Error)]
+pub enum ConvertError{
+	#[error("BSP read error {0:?}")]
+	BspRead(strafesnet_bsp_loader::ReadError),
+	#[error("BSP load error {0:?}")]
+	BspLoad(strafesnet_bsp_loader::LoadError),
+	#[error("SNF map error {0:?}")]
+	SNFMap(strafesnet_snf::map::Error),
+	#[error("BSP parse error {0:?}")]
+	BspParse(#[from]vbsp::BspError),
+}
+
+/// Convert a Source BSP file (bytes) to SNF map format (bytes).
+pub fn convert_to_snf(bsp_data:&[u8],vpk_list:&[vpk::VPK])->Result<Vec<u8>,ConvertError>{
+	let bsp=strafesnet_bsp_loader::read(
+		Cursor::new(bsp_data)
+	).map_err(ConvertError::BspRead)?;
+
+	let map=bsp.to_snf(LoadFailureMode::DefaultToNone,vpk_list).map_err(ConvertError::BspLoad)?;
+
+	let mut buf=Vec::new();
+	strafesnet_snf::map::write_map(Cursor::new(&mut buf),map).map_err(ConvertError::SNFMap)?;
+	Ok(buf)
+}
+
+/// Read VPK archives from paths. Useful for loading VPKs needed by `convert_to_snf`.
+pub async fn read_vpks(vpk_paths:Vec<std::path::PathBuf>,thread_limit:usize)->Vec<vpk::VPK>{
+	use futures::StreamExt;
+	futures::stream::iter(vpk_paths).map(|vpk_path|async{
+		tokio::task::spawn_blocking(move||vpk::VPK::read(&vpk_path)).await.unwrap().unwrap()
+	})
+	.buffer_unordered(thread_limit)
+	.collect().await
+}
+
+// === CLI ===
+
+#[cfg(feature="cli")]
+mod cli{
+	use super::*;
+	use std::path::{Path,PathBuf};
+	use std::borrow::Cow;
 	use clap::{Args,Subcommand};
 	use anyhow::Result as AResult;
+	use strafesnet_bsp_loader::loader::BspFinder;
+	use strafesnet_deferred_loader::loader::Loader;
+	use strafesnet_deferred_loader::deferred_loader::{MeshDeferredLoader,RenderConfigDeferredLoader};
+
+	enum VMTContent{
+		VMT(String),
+		VTF(String),
+		Patch(vmt_parser::material::PatchMaterial),
+		Unsupported,
+		Unresolved,
+	}
+	impl VMTContent{
+		fn vtf(opt:Option<String>)->Self{
+			match opt{
+				Some(s)=>Self::VTF(s),
+				None=>Self::Unresolved,
+			}
+		}
+	}
+
+	fn get_some_texture(material:vmt_parser::material::Material)->VMTContent{
+		match material{
+			vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+			vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
+			vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
+			vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture),
+			vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture),
+			vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture),
+			vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+			vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)),
+			vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),
+			vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+			vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture),
+			vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+			vmt_parser::material::Material::Refract(mat)=>VMTContent::vtf(mat.base_texture),
+			vmt_parser::material::Material::Modulate(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+			vmt_parser::material::Material::DecalModulate(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+			vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+			vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported,
+			vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat),
+			_=>unreachable!(),
+		}
+	}
+
+	#[derive(Debug,thiserror::Error)]
+	enum GetVMTError{
+		#[error("Bsp error {0:?}")]
+		Bsp(#[from]vbsp::BspError),
+		#[error("Utf8 error {0:?}")]
+		Utf8(#[from]std::str::Utf8Error),
+		#[error("Vdf error {0:?}")]
+		Vdf(#[from]vmt_parser::VdfError),
+		#[error("Vmt not found")]
+		NotFound,
+	}
+
+	fn get_vmt(finder:BspFinder,search_name:&str)->Result<vmt_parser::material::Material,GetVMTError>{
+		let vmt_data=finder.find(search_name)?.ok_or(GetVMTError::NotFound)?;
+		let vmt_str=core::str::from_utf8(&vmt_data)?;
+		let material=vmt_parser::from_str(vmt_str)?;
+		Ok(material)
+	}
+
+	#[derive(Debug,thiserror::Error)]
+	enum LoadVMTError{
+		#[error("Bsp error {0:?}")]
+		Bsp(#[from]vbsp::BspError),
+		#[error("GetVMT error {0:?}")]
+		GetVMT(#[from]GetVMTError),
+		#[error("FromUtf8 error {0:?}")]
+		FromUtf8(#[from]std::string::FromUtf8Error),
+		#[error("Vdf error {0:?}")]
+		Vdf(#[from]vmt_parser::VdfError),
+		#[error("Vmt unsupported")]
+		Unsupported,
+		#[error("Vmt unresolved")]
+		Unresolved,
+		#[error("Vmt not found")]
+		NotFound,
+	}
+	fn recursive_vmt_loader<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,material:vmt_parser::material::Material)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
+		where
+			'bsp:'a,
+			'vpk:'a,
+	{
+		match get_some_texture(material){
+			VMTContent::VMT(s)=>recursive_vmt_loader(finder,get_vmt(finder,s.as_str())?),
+			VMTContent::VTF(s)=>{
+				let mut texture_file_name=PathBuf::from("materials");
+				texture_file_name.push(s);
+				texture_file_name.set_extension("vtf");
+				Ok(finder.find(texture_file_name.to_str().unwrap())?)
+			},
+			VMTContent::Patch(mat)=>recursive_vmt_loader(finder,
+				mat.resolve(|search_name|
+					match finder.find(search_name)?{
+						Some(bytes)=>Ok(String::from_utf8(bytes.into_owned())?),
+						None=>Err(LoadVMTError::NotFound),
+					}
+				)?
+			),
+			VMTContent::Unsupported=>Err(LoadVMTError::Unsupported),
+			VMTContent::Unresolved=>Err(LoadVMTError::Unresolved),
+		}
+	}
+	fn load_texture<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,texture_name:&str)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
+		where
+			'bsp:'a,
+			'vpk:'a,
+	{
+		let mut texture_file_name=PathBuf::from("materials");
+		let texture_file_name_lowercase=texture_name.to_lowercase();
+		texture_file_name.push(texture_file_name_lowercase.clone());
+		let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
+		texture_file_name.pop();
+		texture_file_name.push(stem);
+		if let Some(stuff)=finder.find(texture_file_name.to_str().unwrap())?{
+			return Ok(Some(stuff))
+		}
+		let mut texture_file_name_vmt=texture_file_name.clone();
+		texture_file_name.set_extension("vtf");
+		texture_file_name_vmt.set_extension("vmt");
+		recursive_vmt_loader(finder,get_vmt(finder,texture_file_name_vmt.to_str().unwrap())?)
+	}
 
 	#[derive(Subcommand)]
 	pub enum Commands{
@@ -16,13 +219,15 @@ pub struct SourceToSNFSubcommand {
 		output_folder:PathBuf,
 		#[arg(required=true)]
 		input_files:Vec<PathBuf>,
+		#[arg(long)]
+		vpks:Vec<PathBuf>,
 	}
 	#[derive(Args)]
 	pub struct ExtractTexturesSubcommand{
+		#[arg(required=true)]
+		bsp_files:Vec<PathBuf>,
 		#[arg(long)]
-	bsp_file:PathBuf,
+		vpks:Vec<PathBuf>,
-	#[arg(long)]
-	vpk_dir_files:Vec<PathBuf>
 	}
 	#[derive(Args)]
 	pub struct VPKContentsSubcommand {
@@ -36,223 +241,143 @@ pub struct BSPContentsSubcommand {
 	}
 
 	impl Commands{
-	pub fn run(self)->AResult<()>{
+		pub async fn run(self)->AResult<()>{
 			match self{
-			Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder),
+				Commands::SourceToSNF(subcommand)=>cli_source_to_snf(subcommand.input_files,subcommand.output_folder,subcommand.vpks).await,
-			Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
+				Commands::ExtractTextures(subcommand)=>cli_extract_textures(subcommand.bsp_files,subcommand.vpks).await,
 				Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
 				Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
 			}
 		}
 	}
 
+	#[derive(Debug,thiserror::Error)]
+	enum ExtractTextureError{
+		#[error("Io error {0:?}")]
+		Io(#[from]std::io::Error),
+		#[error("Bsp error {0:?}")]
+		Bsp(#[from]vbsp::BspError),
+		#[error("MeshLoad error {0:?}")]
+		MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError),
+		#[error("Load VMT error {0:?}")]
+		LoadVMT(#[from]LoadVMTError),
+	}
+	async fn gimme_them_textures(path:&Path,vpk_list:&[vpk::VPK],send_texture:tokio::sync::mpsc::Sender<(Vec<u8>,String)>)->Result<(),ExtractTextureError>{
+		let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?;
+		let loader_bsp=strafesnet_bsp_loader::Bsp::new(bsp);
+		let bsp=loader_bsp.as_ref();
 
-enum VMTContent{
+		let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
-	VMT(String),
-	VTF(String),
-	Patch(vmt_parser::material::PatchMaterial),
-	Unsupported,//don't want to deal with whatever vmt variant
-	Unresolved,//could not locate a texture because of vmt content
-}
-impl VMTContent{
-	fn vtf(opt:Option<String>)->Self{
-		match opt{
-			Some(s)=>Self::VTF(s),
-			None=>Self::Unresolved,
-		}
-	}
-}
-
-fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent>{
-	//just grab some texture from somewhere for now
-	Ok(match material{
-		vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none
-		vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
-		vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture),
-		vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture),
-		vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture),
-		vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)),
-		vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),//recursive
-		vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture),
-		vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::Refract(mat)=>VMTContent::vtf(mat.base_texture),
-		vmt_parser::material::Material::Modulate(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::DecalModulate(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-		vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported,
-		vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat),
-		_=>return Err(anyhow::Error::msg("vmt failed to parse")),
-	})
-}
-
-fn get_vmt<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,search_name:String)->AResult<vmt_parser::material::Material>{
-	if let Some(stuff)=find_stuff(search_name)?{
-		//decode vmt and then write
-		let stuff=String::from_utf8(stuff)?;
-		let material=vmt_parser::from_str(stuff.as_str())?;
-		println!("vmt material={:?}",material);
-		return Ok(material);
-	}
-	Err(anyhow::Error::msg("vmt not found"))
-}
-
-fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,material:vmt_parser::material::Material)->AResult<Option<Vec<u8>>>{
-	match get_some_texture(material)?{
-		VMTContent::VMT(s)=>recursive_vmt_loader(find_stuff,get_vmt(find_stuff,s)?),
-		VMTContent::VTF(s)=>{
-			let mut texture_file_name=PathBuf::from("materials");
-			texture_file_name.push(s);
-			texture_file_name.set_extension("vtf");
-			find_stuff(texture_file_name.into_os_string().into_string().unwrap())
-		},
-		VMTContent::Patch(mat)=>recursive_vmt_loader(find_stuff,
-			mat.resolve(|search_name|{
-				match find_stuff(search_name.to_string())?{
-					Some(bytes)=>Ok(String::from_utf8(bytes)?),
-					None=>Err(anyhow::Error::msg("could not find vmt")),
-				}
-			})?
-		),
-		VMTContent::Unsupported=>{println!("Unsupported vmt");Ok(None)},//print and move on
-		VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)},
-	}
-}
-
-fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
-	std::fs::create_dir_all("textures")?;
-	let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
-	for path in paths{
-		let mut deduplicate=std::collections::HashSet::new();
-		let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
 		for texture in bsp.textures(){
-			deduplicate.insert(PathBuf::from(texture.name()));
+			texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name())));
-		}
-		//dedupe prop models
-		let mut model_dedupe=std::collections::HashSet::new();
-		for prop in bsp.static_props(){
-			model_dedupe.insert(prop.model());
 		}
 
-		//grab texture names from props
+		let mut mesh_deferred_loader=MeshDeferredLoader::new();
-		for model_name in model_dedupe{
+		for prop in bsp.static_props(){
-			//.mdl, .vvd, .dx90.vtx
+			mesh_deferred_loader.acquire_mesh_id(prop.model());
-			let mut path=PathBuf::from(model_name);
+		}
-			let file_name=PathBuf::from(path.file_stem().unwrap());
+
-			path.pop();
+		let finder=BspFinder{
-			path.push(file_name);
+			bsp:&loader_bsp,
-			let mut vvd_path=path.clone();
+			vpks:vpk_list
-			let mut vtx_path=path.clone();
+		};
-			vvd_path.set_extension("vvd");
+
-			vtx_path.set_extension("dx90.vtx");
+		let mut mesh_loader=strafesnet_bsp_loader::loader::ModelLoader::new(finder);
-			match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){
+		for model_path in mesh_deferred_loader.into_indices(){
-				(Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{
+			let model:vmdl::Model=match mesh_loader.load(model_path){
-					match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){
+				Ok(model)=>model,
-						(Ok(mdl),Ok(vvd),Ok(vtx))=>{
+				Err(e)=>{
-							let model=vmdl::Model::from_parts(mdl,vtx,vvd);
+					println!("Model={model_path} Load model error: {e}");
+					continue;
+				},
+			};
 			for texture in model.textures(){
 				for search_path in &texture.search_paths{
 					let mut path=PathBuf::from(search_path.as_str());
 					path.push(texture.name.as_str());
-									deduplicate.insert(path);
+					let path=path.to_str().unwrap().to_owned();
+					texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path)));
 				}
 			}
-						},
-						_=>println!("model_name={} error",model_name),
-					}
-				},
-				_=>println!("no model name={}",model_name),
-			}
 		}
 
-		let pack=&bsp.pack;
+		for texture_path in texture_deferred_loader.into_indices(){
-		let vpk_list=&vpk_list;
+			match load_texture(finder,&texture_path){
-		std::thread::scope(move|s|{
+				Ok(Some(texture))=>send_texture.send(
-			let mut thread_handles=Vec::new();
+					(texture.into_owned(),texture_path.into_owned())
-			for texture_name in deduplicate{
+				).await.unwrap(),
-				let mut found_texture=false;
+				Ok(None)=>(),
-				//LMAO imagine having to write type names
+				Err(e)=>println!("Texture={texture_path} Load error: {e}"),
-				let write_image=|mut stuff,write_file_name|{
+			}
-					let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
+		}
 
-					let format=if image.width()%4!=0||image.height()%4!=0{
+		Ok(())
-						image_dds::ImageFormat::Rgba8UnormSrgb
+	}
-					}else{
+
-						image_dds::ImageFormat::BC7RgbaUnormSrgb
+	#[derive(Debug,thiserror::Error)]
-					};
+	enum CliConvertTextureError{
-					//this fails if the image dimensions are not a multiple of 4
+		#[error("IO error {0:?}")]
-					let dds = image_dds::dds_from_image(
+		Io(#[from]std::io::Error),
-						&image,
+		#[error("Convert texture error {0:?}")]
-						format,
+		Convert(#[from]ConvertTextureError),
-						image_dds::Quality::Slow,
+	}
-						image_dds::Mipmaps::GeneratedAutomatic,
+
-					)?;
+	async fn cli_convert_texture(texture:Vec<u8>,write_file_name:impl AsRef<Path>)->Result<(),CliConvertTextureError>{
+		let dds_data=convert_texture_to_dds(&texture)?;
 
-					//write dds
 		let mut dest=PathBuf::from("textures");
 		dest.push(write_file_name);
 		dest.set_extension("dds");
 		std::fs::create_dir_all(dest.parent().unwrap())?;
 		let mut writer=std::io::BufWriter::new(std::fs::File::create(dest)?);
-					dds.write(&mut writer)?;
+		std::io::Write::write_all(&mut writer,&dds_data)?;
-					Ok::<(),anyhow::Error>(())
+
-				};
+		Ok(())
-				let find_stuff=|search_file_name:String|{
-					println!("search_file_name={}",search_file_name);
-					match pack.get(search_file_name.as_str())?{
-						Some(file)=>return Ok(Some(file)),
-						_=>(),
 	}
-					//search pak list
+
-					for vpk_index in vpk_list{
+	async fn cli_extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
-						if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
+		tokio::try_join!(
-							return Ok(Some(match vpk_entry.get()?{
+			tokio::fs::create_dir_all("extracted_textures"),
-								std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
+			tokio::fs::create_dir_all("textures"),
-								std::borrow::Cow::Owned(bytes)=>bytes,
+			tokio::fs::create_dir_all("meshes"),
-							}));
+		)?;
+		let thread_limit=std::thread::available_parallelism()?.get();
+
+		let vpk_list=read_vpks(vpk_paths,thread_limit).await;
+		let vpk_list:&[vpk::VPK]=vpk_list.leak();
+
+		let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+		let mut it=paths.into_iter();
+		let extract_thread=tokio::spawn(async move{
+			static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+			SEM.add_permits(thread_limit);
+			while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+				let send=send_texture.clone();
+				tokio::spawn(async move{
+					let result=gimme_them_textures(&path,vpk_list,send).await;
+					drop(permit);
+					match result{
+						Ok(())=>(),
+						Err(e)=>println!("Map={path:?} Decode error: {e:?}"),
 					}
+				});
 			}
-					Ok::<Option<Vec<u8>>,anyhow::Error>(None)
+		});
-				};
+
-				let loader=|texture_name:String|{
+		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
-					let mut texture_file_name=PathBuf::from("materials");
+		SEM.add_permits(thread_limit);
-					//lower case
+		while let (Ok(permit),Some((data,dest)))=(SEM.acquire().await,recv_texture.recv().await){
-					let texture_file_name_lowercase=texture_name.to_lowercase();
+			tokio::spawn(async move{
-					texture_file_name.push(texture_file_name_lowercase.clone());
+				let result=cli_convert_texture(data,dest).await;
-					//remove stem and search for both vtf and vmt files
+				drop(permit);
-					let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
+				match result{
-					texture_file_name.pop();
+					Ok(())=>(),
-					texture_file_name.push(stem);
+					Err(e)=>println!("Convert error: {e:?}"),
-					//somehow search for both files
-					let mut texture_file_name_vmt=texture_file_name.clone();
-					texture_file_name.set_extension("vtf");
-					texture_file_name_vmt.set_extension("vmt");
-					if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
-						return Ok(Some(stuff))
 				}
-					recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
+			});
-				};
-				if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{
-					found_texture=true;
-					let texture_name=texture_name.clone();
-					thread_handles.push(s.spawn(move||write_image(stuff,texture_name)));
-				}
-				if !found_texture{
-					println!("no data");
-				}
-			}
-			for thread in thread_handles{
-				match thread.join(){
-					Ok(Err(e))=>println!("write error: {:?}",e),
-					Err(e)=>println!("thread error: {:?}",e),
-					Ok(_)=>(),
-				}
-			}
-			Ok::<(),anyhow::Error>(())
-		})?
 		}
+		extract_thread.await?;
+		_=SEM.acquire_many(thread_limit as u32).await?;
 		Ok(())
 	}
 
@@ -272,93 +397,46 @@ fn bsp_contents(path:PathBuf)->AResult<()>{
 		Ok(())
 	}
 
-#[derive(Debug)]
+	async fn cli_convert_to_snf(path:&Path,vpk_list:&[vpk::VPK],output_folder:PathBuf)->AResult<()>{
-#[allow(dead_code)]
+		let entire_file=tokio::fs::read(path).await?;
-enum ConvertError{
+		let snf_data=convert_to_snf(&entire_file,vpk_list)?;
-	IO(std::io::Error),
-	SNFMap(strafesnet_snf::map::Error),
-	BspLoader(strafesnet_bsp_loader::ReadError),
-}
-impl std::fmt::Display for ConvertError{
-	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
-		write!(f,"{self:?}")
-	}
-}
-impl std::error::Error for ConvertError{}
 
-type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
+		let mut dest=output_folder;
-
-fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
-	let n_paths=pathlist.len();
-	let start = std::time::Instant::now();
-	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
-	let mut i=0;
-	let mut join_thread=|thread:MapThread|{
-		i+=1;
-		if let Err(e)=thread.join(){
-			println!("thread error: {:?}",e);
-		}else{
-			println!("{}/{}",i,n_paths);
-		}
-	};
-	for path in pathlist{
-		if 32<=threads.len(){
-			join_thread(threads.pop_front().unwrap());
-		}
-		let output_folder=output_folder.clone();
-		threads.push_back(std::thread::spawn(move ||{
-			let bsp=strafesnet_bsp_loader::read(
-				std::fs::File::open(path.as_path())
-				.map_err(ConvertError::IO)?
-			).map_err(ConvertError::BspLoader)?;
-			let mut loader=strafesnet_deferred_loader::source_legacy();
-
-				let (texture_loader,mesh_loader)=loader.get_inner_mut();
-
-				let map_step1=strafesnet_bsp_loader::convert(
-					&bsp,
-					|name|texture_loader.acquire_render_config_id(name),
-					|name|mesh_loader.acquire_mesh_id(name),
-				);
-
-				let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref());
-
-				let map_step2=map_step1.add_prop_meshes(
-					//the type conflagulator 9000
-					prop_meshes.into_iter().map(|(mesh_id,loader_model)|
-						(mesh_id,strafesnet_bsp_loader::data::ModelData{
-							mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()),
-							vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()),
-							vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()),
-						})
-					),
-					|name|texture_loader.acquire_render_config_id(name),
-				);
-
-				let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
-
-				let map=map_step2.add_render_configs_and_textures(
-					render_configs.into_iter(),
-					textures.into_iter().map(|(texture_id,texture)|
-						(texture_id,match texture{
-							strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
-						})
-					),
-				);
-
-			let mut dest=output_folder.clone();
 		dest.push(path.file_stem().unwrap());
 		dest.set_extension("snfm");
-			let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
+		tokio::fs::write(dest,snf_data).await?;
 
-			strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
-			Ok(())
-		}));
-	}
-
-	for thread in threads{
-		join_thread(thread);
-	}
-	println!("{:?}", start.elapsed());
 		Ok(())
 	}
+
+	async fn cli_source_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf,vpk_paths:Vec<PathBuf>)->AResult<()>{
+		let start=std::time::Instant::now();
+
+		let thread_limit=std::thread::available_parallelism()?.get();
+
+		let vpk_list=read_vpks(vpk_paths,thread_limit).await;
+		let vpk_list:&[vpk::VPK]=vpk_list.leak();
+
+		let mut it=paths.into_iter();
+		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+		SEM.add_permits(thread_limit);
+
+		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+			let output_folder=output_folder.clone();
+			tokio::spawn(async move{
+				let result=cli_convert_to_snf(path.as_path(),vpk_list,output_folder).await;
+				drop(permit);
+				match result{
+					Ok(())=>(),
+					Err(e)=>println!("Convert error: {e:?}"),
+				}
+			});
+		}
+		_=SEM.acquire_many(thread_limit as u32).await.unwrap();
+
+		println!("elapsed={:?}", start.elapsed());
+		Ok(())
+	}
+}
+#[cfg(feature="cli")]
+pub use cli::Commands;