tokio-ify source-to-snf

this will probably still hit the roblox rate limit

Cargo.lock

@@ -698,9 +698,7 @@ checksum = "835a3dc7d1ec9e75e2b5fb4ba75396837112d2060b03f7d43bc1897c7f7211da"
 
 [[package]]
 name = "fixed_wide"
-version = "0.1.2"
+version = "0.1.1"
-source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "a7e01a5b738e313c912fc41c425cf36e10c51647d3fd21d96db3d616344549fa"
 dependencies = [
  "arrayvec",
  "bnum",
@@ -1261,7 +1259,7 @@ dependencies = [
  "image",
  "intel_tex_2",
  "strum",
- "thiserror 1.0.69",
+ "thiserror",
 ]
 
 [[package]]
@@ -1441,8 +1439,6 @@ checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
 [[package]]
 name = "linear_ops"
 version = "0.1.0"
-source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "b2e6977ac24f47086d8a7a2d4ae1c720e86dfdc8407cf5e34c18bfa01053c456"
 dependencies = [
  "fixed_wide",
  "paste",
@@ -1568,7 +1564,7 @@ dependencies = [
 
 [[package]]
 name = "map-tool"
-version = "2.0.0"
+version = "1.6.0"
 dependencies = [
  "anyhow",
  "clap",
@@ -1587,7 +1583,6 @@ dependencies = [
  "strafesnet_deferred_loader",
  "strafesnet_rbx_loader",
  "strafesnet_snf",
- "thiserror 2.0.11",
  "tokio",
  "vbsp",
  "vmdl",
@@ -1620,7 +1615,7 @@ checksum = "317f146e2eb7021892722af37cf1b971f0a70c8406f487e24952667616192c64"
 dependencies = [
  "cfg-if",
  "miette-derive",
- "thiserror 1.0.69",
+ "thiserror",
  "unicode-width",
 ]
 
@@ -2175,8 +2170,6 @@ dependencies = [
 [[package]]
 name = "ratio_ops"
 version = "0.1.0"
-source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "01239195d6afe0509e7e3511b716c0540251dfe7ece0a9a5a27116afb766c42c"
 
 [[package]]
 name = "rav1e"
@@ -2208,7 +2201,7 @@ dependencies = [
  "rand_chacha",
  "simd_helpers",
  "system-deps",
- "thiserror 1.0.69",
+ "thiserror",
  "v_frame",
  "wasm-bindgen",
 ]
@@ -2274,7 +2267,7 @@ dependencies = [
  "rbx_dom_weak",
  "rbx_reflection",
  "rbx_reflection_database",
- "thiserror 1.0.69",
+ "thiserror",
 ]
 
 [[package]]
@@ -2305,7 +2298,7 @@ checksum = "c1b43fe592a4ce6fe54eb215fb82735efbb516d2cc045a94e3dc0234ff293620"
 dependencies = [
  "rbx_types",
  "serde",
- "thiserror 1.0.69",
+ "thiserror",
 ]
 
 [[package]]
@@ -2332,7 +2325,7 @@ dependencies = [
  "lazy_static",
  "rand",
  "serde",
- "thiserror 1.0.69",
+ "thiserror",
 ]
 
 [[package]]
@@ -2352,8 +2345,6 @@ dependencies = [
 [[package]]
 name = "rbxassetid"
 version = "0.1.0"
-source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "e6821fe9eaff54cd142932cb04c612b7599d9b8586973145b7ec1230ae84d184"
 dependencies = [
  "url",
 ]
@@ -2487,8 +2478,6 @@ dependencies = [
 [[package]]
 name = "roblox_emulator"
 version = "0.4.7"
-source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "7fc98335ce4b8548b725d727c5b32bd0b38274606c48fce6b6e7e5807d94db6b"
 dependencies = [
  "glam",
  "mlua",
@@ -2750,22 +2739,17 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 [[package]]
 name = "strafesnet_bsp_loader"
 version = "0.3.0"
-source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "3cc98773f2b98eb708b098946870b769975b63a396b84698b67e3d968029005d"
 dependencies = [
  "glam",
  "strafesnet_common",
  "strafesnet_deferred_loader",
  "vbsp",
  "vmdl",
- "vpk",
 ]
 
 [[package]]
 name = "strafesnet_common"
 version = "0.6.0"
-source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "0c1d7a83e1f6b579c6a9b4dc70c92373ab53b938601cd75928dd6795b5ffef21"
 dependencies = [
  "arrayvec",
  "bitflags 2.8.0",
@@ -2779,8 +2763,6 @@ dependencies = [
 [[package]]
 name = "strafesnet_deferred_loader"
 version = "0.5.0"
-source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "63d5d48e587d5f8bf5385bee3505ed790727fef68de855cf58247a08c5952bef"
 dependencies = [
  "strafesnet_common",
 ]
@@ -2788,8 +2770,6 @@ dependencies = [
 [[package]]
 name = "strafesnet_rbx_loader"
 version = "0.6.0"
-source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "e4659a49128c8d12b9fbdb289969cae04bfc5c1750d4273897700c5c17730d8a"
 dependencies = [
  "bytemuck",
  "glam",
@@ -2807,9 +2787,7 @@ dependencies = [
 
 [[package]]
 name = "strafesnet_snf"
-version = "0.3.0"
+version = "0.2.0"
-source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "fd24a22c484ca04213fa44b1d34bfbec385f0d176a2b5829cfa59ba7987b80d5"
 dependencies = [
  "binrw 0.14.1",
  "id",
@@ -2996,16 +2974,7 @@ version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
 dependencies = [
- "thiserror-impl 1.0.69",
+ "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror"
-version = "2.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"
-dependencies = [
- "thiserror-impl 2.0.11",
 ]
 
 [[package]]
@@ -3019,17 +2988,6 @@ dependencies = [
  "syn 2.0.96",
 ]
 
-[[package]]
-name = "thiserror-impl"
-version = "2.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.96",
-]
-
 [[package]]
 name = "tiff"
 version = "0.9.1"
@@ -3296,7 +3254,7 @@ dependencies = [
  "num_enum",
  "serde",
  "static_assertions",
- "thiserror 1.0.69",
+ "thiserror",
  "vdf-reader",
  "zip-lzma",
 ]
@@ -3317,7 +3275,7 @@ dependencies = [
  "miette",
  "parse-display 0.9.1",
  "serde",
- "thiserror 1.0.69",
+ "thiserror",
 ]
 
 [[package]]
@@ -3346,7 +3304,7 @@ dependencies = [
  "itertools 0.13.0",
  "num_enum",
  "static_assertions",
- "thiserror 1.0.69",
+ "thiserror",
  "tracing",
 ]
 
@@ -3359,7 +3317,7 @@ dependencies = [
  "miette",
  "serde",
  "serde_repr",
- "thiserror 1.0.69",
+ "thiserror",
  "vdf-reader",
 ]
 
@@ -3371,7 +3329,7 @@ checksum = "60ec10e731515f58d5494d472f027d9c6fc8500fcb790ff55751031bcad87b6b"
 dependencies = [
  "ahash",
  "binrw 0.13.3",
- "thiserror 1.0.69",
+ "thiserror",
 ]
 
 [[package]]

Cargo.toml

@@ -1,25 +1,13 @@
 [package]
 name = "map-tool"
-version = "2.0.0"
+version = "1.6.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
-[lib]
-name = "map_tool"
-path = "src/lib.rs"
-
-[[bin]]
-name = "map-tool"
-path = "src/main.rs"
-required-features = ["cli"]
-
-[features]
-cli = ["dep:clap"]
-
 [dependencies]
 anyhow = "1.0.75"
-clap = { version = "4.4.2", features = ["derive"], optional = true }
+clap = { version = "4.4.2", features = ["derive"] }
 flate2 = "1.0.27"
 futures = "0.3.31"
 image = "0.25.2"
@@ -30,12 +18,10 @@ rbx_binary = { version = "0.7.4", registry = "strafesnet" }
 rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
 rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
 rbx_xml = { version = "0.13.3", registry = "strafesnet" }
-rbxassetid = { version = "0.1.0", registry = "strafesnet" }
+strafesnet_bsp_loader = { version = "0.2.1", registry = "strafesnet" }
-strafesnet_bsp_loader = { version = "0.3.0", registry = "strafesnet" }
+strafesnet_deferred_loader = { version = "0.4.0", features = ["legacy"], registry = "strafesnet" }
-strafesnet_deferred_loader = { version = "0.5.0", registry = "strafesnet" }
+strafesnet_rbx_loader = { version = "0.5.1", registry = "strafesnet" }
-strafesnet_rbx_loader = { version = "0.6.0", registry = "strafesnet" }
+strafesnet_snf = { version = "0.2.0", registry = "strafesnet" }
-strafesnet_snf = { version = "0.3.0", registry = "strafesnet" }
-thiserror = "2.0.11"
 tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread", "fs"] }
 vbsp = "0.6.0"
 vmdl = "0.2.0"

src/common.rs

@@ -0,0 +1,75 @@
+use std::path::PathBuf;
+use std::io::{Read,Seek};
+use anyhow::Result as AResult;
+
+fn load_image<R:Read+Seek+std::io::BufRead>(input:&mut R)->AResult<image::DynamicImage>{
+	let mut fourcc=[0u8;4];
+	input.read_exact(&mut fourcc)?;
+	input.rewind()?;
+	match &fourcc{
+		b"\x89PNG"=>Ok(image::load(input,image::ImageFormat::Png)?),
+		b"\xFF\xD8\xFF\xE0"=>Ok(image::load(input,image::ImageFormat::Jpeg)?),//JFIF
+		b"<rob"=>Err(anyhow::Error::msg("Roblox xml garbage is not supported yet")),
+		other=>Err(anyhow::Error::msg(format!("Unknown texture format {:?}",other))),
+	}
+}
+
+fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
+	let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?);
+
+	let image=load_image(&mut input)?.to_rgba8();//this sets a=255, arcane is actually supposed to look like that
+
+	let format=if image.width()%4!=0||image.height()%4!=0{
+		image_dds::ImageFormat::Rgba8UnormSrgb
+	}else{
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
+	};
+	//this fails if the image dimensions are not a multiple of 4
+	let dds = image_dds::dds_from_image(
+		&image,
+		format,
+		image_dds::Quality::Slow,
+		image_dds::Mipmaps::GeneratedAutomatic,
+	)?;
+
+	//write dds
+	let mut dest=PathBuf::from("textures");
+	dest.push(file_thing.file_name());
+	dest.set_extension("dds");
+	let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
+	dds.write(&mut writer)?;
+
+	//move file to processed
+	let mut dest=PathBuf::from("textures/processed");
+	dest.push(file_thing.file_name());
+	std::fs::rename(file_thing.path(), dest)?;
+	Ok(())
+}
+pub fn convert_textures() -> AResult<()>{
+	std::fs::create_dir_all("textures/unprocessed")?;
+	std::fs::create_dir_all("textures/processed")?;
+	let start = std::time::Instant::now();
+	let mut threads=Vec::new();
+	for entry in std::fs::read_dir("textures/unprocessed")? {
+		let file_thing=entry?;
+		threads.push(std::thread::spawn(move ||{
+			let file_name=format!("{:?}",file_thing);
+			let result=convert(file_thing);
+			if let Err(e)=result{
+				println!("error processing file:{:?} error message:{:?}",file_name,e);
+			}
+		}));
+	}
+	let mut i=0;
+	let n_threads=threads.len();
+	for thread in threads{
+		i+=1;
+		if let Err(e)=thread.join(){
+			println!("thread error: {:?}",e);
+		}else{
+			println!("{}/{}",i,n_threads);
+		}
+	}
+	println!("{:?}", start.elapsed());
+	Ok(())
+}

src/lib.rs

@@ -1,2 +0,0 @@
-pub mod roblox;
-pub mod source;

src/main.rs

@@ -1,3 +1,7 @@
+mod common;
+mod roblox;
+mod source;
+
 use clap::{Parser,Subcommand};
 use anyhow::Result as AResult;
 
@@ -12,9 +16,10 @@ struct Cli {
 #[derive(Subcommand)]
 enum Commands{
 	#[command(flatten)]
-	Roblox(map_tool::roblox::Commands),
+	Roblox(roblox::Commands),
 	#[command(flatten)]
-	Source(map_tool::source::Commands),
+	Source(source::Commands),
+	ConvertTextures,
 }
 
 #[tokio::main]
@@ -23,5 +28,6 @@ async fn main()->AResult<()>{
 	match cli.command{
 		Commands::Roblox(commands)=>commands.run().await,
 		Commands::Source(commands)=>commands.run().await,
+		Commands::ConvertTextures=>common::convert_textures(),
 	}
 }

src/roblox.rs

@@ -1,40 +1,60 @@
+use std::path::{Path,PathBuf};
 use std::io::{Cursor,Read,Seek};
 use std::collections::HashSet;
+use clap::{Args,Subcommand};
+use anyhow::Result as AResult;
 use rbx_dom_weak::Instance;
 use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
 use rbxassetid::RobloxAssetId;
+use tokio::io::AsyncReadExt;
 
-// === Public library API ===
+const DOWNLOAD_LIMIT:usize=16;
 
-/// Unique asset IDs referenced by a Roblox place/model file.
+#[derive(Subcommand)]
-#[derive(Default)]
+pub enum Commands{
-pub struct UniqueAssets{
+	RobloxToSNF(RobloxToSNFSubcommand),
-	pub meshes:HashSet<RobloxAssetId>,
+	DownloadAssets(DownloadAssetsSubcommand),
-	pub unions:HashSet<RobloxAssetId>,
-	pub textures:HashSet<RobloxAssetId>,
 }
 
-#[derive(Debug,thiserror::Error)]
+#[derive(Args)]
-pub enum LoadDomError{
+pub struct RobloxToSNFSubcommand {
-	#[error("IO error {0:?}")]
+	#[arg(long)]
-	IO(#[from]std::io::Error),
+	output_folder:PathBuf,
-	#[error("Binary decode error {0:?}")]
+	#[arg(required=true)]
+	input_files:Vec<PathBuf>,
+}
+#[derive(Args)]
+pub struct DownloadAssetsSubcommand{
+	#[arg(required=true)]
+	roblox_files:Vec<PathBuf>,
+	// #[arg(long)]
+	// cookie_file:Option<String>,
+}
+
+impl Commands{
+	pub async fn run(self)->AResult<()>{
+		match self{
+			Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder).await,
+			Commands::DownloadAssets(subcommand)=>download_assets(
+				subcommand.roblox_files,
+				rbx_asset::cookie::Cookie::new("".to_string()),
+			).await,
+		}
+	}
+}
+
+#[allow(unused)]
+#[derive(Debug)]
+enum LoadDomError{
+	IO(std::io::Error),
 	Binary(rbx_binary::DecodeError),
-	#[error("XML decode error {0:?}")]
 	Xml(rbx_xml::DecodeError),
-	#[error("Unknown file format")]
 	UnknownFormat,
 }
-
+fn load_dom<R:Read+Seek>(mut input:R)->Result<rbx_dom_weak::WeakDom,LoadDomError>{
-/// Parse a Roblox file (binary or XML) from bytes into a WeakDom.
-pub fn load_dom(data:&[u8])->Result<rbx_dom_weak::WeakDom,LoadDomError>{
-	load_dom_reader(Cursor::new(data))
-}
-
-fn load_dom_reader<R:Read+Seek>(mut input:R)->Result<rbx_dom_weak::WeakDom,LoadDomError>{
 	let mut first_8=[0u8;8];
-	input.read_exact(&mut first_8)?;
+	input.read_exact(&mut first_8).map_err(LoadDomError::IO)?;
-	input.rewind()?;
+	input.rewind().map_err(LoadDomError::IO)?;
 	match &first_8{
 		b"<roblox!"=>rbx_binary::from_reader(input).map_err(LoadDomError::Binary),
 		b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(LoadDomError::Xml),
@@ -42,113 +62,56 @@ fn load_dom_reader<R:Read+Seek>(mut input:R)->Result<rbx_dom_weak::WeakDom,LoadD
 	}
 }
 
-/// Scan a parsed DOM and return all unique asset IDs (meshes, textures, unions).
+/* The ones I'm interested in:
-pub fn get_unique_assets(dom:rbx_dom_weak::WeakDom)->UniqueAssets{
+Beam.Texture
-	let mut assets=UniqueAssets::default();
+Decal.Texture
-	for object in dom.into_raw().1.into_values(){
+FileMesh.MeshId
-		assets.collect(&object);
+FileMesh.TextureId
-	}
+MaterialVariant.ColorMap
-	assets
+MaterialVariant.MetalnessMap
-}
+MaterialVariant.NormalMap
-
+MaterialVariant.RoughnessMap
-/// Scan a Roblox file (bytes) and return all unique asset IDs.
+MeshPart.MeshId
-pub fn get_unique_assets_from_file(data:&[u8])->Result<UniqueAssets,LoadDomError>{
+MeshPart.TextureID
-	let dom=load_dom(data)?;
+ParticleEmitter.Texture
-	Ok(get_unique_assets(dom))
+Sky.MoonTextureId
-}
+Sky.SkyboxBk
-
+Sky.SkyboxDn
-#[derive(Debug,thiserror::Error)]
+Sky.SkyboxFt
-pub enum ConvertTextureError{
+Sky.SkyboxLf
-	#[error("Image error {0:?}")]
+Sky.SkyboxRt
-	Image(#[from]image::ImageError),
+Sky.SkyboxUp
-	#[error("DDS create error {0:?}")]
+Sky.SunTextureId
-	DDS(#[from]image_dds::CreateDdsError),
+SurfaceAppearance.ColorMap
-	#[error("DDS write error {0:?}")]
+SurfaceAppearance.MetalnessMap
-	DDSWrite(#[from]image_dds::ddsfile::Error),
+SurfaceAppearance.NormalMap
-}
+SurfaceAppearance.RoughnessMap
-
+SurfaceAppearance.TexturePack
-/// Convert image bytes (PNG, JPEG, etc.) into DDS texture bytes.
+*/
-pub fn convert_texture_to_dds(image_data:&[u8])->Result<Vec<u8>,ConvertTextureError>{
+fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instance,property:&str){
-	let image=image::load_from_memory(image_data)?.to_rgba8();
+	if let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(property){
-
+		let url:&str=content.as_ref();
-	let format=if image.width()%4!=0||image.height()%4!=0{
+		if let Ok(asset_id)=url.parse(){
-		image_dds::ImageFormat::Rgba8UnormSrgb
+			content_list.insert(asset_id);
 		}else{
-		image_dds::ImageFormat::BC7RgbaUnormSrgb
+			println!("Content failed to parse into AssetID: {:?}",content);
-	};
+		}
-
-	let dds=image_dds::dds_from_image(
-		&image,
-		format,
-		image_dds::Quality::Slow,
-		image_dds::Mipmaps::GeneratedAutomatic,
-	)?;
-
-	let mut buf=Vec::new();
-	dds.write(&mut Cursor::new(&mut buf))?;
-	Ok(buf)
-}
-
-#[derive(Debug,thiserror::Error)]
-pub enum ConvertError{
-	#[error("IO error {0:?}")]
-	IO(#[from]std::io::Error),
-	#[error("SNF map error {0:?}")]
-	SNFMap(strafesnet_snf::map::Error),
-	#[error("Roblox read error {0:?}")]
-	RobloxRead(strafesnet_rbx_loader::ReadError),
-	#[error("Roblox load error {0:?}")]
-	RobloxLoad(strafesnet_rbx_loader::LoadError),
-}
-
-/// Convert a Roblox place/model file (bytes) to SNF map format (bytes).
-pub fn convert_to_snf(data:&[u8])->Result<Vec<u8>,ConvertError>{
-	let model=strafesnet_rbx_loader::read(
-		Cursor::new(data)
-	).map_err(ConvertError::RobloxRead)?;
-
-	let mut place=model.into_place();
-	place.run_scripts();
-
-	let map=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?;
-
-	let mut buf=Vec::new();
-	strafesnet_snf::map::write_map(Cursor::new(&mut buf),map).map_err(ConvertError::SNFMap)?;
-	Ok(buf)
-}
-
-/// Download a single asset from Roblox by ID. Returns raw asset bytes.
-pub async fn download_asset(context:&rbx_asset::cookie::CookieContext,asset_id:u64)->Result<Vec<u8>,rbx_asset::cookie::GetError>{
-	context.get_asset(rbx_asset::cookie::GetAssetRequest{
-		asset_id,
-		version:None,
-	}).await
-}
-
-/// Download a single asset with retry and exponential backoff for rate limiting.
-/// Returns None if all retries are exhausted or a non-rate-limit error occurs.
-pub async fn download_asset_retry(context:&rbx_asset::cookie::CookieContext,asset_id:u64)->Option<Vec<u8>>{
-	const BACKOFF_MUL:f32=1.3956124250860895286;
-	let mut backoff=1000f32;
-	for _ in 0..12{
-		match download_asset(context,asset_id).await{
-			Ok(data)=>return Some(data),
-			Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{
-				if scwuab.status_code.as_u16()==429{
-					tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
-					backoff*=BACKOFF_MUL;
 	}else{
-					return None;
+		println!("property={} does not exist for class={}",property,object.class.as_str());
 	}
-			},
-			Err(_)=>return None,
-		}
-	}
-	None
 }
-
+async fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
-// === Private helpers ===
+	let mut file=tokio::fs::File::open(path).await?;
-
+	let mut data=Vec::new();
+	file.read_to_end(&mut data).await?;
+	Ok(Cursor::new(data))
+}
+#[derive(Default)]
+struct UniqueAssets{
+	meshes:HashSet<RobloxAssetId>,
+	unions:HashSet<RobloxAssetId>,
+	textures:HashSet<RobloxAssetId>,
+}
 impl UniqueAssets{
 	fn collect(&mut self,object:&Instance){
 		match object.class.as_str(){
@@ -178,75 +141,28 @@ impl UniqueAssets{
 	}
 }
 
-fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instance,property:&str){
+#[allow(unused)]
-	if let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(property){
+#[derive(Debug)]
-		let url:&str=content.as_ref();
+enum UniqueAssetError{
-		if let Ok(asset_id)=url.parse(){
+	IO(std::io::Error),
-			content_list.insert(asset_id);
+	LoadDom(LoadDomError),
-		}else{
-			println!("Content failed to parse into AssetID: {:?}",content);
-		}
-	}else{
-		println!("property={} does not exist for class={}",property,object.class.as_str());
-	}
 }
-
+async fn unique_assets(path:&Path)->Result<UniqueAssets,UniqueAssetError>{
-// === CLI ===
+	// read entire file
-
+	let mut assets=UniqueAssets::default();
-#[cfg(feature="cli")]
+	let data=read_entire_file(path).await.map_err(UniqueAssetError::IO)?;
-mod cli{
+	let dom=load_dom(data).map_err(UniqueAssetError::LoadDom)?;
-	use super::*;
+	for object in dom.into_raw().1.into_values(){
-	use std::path::{Path,PathBuf};
+		assets.collect(&object);
-	use clap::{Args,Subcommand};
-	use anyhow::Result as AResult;
-	use tokio::io::AsyncReadExt;
-
-	const DOWNLOAD_LIMIT:usize=16;
-
-	#[derive(Subcommand)]
-	pub enum Commands{
-		RobloxToSNF(RobloxToSNFSubcommand),
-		DownloadAssets(DownloadAssetsSubcommand),
 	}
-
+	Ok(assets)
-	#[derive(Args)]
+}
-	pub struct RobloxToSNFSubcommand {
+enum DownloadType{
-		#[arg(long)]
-		output_folder:PathBuf,
-		#[arg(required=true)]
-		input_files:Vec<PathBuf>,
-	}
-	#[derive(Args)]
-	pub struct DownloadAssetsSubcommand{
-		#[arg(required=true)]
-		roblox_files:Vec<PathBuf>,
-	}
-
-	impl Commands{
-		pub async fn run(self)->AResult<()>{
-			match self{
-				Commands::RobloxToSNF(subcommand)=>cli_roblox_to_snf(subcommand.input_files,subcommand.output_folder).await,
-				Commands::DownloadAssets(subcommand)=>cli_download_assets(
-					subcommand.roblox_files,
-					rbx_asset::cookie::Cookie::new("".to_string()),
-				).await,
-			}
-		}
-	}
-
-	async fn read_entire_file(path:impl AsRef<Path>)->Result<Vec<u8>,std::io::Error>{
-		let mut file=tokio::fs::File::open(path).await?;
-		let mut data=Vec::new();
-		file.read_to_end(&mut data).await?;
-		Ok(data)
-	}
-
-	enum DownloadType{
 	Texture(RobloxAssetId),
 	Mesh(RobloxAssetId),
 	Union(RobloxAssetId),
-	}
+}
-	impl DownloadType{
+impl DownloadType{
 	fn path(&self)->PathBuf{
 		match self{
 			DownloadType::Texture(asset_id)=>format!("downloaded_textures/{}",asset_id.0.to_string()).into(),
@@ -261,30 +177,33 @@ mod cli{
 			DownloadType::Union(asset_id)=>asset_id.0,
 		}
 	}
-	}
+}
-	enum DownloadResult{
+enum DownloadResult{
 	Cached(PathBuf),
 	Data(Vec<u8>),
 	Failed,
-	}
+}
-	#[derive(Default,Debug)]
+#[derive(Default,Debug)]
-	struct Stats{
+struct Stats{
 	total_assets:u32,
 	cached_assets:u32,
 	downloaded_assets:u32,
 	failed_downloads:u32,
 	timed_out_downloads:u32,
-	}
+}
-	async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
+async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
 	stats.total_assets+=1;
+	let download_instruction=download_instruction;
+	// check if file exists on disk
 	let path=download_instruction.path();
 	if tokio::fs::try_exists(path.as_path()).await?{
 		stats.cached_assets+=1;
 		return Ok(DownloadResult::Cached(path));
 	}
 	let asset_id=download_instruction.asset_id();
+	// if not, download file
 	let mut retry=0;
-		const BACKOFF_MUL:f32=1.3956124250860895286;
+	const BACKOFF_MUL:f32=1.3956124250860895286;//exp(1/3)
 	let mut backoff=1000f32;
 	loop{
 		let asset_result=context.get_asset(rbx_asset::cookie::GetAssetRequest{
@@ -321,58 +240,109 @@ mod cli{
 			},
 		}
 	}
+}
+#[allow(unused)]
+#[derive(Debug)]
+enum ConvertTextureError{
+	Io(std::io::Error),
+	Image(image::ImageError),
+	DDS(image_dds::CreateDdsError),
+	DDSWrite(image_dds::ddsfile::Error),
+}
+impl From<std::io::Error> for ConvertTextureError{
+	fn from(value:std::io::Error)->Self{
+		Self::Io(value)
 	}
-
+}
-	async fn cli_convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),CliConvertTextureError>{
+impl From<image::ImageError> for ConvertTextureError{
+	fn from(value:image::ImageError)->Self{
+		Self::Image(value)
+	}
+}
+impl From<image_dds::CreateDdsError> for ConvertTextureError{
+	fn from(value:image_dds::CreateDdsError)->Self{
+		Self::DDS(value)
+	}
+}
+impl From<image_dds::ddsfile::Error> for ConvertTextureError{
+	fn from(value:image_dds::ddsfile::Error)->Self{
+		Self::DDSWrite(value)
+	}
+}
+async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
 	let data=match download_result{
 		DownloadResult::Cached(path)=>tokio::fs::read(path).await?,
 		DownloadResult::Data(data)=>data,
 		DownloadResult::Failed=>return Ok(()),
 	};
-		let dds_data=convert_texture_to_dds(&data)?;
+	// image::ImageFormat::Png
+	// image::ImageFormat::Jpeg
+	let image=image::load_from_memory(&data)?.to_rgba8();
+
+	// pick format
+	let format=if image.width()%4!=0||image.height()%4!=0{
+		image_dds::ImageFormat::Rgba8UnormSrgb
+	}else{
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
+	};
+
+	//this fails if the image dimensions are not a multiple of 4
+	let dds=image_dds::dds_from_image(
+		&image,
+		format,
+		image_dds::Quality::Slow,
+		image_dds::Mipmaps::GeneratedAutomatic,
+	)?;
+
 	let file_name=format!("textures/{}.dds",asset_id.0);
-		tokio::fs::write(file_name,dds_data).await?;
+	let mut file=std::fs::File::create(file_name)?;
+	dds.write(&mut file)?;
 	Ok(())
-	}
+}
-
+async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
-	#[derive(Debug,thiserror::Error)]
-	enum CliConvertTextureError{
-		#[error("IO error {0:?}")]
-		Io(#[from]std::io::Error),
-		#[error("Convert texture error {0:?}")]
-		Convert(#[from]ConvertTextureError),
-	}
-
-	async fn cli_download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
 	tokio::try_join!(
 		tokio::fs::create_dir_all("downloaded_textures"),
 		tokio::fs::create_dir_all("textures"),
 		tokio::fs::create_dir_all("meshes"),
 		tokio::fs::create_dir_all("unions"),
 	)?;
+	// use mpsc
 	let thread_limit=std::thread::available_parallelism()?.get();
 	let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
 	let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+	// map decode dispatcher
+	// read files multithreaded
+	// produce UniqueAssetsResult per file
 	tokio::spawn(async move{
+		// move send so it gets dropped when all maps have been decoded
+		// closing the channel
 		let mut it=paths.into_iter();
 		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
 		SEM.add_permits(thread_limit);
 		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
 			let send=send_assets.clone();
 			tokio::spawn(async move{
-					let data=read_entire_file(path.as_path()).await;
+				let result=unique_assets(path.as_path()).await;
-					let result=data.map_err(LoadDomError::from).and_then(|d|{
-						let dom=load_dom(&d)?;
-						Ok(get_unique_assets(dom))
-					});
 				_=send.send(result).await;
 				drop(permit);
 			});
 		}
 	});
+	// download manager
+	// insert into global unique assets guy
+	// add to download queue if the asset is globally unique and does not already exist on disk
 	let mut stats=Stats::default();
 	let context=rbx_asset::cookie::CookieContext::new(cookie);
 	let mut globally_unique_assets=UniqueAssets::default();
+	// pop a job = retry_queue.pop_front() or ingest(recv.recv().await)
+	// SLOW MODE:
+	// acquire all permits
+	// drop all permits
+	// pop one job
+	// if it succeeds go into fast mode
+	// FAST MODE:
+	// acquire one permit
+	// pop a job
 	let download_thread=tokio::spawn(async move{
 		while let Some(result)=recv_assets.recv().await{
 			let unique_assets=match result{
@@ -406,7 +376,7 @@ mod cli{
 	SEM.add_permits(thread_limit);
 	while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){
 		tokio::spawn(async move{
-				let result=cli_convert_texture(asset_id,download_result).await;
+			let result=convert_texture(asset_id,download_result).await;
 			drop(permit);
 			result.unwrap();
 		});
@@ -414,9 +384,45 @@ mod cli{
 	download_thread.await??;
 	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
 	Ok(())
-	}
+}
 
-	async fn cli_roblox_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+#[derive(Debug)]
+#[allow(dead_code)]
+enum ConvertError{
+	IO(std::io::Error),
+	SNFMap(strafesnet_snf::map::Error),
+	RobloxRead(strafesnet_rbx_loader::ReadError),
+	RobloxLoad(strafesnet_rbx_loader::LoadError),
+}
+impl std::fmt::Display for ConvertError{
+	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+		write!(f,"{self:?}")
+	}
+}
+impl std::error::Error for ConvertError{}
+async fn convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
+	let entire_file=tokio::fs::read(path).await?;
+
+	let model=strafesnet_rbx_loader::read(
+		std::io::Cursor::new(entire_file)
+	).map_err(ConvertError::RobloxRead)?;
+
+	let mut place=model.into_place();
+	place.run_scripts();
+
+	let map=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?;
+
+	let mut dest=output_folder;
+	dest.push(path.file_stem().unwrap());
+	dest.set_extension("snfm");
+	let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
+
+	strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
+
+	Ok(())
+}
+
+async fn roblox_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
 	let start=std::time::Instant::now();
 
 	let thread_limit=std::thread::available_parallelism()?.get();
@@ -427,7 +433,7 @@ mod cli{
 	while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
 		let output_folder=output_folder.clone();
 		tokio::spawn(async move{
-				let result=cli_convert_to_snf(path.as_path(),output_folder).await;
+			let result=convert_to_snf(path.as_path(),output_folder).await;
 			drop(permit);
 			match result{
 				Ok(())=>(),
@@ -439,19 +445,4 @@ mod cli{
 
 	println!("elapsed={:?}", start.elapsed());
 	Ok(())
-	}
-
-	async fn cli_convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
-		let entire_file=tokio::fs::read(path).await?;
-		let snf_data=convert_to_snf(&entire_file)?;
-
-		let mut dest=output_folder;
-		dest.push(path.file_stem().unwrap());
-		dest.set_extension("snfm");
-		tokio::fs::write(dest,snf_data).await?;
-
-		Ok(())
-	}
 }
-#[cfg(feature="cli")]
-pub use cli::Commands;

src/source.rs

@@ -1,116 +1,81 @@
-use std::io::Cursor;
+use std::path::{Path,PathBuf};
+use clap::{Args,Subcommand};
+use anyhow::Result as AResult;
 use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
 
-// === Public library API ===
+#[derive(Subcommand)]
-
+pub enum Commands{
-#[derive(Debug,thiserror::Error)]
+	SourceToSNF(SourceToSNFSubcommand),
-pub enum ConvertTextureError{
+	ExtractTextures(ExtractTexturesSubcommand),
-	#[error("Vtf error {0:?}")]
+	VPKContents(VPKContentsSubcommand),
-	Vtf(#[from]vtf::Error),
+	BSPContents(BSPContentsSubcommand),
-	#[error("DDS create error {0:?}")]
-	DDS(#[from]image_dds::CreateDdsError),
-	#[error("DDS write error {0:?}")]
-	DDSWrite(#[from]image_dds::ddsfile::Error),
 }
 
-/// Convert VTF texture bytes to DDS texture bytes.
+#[derive(Args)]
-pub fn convert_texture_to_dds(vtf_data:&[u8])->Result<Vec<u8>,ConvertTextureError>{
+pub struct SourceToSNFSubcommand {
-	let vtf_data=vtf_data.to_vec();
+	#[arg(long)]
-	let image=vtf::from_bytes(&vtf_data)?.highres_image.decode(0)?.to_rgba8();
+	output_folder:PathBuf,
-
+	#[arg(required=true)]
-	let format=if image.width()%4!=0||image.height()%4!=0{
+	input_files:Vec<PathBuf>,
-		image_dds::ImageFormat::Rgba8UnormSrgb
+}
-	}else{
+#[derive(Args)]
-		image_dds::ImageFormat::BC7RgbaUnormSrgb
+pub struct ExtractTexturesSubcommand {
-	};
+	#[arg(long)]
-
+	bsp_file:PathBuf,
-	let dds=image_dds::dds_from_image(
+	#[arg(long)]
-		&image,
+	vpk_dir_files:Vec<PathBuf>
-		format,
+}
-		image_dds::Quality::Slow,
+#[derive(Args)]
-		image_dds::Mipmaps::GeneratedAutomatic,
+pub struct VPKContentsSubcommand {
-	)?;
+	#[arg(long)]
-
+	input_file:PathBuf,
-	let mut buf=Vec::new();
+}
-	dds.write(&mut Cursor::new(&mut buf))?;
+#[derive(Args)]
-	Ok(buf)
+pub struct BSPContentsSubcommand {
+	#[arg(long)]
+	input_file:PathBuf,
 }
 
-#[derive(Debug,thiserror::Error)]
+impl Commands{
-pub enum ConvertError{
+	pub async fn run(self)->AResult<()>{
-	#[error("BSP read error {0:?}")]
+		match self{
-	BspRead(strafesnet_bsp_loader::ReadError),
+			Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
-	#[error("BSP load error {0:?}")]
+			Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder).await,
-	BspLoad(strafesnet_bsp_loader::LoadError),
+			Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
-	#[error("SNF map error {0:?}")]
+			Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
-	SNFMap(strafesnet_snf::map::Error),
+		}
-	#[error("BSP parse error {0:?}")]
+	}
-	BspParse(#[from]vbsp::BspError),
 }
 
-/// Convert a Source BSP file (bytes) to SNF map format (bytes).
-pub fn convert_to_snf(bsp_data:&[u8],vpk_list:&[vpk::VPK])->Result<Vec<u8>,ConvertError>{
-	let bsp=strafesnet_bsp_loader::read(
-		Cursor::new(bsp_data)
-	).map_err(ConvertError::BspRead)?;
 
-	let map=bsp.to_snf(LoadFailureMode::DefaultToNone,vpk_list).map_err(ConvertError::BspLoad)?;
+enum VMTContent{
-
-	let mut buf=Vec::new();
-	strafesnet_snf::map::write_map(Cursor::new(&mut buf),map).map_err(ConvertError::SNFMap)?;
-	Ok(buf)
-}
-
-/// Read VPK archives from paths. Useful for loading VPKs needed by `convert_to_snf`.
-pub async fn read_vpks(vpk_paths:Vec<std::path::PathBuf>,thread_limit:usize)->Vec<vpk::VPK>{
-	use futures::StreamExt;
-	futures::stream::iter(vpk_paths).map(|vpk_path|async{
-		tokio::task::spawn_blocking(move||vpk::VPK::read(&vpk_path)).await.unwrap().unwrap()
-	})
-	.buffer_unordered(thread_limit)
-	.collect().await
-}
-
-// === CLI ===
-
-#[cfg(feature="cli")]
-mod cli{
-	use super::*;
-	use std::path::{Path,PathBuf};
-	use std::borrow::Cow;
-	use clap::{Args,Subcommand};
-	use anyhow::Result as AResult;
-	use strafesnet_bsp_loader::loader::BspFinder;
-	use strafesnet_deferred_loader::loader::Loader;
-	use strafesnet_deferred_loader::deferred_loader::{MeshDeferredLoader,RenderConfigDeferredLoader};
-
-	enum VMTContent{
 	VMT(String),
 	VTF(String),
 	Patch(vmt_parser::material::PatchMaterial),
-		Unsupported,
+	Unsupported,//don't want to deal with whatever vmt variant
-		Unresolved,
+	Unresolved,//could not locate a texture because of vmt content
-	}
+}
-	impl VMTContent{
+impl VMTContent{
 	fn vtf(opt:Option<String>)->Self{
 		match opt{
 			Some(s)=>Self::VTF(s),
 			None=>Self::Unresolved,
 		}
 	}
-	}
+}
 
-	fn get_some_texture(material:vmt_parser::material::Material)->VMTContent{
+fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent>{
-		match material{
+	//just grab some texture from somewhere for now
+	Ok(match material{
 		vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)),
-			vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
+		vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none
 		vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
 		vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture),
 		vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture),
 		vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture),
 		vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)),
 		vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)),
-			vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),
+		vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),//recursive
 		vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)),
 		vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture),
 		vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)),
@@ -120,303 +85,231 @@ mod cli{
 		vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)),
 		vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported,
 		vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat),
-			_=>unreachable!(),
+		_=>return Err(anyhow::Error::msg("vmt failed to parse")),
-		}
+	})
-	}
+}
 
-	#[derive(Debug,thiserror::Error)]
+fn get_vmt<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,search_name:String)->AResult<vmt_parser::material::Material>{
-	enum GetVMTError{
+	if let Some(stuff)=find_stuff(search_name)?{
-		#[error("Bsp error {0:?}")]
+		//decode vmt and then write
-		Bsp(#[from]vbsp::BspError),
+		let stuff=String::from_utf8(stuff)?;
-		#[error("Utf8 error {0:?}")]
+		let material=vmt_parser::from_str(stuff.as_str())?;
-		Utf8(#[from]std::str::Utf8Error),
+		println!("vmt material={:?}",material);
-		#[error("Vdf error {0:?}")]
+		return Ok(material);
-		Vdf(#[from]vmt_parser::VdfError),
-		#[error("Vmt not found")]
-		NotFound,
 	}
+	Err(anyhow::Error::msg("vmt not found"))
+}
 
-	fn get_vmt(finder:BspFinder,search_name:&str)->Result<vmt_parser::material::Material,GetVMTError>{
+fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,material:vmt_parser::material::Material)->AResult<Option<Vec<u8>>>{
-		let vmt_data=finder.find(search_name)?.ok_or(GetVMTError::NotFound)?;
+	match get_some_texture(material)?{
-		let vmt_str=core::str::from_utf8(&vmt_data)?;
+		VMTContent::VMT(s)=>recursive_vmt_loader(find_stuff,get_vmt(find_stuff,s)?),
-		let material=vmt_parser::from_str(vmt_str)?;
-		Ok(material)
-	}
-
-	#[derive(Debug,thiserror::Error)]
-	enum LoadVMTError{
-		#[error("Bsp error {0:?}")]
-		Bsp(#[from]vbsp::BspError),
-		#[error("GetVMT error {0:?}")]
-		GetVMT(#[from]GetVMTError),
-		#[error("FromUtf8 error {0:?}")]
-		FromUtf8(#[from]std::string::FromUtf8Error),
-		#[error("Vdf error {0:?}")]
-		Vdf(#[from]vmt_parser::VdfError),
-		#[error("Vmt unsupported")]
-		Unsupported,
-		#[error("Vmt unresolved")]
-		Unresolved,
-		#[error("Vmt not found")]
-		NotFound,
-	}
-	fn recursive_vmt_loader<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,material:vmt_parser::material::Material)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
-		where
-			'bsp:'a,
-			'vpk:'a,
-	{
-		match get_some_texture(material){
-			VMTContent::VMT(s)=>recursive_vmt_loader(finder,get_vmt(finder,s.as_str())?),
 		VMTContent::VTF(s)=>{
 			let mut texture_file_name=PathBuf::from("materials");
 			texture_file_name.push(s);
 			texture_file_name.set_extension("vtf");
-				Ok(finder.find(texture_file_name.to_str().unwrap())?)
+			find_stuff(texture_file_name.into_os_string().into_string().unwrap())
 		},
-			VMTContent::Patch(mat)=>recursive_vmt_loader(finder,
+		VMTContent::Patch(mat)=>recursive_vmt_loader(find_stuff,
-				mat.resolve(|search_name|
+			mat.resolve(|search_name|{
-					match finder.find(search_name)?{
+				match find_stuff(search_name.to_string())?{
-						Some(bytes)=>Ok(String::from_utf8(bytes.into_owned())?),
+					Some(bytes)=>Ok(String::from_utf8(bytes)?),
-						None=>Err(LoadVMTError::NotFound),
+					None=>Err(anyhow::Error::msg("could not find vmt")),
 				}
-				)?
+			})?
 		),
-			VMTContent::Unsupported=>Err(LoadVMTError::Unsupported),
+		VMTContent::Unsupported=>{println!("Unsupported vmt");Ok(None)},//print and move on
-			VMTContent::Unresolved=>Err(LoadVMTError::Unresolved),
+		VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)},
-		}
-	}
-	fn load_texture<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,texture_name:&str)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
-		where
-			'bsp:'a,
-			'vpk:'a,
-	{
-		let mut texture_file_name=PathBuf::from("materials");
-		let texture_file_name_lowercase=texture_name.to_lowercase();
-		texture_file_name.push(texture_file_name_lowercase.clone());
-		let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
-		texture_file_name.pop();
-		texture_file_name.push(stem);
-		if let Some(stuff)=finder.find(texture_file_name.to_str().unwrap())?{
-			return Ok(Some(stuff))
-		}
-		let mut texture_file_name_vmt=texture_file_name.clone();
-		texture_file_name.set_extension("vtf");
-		texture_file_name_vmt.set_extension("vmt");
-		recursive_vmt_loader(finder,get_vmt(finder,texture_file_name_vmt.to_str().unwrap())?)
 	}
+}
 
-	#[derive(Subcommand)]
+fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
-	pub enum Commands{
+	std::fs::create_dir_all("textures")?;
-		SourceToSNF(SourceToSNFSubcommand),
+	let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
-		ExtractTextures(ExtractTexturesSubcommand),
+	for path in paths{
-		VPKContents(VPKContentsSubcommand),
+		let mut deduplicate=std::collections::HashSet::new();
-		BSPContents(BSPContentsSubcommand),
+		let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
-	}
-
-	#[derive(Args)]
-	pub struct SourceToSNFSubcommand {
-		#[arg(long)]
-		output_folder:PathBuf,
-		#[arg(required=true)]
-		input_files:Vec<PathBuf>,
-		#[arg(long)]
-		vpks:Vec<PathBuf>,
-	}
-	#[derive(Args)]
-	pub struct ExtractTexturesSubcommand{
-		#[arg(required=true)]
-		bsp_files:Vec<PathBuf>,
-		#[arg(long)]
-		vpks:Vec<PathBuf>,
-	}
-	#[derive(Args)]
-	pub struct VPKContentsSubcommand {
-		#[arg(long)]
-		input_file:PathBuf,
-	}
-	#[derive(Args)]
-	pub struct BSPContentsSubcommand {
-		#[arg(long)]
-		input_file:PathBuf,
-	}
-
-	impl Commands{
-		pub async fn run(self)->AResult<()>{
-			match self{
-				Commands::SourceToSNF(subcommand)=>cli_source_to_snf(subcommand.input_files,subcommand.output_folder,subcommand.vpks).await,
-				Commands::ExtractTextures(subcommand)=>cli_extract_textures(subcommand.bsp_files,subcommand.vpks).await,
-				Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
-				Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
-			}
-		}
-	}
-
-	#[derive(Debug,thiserror::Error)]
-	enum ExtractTextureError{
-		#[error("Io error {0:?}")]
-		Io(#[from]std::io::Error),
-		#[error("Bsp error {0:?}")]
-		Bsp(#[from]vbsp::BspError),
-		#[error("MeshLoad error {0:?}")]
-		MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError),
-		#[error("Load VMT error {0:?}")]
-		LoadVMT(#[from]LoadVMTError),
-	}
-	async fn gimme_them_textures(path:&Path,vpk_list:&[vpk::VPK],send_texture:tokio::sync::mpsc::Sender<(Vec<u8>,String)>)->Result<(),ExtractTextureError>{
-		let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?;
-		let loader_bsp=strafesnet_bsp_loader::Bsp::new(bsp);
-		let bsp=loader_bsp.as_ref();
-
-		let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
 		for texture in bsp.textures(){
-			texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name())));
+			deduplicate.insert(PathBuf::from(texture.name()));
 		}
-
+		//dedupe prop models
-		let mut mesh_deferred_loader=MeshDeferredLoader::new();
+		let mut model_dedupe=std::collections::HashSet::new();
 		for prop in bsp.static_props(){
-			mesh_deferred_loader.acquire_mesh_id(prop.model());
+			model_dedupe.insert(prop.model());
 		}
 
-		let finder=BspFinder{
+		//grab texture names from props
-			bsp:&loader_bsp,
+		for model_name in model_dedupe{
-			vpks:vpk_list
+			//.mdl, .vvd, .dx90.vtx
-		};
+			let mut path=PathBuf::from(model_name);
-
+			let file_name=PathBuf::from(path.file_stem().unwrap());
-		let mut mesh_loader=strafesnet_bsp_loader::loader::ModelLoader::new(finder);
+			path.pop();
-		for model_path in mesh_deferred_loader.into_indices(){
+			path.push(file_name);
-			let model:vmdl::Model=match mesh_loader.load(model_path){
+			let mut vvd_path=path.clone();
-				Ok(model)=>model,
+			let mut vtx_path=path.clone();
-				Err(e)=>{
+			vvd_path.set_extension("vvd");
-					println!("Model={model_path} Load model error: {e}");
+			vtx_path.set_extension("dx90.vtx");
-					continue;
+			match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){
-				},
+				(Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{
-			};
+					match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){
+						(Ok(mdl),Ok(vvd),Ok(vtx))=>{
+							let model=vmdl::Model::from_parts(mdl,vtx,vvd);
 							for texture in model.textures(){
 								for search_path in &texture.search_paths{
 									let mut path=PathBuf::from(search_path.as_str());
 									path.push(texture.name.as_str());
-					let path=path.to_str().unwrap().to_owned();
+									deduplicate.insert(path);
-					texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path)));
 								}
 							}
+						},
+						_=>println!("model_name={} error",model_name),
 					}
-
+				},
-		for texture_path in texture_deferred_loader.into_indices(){
+				_=>println!("no model name={}",model_name),
-			match load_texture(finder,&texture_path){
-				Ok(Some(texture))=>send_texture.send(
-					(texture.into_owned(),texture_path.into_owned())
-				).await.unwrap(),
-				Ok(None)=>(),
-				Err(e)=>println!("Texture={texture_path} Load error: {e}"),
 			}
 		}
 
-		Ok(())
+		let pack=&bsp.pack;
-	}
+		let vpk_list=&vpk_list;
+		std::thread::scope(move|s|{
+			let mut thread_handles=Vec::new();
+			for texture_name in deduplicate{
+				let mut found_texture=false;
+				//LMAO imagine having to write type names
+				let write_image=|mut stuff,write_file_name|{
+					let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
 
-	#[derive(Debug,thiserror::Error)]
+					let format=if image.width()%4!=0||image.height()%4!=0{
-	enum CliConvertTextureError{
+						image_dds::ImageFormat::Rgba8UnormSrgb
-		#[error("IO error {0:?}")]
+					}else{
-		Io(#[from]std::io::Error),
+						image_dds::ImageFormat::BC7RgbaUnormSrgb
-		#[error("Convert texture error {0:?}")]
+					};
-		Convert(#[from]ConvertTextureError),
+					//this fails if the image dimensions are not a multiple of 4
-	}
+					let dds = image_dds::dds_from_image(
-
+						&image,
-	async fn cli_convert_texture(texture:Vec<u8>,write_file_name:impl AsRef<Path>)->Result<(),CliConvertTextureError>{
+						format,
-		let dds_data=convert_texture_to_dds(&texture)?;
+						image_dds::Quality::Slow,
+						image_dds::Mipmaps::GeneratedAutomatic,
+					)?;
 
+					//write dds
 					let mut dest=PathBuf::from("textures");
 					dest.push(write_file_name);
 					dest.set_extension("dds");
 					std::fs::create_dir_all(dest.parent().unwrap())?;
-		let mut writer=std::io::BufWriter::new(std::fs::File::create(dest)?);
+					let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
-		std::io::Write::write_all(&mut writer,&dds_data)?;
+					dds.write(&mut writer)?;
-
+					Ok::<(),anyhow::Error>(())
+				};
+				let find_stuff=|search_file_name:String|{
+					println!("search_file_name={}",search_file_name);
+					match pack.get(search_file_name.as_str())?{
+						Some(file)=>return Ok(Some(file)),
+						_=>(),
+					}
+					//search pak list
+					for vpk_index in vpk_list{
+						if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
+							return Ok(Some(match vpk_entry.get()?{
+								std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
+								std::borrow::Cow::Owned(bytes)=>bytes,
+							}));
+						}
+					}
+					Ok::<Option<Vec<u8>>,anyhow::Error>(None)
+				};
+				let loader=|texture_name:String|{
+					let mut texture_file_name=PathBuf::from("materials");
+					//lower case
+					let texture_file_name_lowercase=texture_name.to_lowercase();
+					texture_file_name.push(texture_file_name_lowercase.clone());
+					//remove stem and search for both vtf and vmt files
+					let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
+					texture_file_name.pop();
+					texture_file_name.push(stem);
+					//somehow search for both files
+					let mut texture_file_name_vmt=texture_file_name.clone();
+					texture_file_name.set_extension("vtf");
+					texture_file_name_vmt.set_extension("vmt");
+					if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
+						return Ok(Some(stuff))
+					}
+					recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
+				};
+				if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{
+					found_texture=true;
+					let texture_name=texture_name.clone();
+					thread_handles.push(s.spawn(move||write_image(stuff,texture_name)));
+				}
+				if !found_texture{
+					println!("no data");
+				}
+			}
+			for thread in thread_handles{
+				match thread.join(){
+					Ok(Err(e))=>println!("write error: {:?}",e),
+					Err(e)=>println!("thread error: {:?}",e),
+					Ok(_)=>(),
+				}
+			}
+			Ok::<(),anyhow::Error>(())
+		})?
+	}
 	Ok(())
-	}
+}
 
-	async fn cli_extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
+fn vpk_contents(vpk_path:PathBuf)->AResult<()>{
-		tokio::try_join!(
-			tokio::fs::create_dir_all("extracted_textures"),
-			tokio::fs::create_dir_all("textures"),
-			tokio::fs::create_dir_all("meshes"),
-		)?;
-		let thread_limit=std::thread::available_parallelism()?.get();
-
-		let vpk_list=read_vpks(vpk_paths,thread_limit).await;
-		let vpk_list:&[vpk::VPK]=vpk_list.leak();
-
-		let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
-		let mut it=paths.into_iter();
-		let extract_thread=tokio::spawn(async move{
-			static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
-			SEM.add_permits(thread_limit);
-			while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
-				let send=send_texture.clone();
-				tokio::spawn(async move{
-					let result=gimme_them_textures(&path,vpk_list,send).await;
-					drop(permit);
-					match result{
-						Ok(())=>(),
-						Err(e)=>println!("Map={path:?} Decode error: {e:?}"),
-					}
-				});
-			}
-		});
-
-		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
-		SEM.add_permits(thread_limit);
-		while let (Ok(permit),Some((data,dest)))=(SEM.acquire().await,recv_texture.recv().await){
-			tokio::spawn(async move{
-				let result=cli_convert_texture(data,dest).await;
-				drop(permit);
-				match result{
-					Ok(())=>(),
-					Err(e)=>println!("Convert error: {e:?}"),
-				}
-			});
-		}
-		extract_thread.await?;
-		_=SEM.acquire_many(thread_limit as u32).await?;
-		Ok(())
-	}
-
-	fn vpk_contents(vpk_path:PathBuf)->AResult<()>{
 	let vpk_index=vpk::VPK::read(&vpk_path)?;
 	for (label,entry) in vpk_index.tree.into_iter(){
 		println!("vpk label={} entry={:?}",label,entry);
 	}
 	Ok(())
-	}
+}
 
-	fn bsp_contents(path:PathBuf)->AResult<()>{
+fn bsp_contents(path:PathBuf)->AResult<()>{
 	let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
 	for file_name in bsp.pack.into_zip().into_inner().unwrap().file_names(){
 		println!("file_name={:?}",file_name);
 	}
 	Ok(())
-	}
+}
 
-	async fn cli_convert_to_snf(path:&Path,vpk_list:&[vpk::VPK],output_folder:PathBuf)->AResult<()>{
+#[derive(Debug)]
+#[allow(dead_code)]
+enum ConvertError{
+	IO(std::io::Error),
+	SNFMap(strafesnet_snf::map::Error),
+	BspRead(strafesnet_bsp_loader::ReadError),
+	BspLoad(strafesnet_bsp_loader::LoadError),
+}
+impl std::fmt::Display for ConvertError{
+	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+		write!(f,"{self:?}")
+	}
+}
+impl std::error::Error for ConvertError{}
+
+async fn convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
 	let entire_file=tokio::fs::read(path).await?;
-		let snf_data=convert_to_snf(&entire_file,vpk_list)?;
+
+	let bsp=strafesnet_bsp_loader::read(
+		std::io::Cursor::new(entire_file)
+	).map_err(ConvertError::BspRead)?;
+
+	let map=bsp.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::BspLoad)?;
 
 	let mut dest=output_folder;
 	dest.push(path.file_stem().unwrap());
 	dest.set_extension("snfm");
-		tokio::fs::write(dest,snf_data).await?;
+	let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
+
+	strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
 
 	Ok(())
-	}
+}
-
+async fn source_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
-	async fn cli_source_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf,vpk_paths:Vec<PathBuf>)->AResult<()>{
 	let start=std::time::Instant::now();
 
 	let thread_limit=std::thread::available_parallelism()?.get();
-
-		let vpk_list=read_vpks(vpk_paths,thread_limit).await;
-		let vpk_list:&[vpk::VPK]=vpk_list.leak();
-
 	let mut it=paths.into_iter();
 	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
 	SEM.add_permits(thread_limit);
@@ -424,7 +317,7 @@ mod cli{
 	while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
 		let output_folder=output_folder.clone();
 		tokio::spawn(async move{
-				let result=cli_convert_to_snf(path.as_path(),vpk_list,output_folder).await;
+			let result=convert_to_snf(path.as_path(),output_folder).await;
 			drop(permit);
 			match result{
 				Ok(())=>(),
@@ -436,7 +329,4 @@ mod cli{
 
 	println!("elapsed={:?}", start.elapsed());
 	Ok(())
-	}
 }
-#[cfg(feature="cli")]
-pub use cli::Commands;