scratchpad: show physics only

the latter change is because roblox-to-snf expects the files in textures, not textures/dds

Cargo.toml

@@ -15,6 +15,7 @@ members = [
 	"lib/rbxassetid",
 	"lib/roblox_emulator",
 	"lib/snf",
+	"map-tool",
 	"strafe-client",
 ]
 resolver = "2"

lib/bsp_loader/src/brush.rs

@@ -0,0 +1,294 @@
+use strafesnet_common::integer::Planar64;
+use strafesnet_common::{model,integer};
+use strafesnet_common::integer::{vec3::Vector3,Fixed,Ratio};
+
+use crate::{valve_transform_normal,valve_transform_dist};
+
+#[derive(Hash,Eq,PartialEq)]
+struct Face{
+	normal:integer::Planar64Vec3,
+	dot:integer::Planar64,
+}
+
+#[derive(Debug)]
+struct Faces{
+	faces:Vec<Vec<integer::Planar64Vec3>>,
+}
+
+fn solve3(c0:&Face,c1:&Face,c2:&Face)->Option<Ratio<Vector3<Fixed<3,96>>,Fixed<3,96>>>{
+	let n0_n1=c0.normal.cross(c1.normal);
+	let det=c2.normal.dot(n0_n1);
+	if det.abs().is_zero(){
+		return None;
+	}
+	Some((
+		c1.normal.cross(c2.normal)*c0.dot
+		+c2.normal.cross(c0.normal)*c1.dot
+		+c0.normal.cross(c1.normal)*c2.dot
+	)/det)
+}
+
+#[derive(Debug)]
+pub enum PlanesToFacesError{
+	InitFace1,
+	InitFace2,
+	InitIntersection,
+	FindNewIntersection,
+	EmptyFaces,
+	InfiniteLoop1,
+	InfiniteLoop2,
+}
+impl std::fmt::Display for PlanesToFacesError{
+	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+		write!(f,"{self:?}")
+	}
+}
+impl core::error::Error for PlanesToFacesError{}
+
+fn planes_to_faces(face_list:std::collections::HashSet<Face>)->Result<Faces,PlanesToFacesError>{
+	let mut faces=Vec::new();
+	// for each face, determine one edge at a time until you complete the face
+	'face: for face0 in &face_list{
+		// 1. find first edge
+		// 2. follow edges around face
+
+		// === finding first edge ===
+		// 1. pick the most perpendicular set of 3 faces
+		// 2. check if any faces occlude the intersection
+		// 3. use this test to replace left and right alternating until they are not occluded
+
+		// find the most perpendicular face to face0
+		let mut face1=face_list.iter().min_by_key(|&p|{
+			face0.normal.dot(p.normal).abs()
+		}).ok_or(PlanesToFacesError::InitFace1)?;
+
+		// direction of edge formed by face0 x face1
+		let edge_dir=face0.normal.cross(face1.normal);
+
+		// find the most perpendicular face to both face0 and face1
+		let mut face2=face_list.iter().max_by_key(|&p|{
+			// find the best *oriented* face (no .abs())
+			edge_dir.dot(p.normal)
+		}).ok_or(PlanesToFacesError::InitFace2)?;
+
+		let mut detect_loop=200u8;
+
+		let mut intersection=solve3(face0,face1,face2).ok_or(PlanesToFacesError::InitIntersection)?;
+
+		// repeatedly update face0, face1 until all faces form part of the convex solid
+		'find: loop{
+			if let Some(a)=detect_loop.checked_sub(1){
+				detect_loop=a;
+			}else{
+				return Err(PlanesToFacesError::InfiniteLoop1);
+			}
+			// test if any *other* faces occlude the intersection
+			for new_face in &face_list{
+				// new face occludes intersection point
+				if (new_face.dot.fix_2()/Planar64::ONE).lt_ratio(new_face.normal.dot(intersection.num)/intersection.den){
+					// replace one of the faces with the new face
+					// dont' try to replace face0 because we are exploring that face in particular
+					if let Some(new_intersection)=solve3(face0,new_face,face2){
+						// face1 does not occlude (or intersect) the new intersection
+						if (face1.dot.fix_2()/Planar64::ONE).gt_ratio(face1.normal.dot(new_intersection.num)/new_intersection.den){
+							face1=new_face;
+							intersection=new_intersection;
+							continue 'find;
+						}
+					}
+					if let Some(new_intersection)=solve3(face0,face1,new_face){
+						// face2 does not occlude (or intersect) the new intersection
+						if (face2.dot.fix_2()/Planar64::ONE).gt_ratio(face2.normal.dot(new_intersection.num)/new_intersection.den){
+							face2=new_face;
+							intersection=new_intersection;
+							continue 'find;
+						}
+					}
+				}
+			}
+
+			// we have found a set of faces for which the intersection is on the convex solid
+			break 'find;
+		}
+
+		// check if face0 must go, meaning it is a degenerate face and does not contribute anything to the convex solid
+		for new_face in &face_list{
+			if core::ptr::eq(face0,new_face){
+				continue;
+			}
+			if core::ptr::eq(face1,new_face){
+				continue;
+			}
+			if core::ptr::eq(face2,new_face){
+				continue;
+			}
+			if let Some(new_intersection)=solve3(new_face,face1,face2){
+				// face0 does not occlude (or intersect) the new intersection
+				if (face0.dot.fix_2()/Planar64::ONE).lt_ratio(face0.normal.dot(new_intersection.num)/new_intersection.den){
+					// abort! reject face0 entirely
+					continue 'face;
+				}
+			}
+		}
+
+		// === follow edges around face ===
+		// Note that we chose face2 such that the 3 faces create a particular winding order.
+		// If we choose a consistent face to follow (face1, face2) it will always wind with a consistent chirality
+
+		let mut detect_loop=200u8;
+
+		// keep looping until we meet this face again
+		let face1=face1;
+		let mut face=Vec::new();
+		loop{
+			// push point onto vertices
+			// problem: this may push a vertex that does not fit in the fixed point range and is thus meaningless
+			face.push(intersection.divide().fix_1());
+
+			// we looped back around to face1, we're done!
+			if core::ptr::eq(face1,face2){
+				break;
+			}
+
+			// the measure
+			let edge_dir=face0.normal.cross(face2.normal);
+
+			// the dot product to beat
+			let d_intersection=edge_dir.dot(intersection.num)/intersection.den;
+
+			// find the next face moving clockwise around face0
+			let (new_face,new_intersection,_)=face_list.iter().filter_map(|new_face|{
+				// ignore faces that are part of the current edge
+				if core::ptr::eq(face0,new_face)
+				  |core::ptr::eq(face2,new_face){
+					return None;
+				}
+				let new_intersection=solve3(face0,face2,new_face)?;
+
+				// the d value must be larger
+				let d_new_intersection=edge_dir.dot(new_intersection.num)/new_intersection.den;
+				if d_new_intersection.le_ratio(d_intersection){
+					return None;
+				}
+
+				Some((new_face,new_intersection,d_new_intersection))
+			}).min_by_key(|&(_,_,d)|d).ok_or(PlanesToFacesError::FindNewIntersection)?;
+
+			face2=new_face;
+			intersection=new_intersection;
+
+			if let Some(a)=detect_loop.checked_sub(1){
+				detect_loop=a;
+			}else{
+				return Err(PlanesToFacesError::InfiniteLoop2);
+			}
+		}
+
+		faces.push(face);
+	}
+
+	if faces.is_empty(){
+		Err(PlanesToFacesError::EmptyFaces)
+	}else{
+		Ok(Faces{
+			faces,
+		})
+	}
+}
+
+#[derive(Debug)]
+pub enum BrushToMeshError{
+	SliceBrushSides,
+	MissingPlane,
+	InvalidFaceCount{
+		count:usize,
+	},
+	InvalidPlanes(PlanesToFacesError),
+}
+impl std::fmt::Display for BrushToMeshError{
+	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+		write!(f,"{self:?}")
+	}
+}
+impl core::error::Error for BrushToMeshError{}
+
+pub fn brush_to_mesh(bsp:&vbsp::Bsp,brush:&vbsp::Brush)->Result<model::Mesh,BrushToMeshError>{
+	let brush_start_idx=brush.brush_side as usize;
+	let sides_range=brush_start_idx..brush_start_idx+brush.num_brush_sides as usize;
+	let sides=bsp.brush_sides.get(sides_range).ok_or(BrushToMeshError::SliceBrushSides)?;
+	let face_list=sides.iter().map(|side|{
+		let plane=bsp.plane(side.plane as usize)?;
+		Some(Face{
+			normal:valve_transform_normal(plane.normal.into()),
+			dot:valve_transform_dist(plane.dist.into()),
+		})
+	}).collect::<Option<std::collections::HashSet<_>>>().ok_or(BrushToMeshError::MissingPlane)?;
+
+	if face_list.len()<4{
+		return Err(BrushToMeshError::InvalidFaceCount{count:face_list.len()});
+	}
+
+	let faces=planes_to_faces(face_list).map_err(BrushToMeshError::InvalidPlanes)?;
+
+	// generate the mesh
+	let mut mb=model::MeshBuilder::new();
+	let color=mb.acquire_color_id(glam::Vec4::ONE);
+	let tex=mb.acquire_tex_id(glam::Vec2::ZERO);
+	// normals are ignored by physics
+	let normal=mb.acquire_normal_id(integer::vec3::ZERO);
+
+	let polygon_list=faces.faces.into_iter().map(|face|{
+		face.into_iter().map(|pos|{
+			let pos=mb.acquire_pos_id(pos);
+			mb.acquire_vertex_id(model::IndexedVertex{
+				pos,
+				tex,
+				normal,
+				color,
+			})
+		}).collect()
+	}).collect();
+
+	let polygon_groups=vec![model::PolygonGroup::PolygonList(model::PolygonList::new(polygon_list))];
+	let physics_groups=vec![model::IndexedPhysicsGroup{
+		groups:vec![model::PolygonGroupId::new(0)],
+	}];
+	let graphics_groups=vec![model::IndexedGraphicsGroup{
+		render:model::RenderConfigId::new(0),
+		groups:vec![model::PolygonGroupId::new(0)],
+	}];
+
+	Ok(mb.build(polygon_groups,graphics_groups,physics_groups))
+}
+
+#[cfg(test)]
+mod test{
+	use super::*;
+	#[test]
+	fn test_cube(){
+		let face_list=[
+			Face{normal:integer::vec3::X,dot:Planar64::ONE},
+			Face{normal:integer::vec3::Y,dot:Planar64::ONE},
+			Face{normal:integer::vec3::Z,dot:Planar64::ONE},
+			Face{normal:integer::vec3::NEG_X,dot:Planar64::ONE},
+			Face{normal:integer::vec3::NEG_Y,dot:Planar64::ONE},
+			Face{normal:integer::vec3::NEG_Z,dot:Planar64::ONE},
+		].into_iter().collect();
+		let faces=planes_to_faces(face_list).unwrap();
+		dbg!(faces);
+	}
+	#[test]
+	fn test_cube_with_degernate_face(){
+		let face_list=[
+			Face{normal:integer::vec3::X,dot:Planar64::ONE},
+			Face{normal:integer::vec3::Y,dot:Planar64::ONE},
+			Face{normal:integer::vec3::Z,dot:Planar64::ONE},
+			Face{normal:integer::vec3::NEG_X,dot:Planar64::ONE},
+			Face{normal:integer::vec3::NEG_Y,dot:Planar64::ONE},
+			Face{normal:integer::vec3::NEG_Z,dot:Planar64::ONE},
+			Face{normal:integer::vec3::NEG_Z,dot:Planar64::EPSILON},
+		].into_iter().collect();
+		let faces=planes_to_faces(face_list).unwrap();
+		dbg!(faces);
+	}
+}

lib/bsp_loader/src/bsp.rs

@@ -41,43 +41,41 @@ pub fn convert<'a>(
 	//figure out real attributes later
 	let mut unique_attributes=Vec::new();
 	unique_attributes.push(gameplay_attributes::CollisionAttributes::Decoration);
-	const TEMP_TOUCH_ME_ATTRIBUTE:gameplay_attributes::CollisionAttributesId=gameplay_attributes::CollisionAttributesId::new(0);
+	unique_attributes.push(gameplay_attributes::CollisionAttributes::contact_default());
+	const ATTRIBUTE_DECORATION:gameplay_attributes::CollisionAttributesId=gameplay_attributes::CollisionAttributesId::new(0);
+	const ATTRIBUTE_CONTACT_DEFAULT:gameplay_attributes::CollisionAttributesId=gameplay_attributes::CollisionAttributesId::new(1);
 
-	let mut prop_mesh_count=0;
 	//declare all prop models to Loader
-	let prop_models=bsp.static_props().map(|prop|{
-		//get or create mesh_id
-		let mesh_id=mesh_deferred_loader.acquire_mesh_id(prop.model());
-		//not the most failsafe code but this is just for the map tool lmao
-		if prop_mesh_count==mesh_id.get(){
-			prop_mesh_count+=1;
-		};
-		let placement=prop.as_prop_placement();
-		model::Model{
-			mesh:mesh_id,
-			attributes:TEMP_TOUCH_ME_ATTRIBUTE,
-			transform:integer::Planar64Affine3::new(
-				integer::mat3::try_from_f32_array_2d((
-					glam::Mat3A::from_diagonal(glam::Vec3::splat(placement.scale))
-					//TODO: figure this out
-					*glam::Mat3A::from_quat(glam::Quat::from_array(placement.rotation.into()))
-				).to_cols_array_2d()).unwrap(),
-				valve_transform(placement.origin.into()),
-			),
-			color:glam::Vec4::ONE,
-		}
-	}).collect();
+	let prop_models=Vec::new();
+	// let prop_models=bsp.static_props().map(|prop|{
+	// 	//get or create mesh_id
+	// 	let mesh_id=mesh_deferred_loader.acquire_mesh_id(prop.model());
+	// 	let placement=prop.as_prop_placement();
+	// 	model::Model{
+	// 		mesh:mesh_id,
+	// 		attributes:ATTRIBUTE_DECORATION,
+	// 		transform:integer::Planar64Affine3::new(
+	// 			integer::mat3::try_from_f32_array_2d((
+	// 				glam::Mat3A::from_diagonal(glam::Vec3::splat(placement.scale))
+	// 				//TODO: figure this out
+	// 				*glam::Mat3A::from_quat(glam::Quat::from_array(placement.rotation.into()))
+	// 			).to_cols_array_2d()).unwrap(),
+	// 			valve_transform(placement.origin.into()),
+	// 		),
+	// 		color:glam::Vec4::ONE,
+	// 	}
+	// }).collect();
 
 	//TODO: make the main map one single mesh with a bunch of different physics groups and graphics groups
 
 	//the generated MeshIds in here will collide with the Loader Mesh Ids
 	//but I can't think of a good workaround other than just remapping one later.
-	let world_meshes:Vec<model::Mesh>=bsp.models().map(|world_model|{
+	let mut world_meshes:Vec<model::Mesh>=bsp.models().map(|world_model|{
 		let mut mb=model::MeshBuilder::new();
 
 		let color=mb.acquire_color_id(glam::Vec4::ONE);
 		let mut graphics_groups=Vec::new();
-		let mut physics_group=model::IndexedPhysicsGroup::default();
+		//let mut render_id_to_graphics_group_id=std::collections::HashMap::new();
 		let polygon_groups=world_model.faces().enumerate().map(|(polygon_group_id,face)|{
 			let polygon_group_id=model::PolygonGroupId::new(polygon_group_id as u32);
 			let face_texture=face.texture();
@@ -86,10 +84,6 @@ pub fn convert<'a>(
 			let texture_transform_u=glam::Vec4::from_array(face_texture.texture_transforms_u)/(face_texture_data.width as f32);
 			let texture_transform_v=glam::Vec4::from_array(face_texture.texture_transforms_v)/(face_texture_data.height as f32);
 
-			//this automatically figures out what the texture is trying to do and creates
-			//a render config for it, and then returns the id to that render config
-			let render_id=render_config_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(face_texture_data.name())));
-
 			//normal
 			let normal=mb.acquire_normal_id(valve_transform(face.normal().into()));
 			let mut polygon_iter=face.vertex_positions().map(|vertex_position|
@@ -107,19 +101,37 @@ pub fn convert<'a>(
 				).to_vec()
 			}).collect();
 			if face.is_visible(){
-				//TODO: deduplicate graphics groups by render id
-				graphics_groups.push(model::IndexedGraphicsGroup{
-					render:render_id,
-					groups:vec![polygon_group_id],
-				})
+				//this automatically figures out what the texture is trying to do and creates
+				//a render config for it, and then returns the id to that render config
+				let render_id=render_config_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(face_texture_data.name())));
+				//deduplicate graphics groups by render id
+				// let graphics_group_id=*render_id_to_graphics_group_id.entry(render_id).or_insert_with(||{
+				// 	let graphics_group_id=graphics_groups.len();
+				// 	graphics_groups.push(model::IndexedGraphicsGroup{
+				// 		render:render_id,
+				// 		groups:vec![],
+				// 	});
+				// 	graphics_group_id
+				// });
+				// graphics_groups[graphics_group_id].groups.push(polygon_group_id);
 			}
-			physics_group.groups.push(polygon_group_id);
 			model::PolygonGroup::PolygonList(model::PolygonList::new(polygon_list))
 		}).collect();
 
-		mb.build(polygon_groups,graphics_groups,vec![physics_group])
+		mb.build(polygon_groups,graphics_groups,vec![])
 	}).collect();
 
+	let brush_mesh_start_idx=world_meshes.len();
+	for brush in &bsp.brushes{
+		let mesh_result=crate::brush::brush_to_mesh(bsp,brush);
+		match mesh_result{
+			Ok(mesh)=>world_meshes.push(mesh),
+			Err(e)=>println!("Brush mesh error: {e}"),
+		}
+	}
+	println!("How many brush: {}",bsp.brushes.len());
+	println!("Generated brush models: {}",world_meshes.len()-brush_mesh_start_idx);
+
 	let world_models:Vec<model::Model>=
 	//one instance of the main world mesh
 	std::iter::once((
@@ -145,17 +157,26 @@ pub fn convert<'a>(
 				(model::MeshId::new(mesh_id),brush.origin,brush.color)
 			)
 		)
-	).map(|(mesh_id,model_origin,vbsp::Color{r,g,b})|{
-		model::Model{
-			mesh:mesh_id,
-			attributes:TEMP_TOUCH_ME_ATTRIBUTE,
+	).map(|(mesh_id,model_origin,vbsp::Color{r,g,b})|model::Model{
+		mesh:mesh_id,
+		attributes:ATTRIBUTE_DECORATION,
+		transform:integer::Planar64Affine3::new(
+			integer::mat3::identity(),
+			valve_transform(model_origin.into())
+		),
+		color:(glam::Vec3::from_array([r as f32,g as f32,b as f32])/255.0).extend(1.0),
+	}).chain(
+		// physics models
+		(brush_mesh_start_idx..world_meshes.len()).map(|mesh_id|model::Model{
+			mesh:model::MeshId::new(mesh_id as u32),
+			attributes:ATTRIBUTE_CONTACT_DEFAULT,
 			transform:integer::Planar64Affine3::new(
 				integer::mat3::identity(),
-				valve_transform(model_origin.into())
+				integer::vec3::ZERO,
 			),
-			color:(glam::Vec3::from_array([r as f32,g as f32,b as f32])/255.0).extend(1.0),
-		}
-	}).collect();
+			color:glam::Vec4::ONE,
+		})
+	).collect();
 
 	PartialMap1{
 		attributes:unique_attributes,

lib/bsp_loader/src/lib.rs

@@ -2,9 +2,16 @@ use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLo
 
 mod bsp;
 mod mesh;
+mod brush;
 pub mod loader;
 
 const VALVE_SCALE:f32=1.0/16.0;
+pub(crate) fn valve_transform_dist(d:f32)->strafesnet_common::integer::Planar64{
+	(d*VALVE_SCALE).try_into().unwrap()
+}
+pub(crate) fn valve_transform_normal([x,y,z]:[f32;3])->strafesnet_common::integer::Planar64Vec3{
+	strafesnet_common::integer::vec3::try_from_f32_array([x,z,-y]).unwrap()
+}
 pub(crate) fn valve_transform([x,y,z]:[f32;3])->strafesnet_common::integer::Planar64Vec3{
 	strafesnet_common::integer::vec3::try_from_f32_array([x*VALVE_SCALE,z*VALVE_SCALE,-y*VALVE_SCALE]).unwrap()
 }
@@ -43,10 +50,13 @@ impl From<loader::MeshError> for LoadError{
 		Self::Mesh(value)
 	}
 }
-pub struct Bsp(vbsp::Bsp);
+pub struct Bsp{
+	bsp:vbsp::Bsp,
+	case_folded_file_names:std::collections::HashMap<String,String>,
+}
 impl AsRef<vbsp::Bsp> for Bsp{
 	fn as_ref(&self)->&vbsp::Bsp{
-		&self.0
+		&self.bsp
 	}
 }
 
@@ -59,10 +69,22 @@ pub fn read<R:std::io::Read>(mut input:R)->Result<Bsp,ReadError>{
 	vbsp::Bsp::read(s.as_slice()).map(Bsp::new).map_err(ReadError::Bsp)
 }
 impl Bsp{
-	pub const fn new(value:vbsp::Bsp)->Self{
-		Self(value)
+	pub fn new(bsp:vbsp::Bsp)->Self{
+		let case_folded_file_names=bsp.pack.clone().into_zip().lock().unwrap().file_names().map(|s|{
+			(s.to_lowercase(),s.to_owned())
+		}).collect();
+		Self{
+			bsp,
+			case_folded_file_names,
+		}
 	}
-	pub fn to_snf(&self,failure_mode:LoadFailureMode,vpk_list:&[vpk::VPK])->Result<strafesnet_common::map::CompleteMap,LoadError>{
+	pub fn pack_get(&self,name_lowercase:&str)->Result<Option<Vec<u8>>,vbsp::BspError>{
+		match self.case_folded_file_names.get(name_lowercase){
+			Some(name_folded)=>self.bsp.pack.get(name_folded),
+			None=>Ok(None),
+		}
+	}
+	pub fn to_snf(&self,failure_mode:LoadFailureMode,vpk_list:&[Vpk])->Result<strafesnet_common::map::CompleteMap,LoadError>{
 		let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
 		let mut mesh_deferred_loader=MeshDeferredLoader::new();
 
@@ -85,3 +107,27 @@ impl Bsp{
 		Ok(map)
 	}
 }
+pub struct Vpk{
+	vpk:vpk::VPK,
+	case_folded_file_names:std::collections::HashMap<String,String>,
+}
+impl AsRef<vpk::VPK> for Vpk{
+	fn as_ref(&self)->&vpk::VPK{
+		&self.vpk
+	}
+}
+impl Vpk{
+	pub fn new(vpk:vpk::VPK)->Vpk{
+		let case_folded_file_names=vpk.tree.keys().map(|s|{
+			(s.to_lowercase(),s.to_owned())
+		}).collect();
+		Vpk{
+			vpk,
+			case_folded_file_names,
+		}
+	}
+	pub fn tree_get(&self,name_lowercase:&str)->Option<&vpk::entry::VPKEntry>{
+		let name_folded=self.case_folded_file_names.get(name_lowercase)?;
+		self.vpk.tree.get(name_folded)
+	}
+}

lib/bsp_loader/src/loader.rs

@@ -3,7 +3,7 @@ use std::{borrow::Cow, io::Read};
 use strafesnet_common::model::Mesh;
 use strafesnet_deferred_loader::{loader::Loader,texture::Texture};
 
-use crate::Bsp;
+use crate::{Bsp,Vpk};
 
 #[allow(dead_code)]
 #[derive(Debug)]
@@ -76,7 +76,7 @@ impl From<vbsp::BspError> for MeshError{
 #[derive(Clone,Copy)]
 pub struct BspFinder<'bsp,'vpk>{
 	pub bsp:&'bsp Bsp,
-	pub vpks:&'vpk [vpk::VPK],
+	pub vpks:&'vpk [Vpk],
 }
 impl<'bsp,'vpk> BspFinder<'bsp,'vpk>{
 	pub fn find<'a>(&self,path:&str)->Result<Option<Cow<'a,[u8]>>,vbsp::BspError>
@@ -85,13 +85,13 @@ impl<'bsp,'vpk> BspFinder<'bsp,'vpk>{
 			'vpk:'a,
 	{
 		// search bsp
-		if let Some(data)=self.bsp.as_ref().pack.get(path)?{
+		if let Some(data)=self.bsp.pack_get(path)?{
 			return Ok(Some(Cow::Owned(data)));
 		}
 
 		//search each vpk
 		for vpk in self.vpks{
-			if let Some(vpk_entry)=vpk.tree.get(path){
+			if let Some(vpk_entry)=vpk.tree_get(path){
 				return Ok(Some(vpk_entry.get()?));
 			}
 		}

lib/linear_ops/src/matrix.rs

@@ -1,5 +1,6 @@
 use crate::vector::Vector;
 
+#[repr(transparent)]
 #[derive(Clone,Copy,Debug,Hash,Eq,PartialEq)]
 pub struct Matrix<const X:usize,const Y:usize,T>{
 	pub(crate) array:[[T;Y];X],

lib/linear_ops/src/vector.rs

@@ -3,6 +3,7 @@
 /// v.x += v.z;
 /// println!("v.x={}",v.x);
 
+#[repr(transparent)]
 #[derive(Clone,Copy,Debug,Hash,Eq,PartialEq)]
 pub struct Vector<const N:usize,T>{
 	pub(crate) array:[T;N],

lib/rbx_loader/src/directories.rs

@@ -1,6 +0,0 @@
-// TODO: make a directories structure like strafe client
-struct Directories{
-	textures:PathBuf,
-	meshes:PathBuf,
-	unions:PathBuf,
-}

map-tool/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "map-tool"
+version = "1.7.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.75"
+clap = { version = "4.4.2", features = ["derive"] }
+flate2 = "1.0.27"
+futures = "0.3.31"
+image = "0.25.2"
+image_dds = "0.7.1"
+lazy-regex = "3.1.0"
+rbx_asset = { version = "0.2.5", registry = "strafesnet" }
+rbx_binary = { version = "0.7.4", registry = "strafesnet" }
+rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
+rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
+rbx_xml = { version = "0.13.3", registry = "strafesnet" }
+rbxassetid = { version = "0.1.0", registry = "strafesnet" }
+strafesnet_bsp_loader = { version = "0.3.0", path = "../lib/bsp_loader", registry = "strafesnet" }
+strafesnet_deferred_loader = { version = "0.5.0", path = "../lib/deferred_loader", registry = "strafesnet" }
+strafesnet_rbx_loader = { version = "0.6.0", path = "../lib/rbx_loader", registry = "strafesnet" }
+strafesnet_snf = { version = "0.3.0", path = "../lib/snf", registry = "strafesnet" }
+thiserror = "2.0.11"
+tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread", "fs"] }
+vbsp = "0.6.0"
+vmdl = "0.2.0"
+vmt-parser = "0.2.0"
+vpk = "0.2.0"
+vtf = "0.3.0"
+
+#[profile.release]
+#lto = true
+#strip = true
+#codegen-units = 1

map-tool/LICENSE

@@ -0,0 +1,23 @@
+Permission is hereby granted, free of charge, to any
+person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without
+limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software
+is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

map-tool/README.md

@@ -0,0 +1,2 @@
+# map-tool
+

map-tool/src/main.rs

@@ -0,0 +1,30 @@
+mod roblox;
+mod source;
+
+use clap::{Parser,Subcommand};
+use anyhow::Result as AResult;
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+#[command(propagate_version = true)]
+struct Cli {
+	#[command(subcommand)]
+	command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands{
+	#[command(flatten)]
+	Roblox(roblox::Commands),
+	#[command(flatten)]
+	Source(source::Commands),
+}
+
+#[tokio::main]
+async fn main()->AResult<()>{
+	let cli=Cli::parse();
+	match cli.command{
+		Commands::Roblox(commands)=>commands.run().await,
+		Commands::Source(commands)=>commands.run().await,
+	}
+}

map-tool/src/roblox.rs

@@ -0,0 +1,431 @@
+use std::path::{Path,PathBuf};
+use std::io::{Cursor,Read,Seek};
+use std::collections::HashSet;
+use clap::{Args,Subcommand};
+use anyhow::Result as AResult;
+use rbx_dom_weak::Instance;
+use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
+use rbxassetid::RobloxAssetId;
+use tokio::io::AsyncReadExt;
+
+const DOWNLOAD_LIMIT:usize=16;
+
+#[derive(Subcommand)]
+pub enum Commands{
+	RobloxToSNF(RobloxToSNFSubcommand),
+	DownloadAssets(DownloadAssetsSubcommand),
+}
+
+#[derive(Args)]
+pub struct RobloxToSNFSubcommand {
+	#[arg(long)]
+	output_folder:PathBuf,
+	#[arg(required=true)]
+	input_files:Vec<PathBuf>,
+}
+#[derive(Args)]
+pub struct DownloadAssetsSubcommand{
+	#[arg(required=true)]
+	roblox_files:Vec<PathBuf>,
+	// #[arg(long)]
+	// cookie_file:Option<String>,
+}
+
+impl Commands{
+	pub async fn run(self)->AResult<()>{
+		match self{
+			Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder).await,
+			Commands::DownloadAssets(subcommand)=>download_assets(
+				subcommand.roblox_files,
+				rbx_asset::cookie::Cookie::new("".to_string()),
+			).await,
+		}
+	}
+}
+
+#[allow(unused)]
+#[derive(Debug)]
+enum LoadDomError{
+	IO(std::io::Error),
+	Binary(rbx_binary::DecodeError),
+	Xml(rbx_xml::DecodeError),
+	UnknownFormat,
+}
+fn load_dom<R:Read+Seek>(mut input:R)->Result<rbx_dom_weak::WeakDom,LoadDomError>{
+	let mut first_8=[0u8;8];
+	input.read_exact(&mut first_8).map_err(LoadDomError::IO)?;
+	input.rewind().map_err(LoadDomError::IO)?;
+	match &first_8{
+		b"<roblox!"=>rbx_binary::from_reader(input).map_err(LoadDomError::Binary),
+		b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(LoadDomError::Xml),
+		_=>Err(LoadDomError::UnknownFormat),
+	}
+}
+
+/* The ones I'm interested in:
+Beam.Texture
+Decal.Texture
+FileMesh.MeshId
+FileMesh.TextureId
+MaterialVariant.ColorMap
+MaterialVariant.MetalnessMap
+MaterialVariant.NormalMap
+MaterialVariant.RoughnessMap
+MeshPart.MeshId
+MeshPart.TextureID
+ParticleEmitter.Texture
+Sky.MoonTextureId
+Sky.SkyboxBk
+Sky.SkyboxDn
+Sky.SkyboxFt
+Sky.SkyboxLf
+Sky.SkyboxRt
+Sky.SkyboxUp
+Sky.SunTextureId
+SurfaceAppearance.ColorMap
+SurfaceAppearance.MetalnessMap
+SurfaceAppearance.NormalMap
+SurfaceAppearance.RoughnessMap
+SurfaceAppearance.TexturePack
+*/
+fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instance,property:&str){
+	if let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(property){
+		let url:&str=content.as_ref();
+		if let Ok(asset_id)=url.parse(){
+			content_list.insert(asset_id);
+		}else{
+			println!("Content failed to parse into AssetID: {:?}",content);
+		}
+	}else{
+		println!("property={} does not exist for class={}",property,object.class.as_str());
+	}
+}
+async fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
+	let mut file=tokio::fs::File::open(path).await?;
+	let mut data=Vec::new();
+	file.read_to_end(&mut data).await?;
+	Ok(Cursor::new(data))
+}
+#[derive(Default)]
+struct UniqueAssets{
+	meshes:HashSet<RobloxAssetId>,
+	unions:HashSet<RobloxAssetId>,
+	textures:HashSet<RobloxAssetId>,
+}
+impl UniqueAssets{
+	fn collect(&mut self,object:&Instance){
+		match object.class.as_str(){
+			"Beam"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+			"Decal"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+			"Texture"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+			"FileMesh"=>accumulate_content_id(&mut self.textures,object,"TextureId"),
+			"MeshPart"=>{
+				accumulate_content_id(&mut self.textures,object,"TextureID");
+				accumulate_content_id(&mut self.meshes,object,"MeshId");
+			},
+			"SpecialMesh"=>accumulate_content_id(&mut self.meshes,object,"MeshId"),
+			"ParticleEmitter"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+			"Sky"=>{
+				accumulate_content_id(&mut self.textures,object,"MoonTextureId");
+				accumulate_content_id(&mut self.textures,object,"SkyboxBk");
+				accumulate_content_id(&mut self.textures,object,"SkyboxDn");
+				accumulate_content_id(&mut self.textures,object,"SkyboxFt");
+				accumulate_content_id(&mut self.textures,object,"SkyboxLf");
+				accumulate_content_id(&mut self.textures,object,"SkyboxRt");
+				accumulate_content_id(&mut self.textures,object,"SkyboxUp");
+				accumulate_content_id(&mut self.textures,object,"SunTextureId");
+			},
+			"UnionOperation"=>accumulate_content_id(&mut self.unions,object,"AssetId"),
+			_=>(),
+		}
+	}
+}
+
+#[allow(unused)]
+#[derive(Debug)]
+enum UniqueAssetError{
+	IO(std::io::Error),
+	LoadDom(LoadDomError),
+}
+async fn unique_assets(path:&Path)->Result<UniqueAssets,UniqueAssetError>{
+	// read entire file
+	let mut assets=UniqueAssets::default();
+	let data=read_entire_file(path).await.map_err(UniqueAssetError::IO)?;
+	let dom=load_dom(data).map_err(UniqueAssetError::LoadDom)?;
+	for object in dom.into_raw().1.into_values(){
+		assets.collect(&object);
+	}
+	Ok(assets)
+}
+enum DownloadType{
+	Texture(RobloxAssetId),
+	Mesh(RobloxAssetId),
+	Union(RobloxAssetId),
+}
+impl DownloadType{
+	fn path(&self)->PathBuf{
+		match self{
+			DownloadType::Texture(asset_id)=>format!("downloaded_textures/{}",asset_id.0.to_string()).into(),
+			DownloadType::Mesh(asset_id)=>format!("meshes/{}",asset_id.0.to_string()).into(),
+			DownloadType::Union(asset_id)=>format!("unions/{}",asset_id.0.to_string()).into(),
+		}
+	}
+	fn asset_id(&self)->u64{
+		match self{
+			DownloadType::Texture(asset_id)=>asset_id.0,
+			DownloadType::Mesh(asset_id)=>asset_id.0,
+			DownloadType::Union(asset_id)=>asset_id.0,
+		}
+	}
+}
+enum DownloadResult{
+	Cached(PathBuf),
+	Data(Vec<u8>),
+	Failed,
+}
+#[derive(Default,Debug)]
+struct Stats{
+	total_assets:u32,
+	cached_assets:u32,
+	downloaded_assets:u32,
+	failed_downloads:u32,
+	timed_out_downloads:u32,
+}
+async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
+	stats.total_assets+=1;
+	let download_instruction=download_instruction;
+	// check if file exists on disk
+	let path=download_instruction.path();
+	if tokio::fs::try_exists(path.as_path()).await?{
+		stats.cached_assets+=1;
+		return Ok(DownloadResult::Cached(path));
+	}
+	let asset_id=download_instruction.asset_id();
+	// if not, download file
+	let mut retry=0;
+	const BACKOFF_MUL:f32=1.3956124250860895286;//exp(1/3)
+	let mut backoff=1000f32;
+	loop{
+		let asset_result=context.get_asset(rbx_asset::cookie::GetAssetRequest{
+			asset_id,
+			version:None,
+		}).await;
+		match asset_result{
+			Ok(asset_result)=>{
+				stats.downloaded_assets+=1;
+				tokio::fs::write(path,&asset_result).await?;
+				break Ok(DownloadResult::Data(asset_result));
+			},
+			Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{
+				if scwuab.status_code.as_u16()==429{
+					if retry==12{
+						println!("Giving up asset download {asset_id}");
+						stats.timed_out_downloads+=1;
+						break Ok(DownloadResult::Failed);
+					}
+					println!("Hit roblox rate limit, waiting {:.0}ms...",backoff);
+					tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
+					backoff*=BACKOFF_MUL;
+					retry+=1;
+				}else{
+					stats.failed_downloads+=1;
+					println!("weird scuwab error: {scwuab:?}");
+					break Ok(DownloadResult::Failed);
+				}
+			},
+			Err(e)=>{
+				stats.failed_downloads+=1;
+				println!("sadly error: {e}");
+				break Ok(DownloadResult::Failed);
+			},
+		}
+	}
+}
+#[derive(Debug,thiserror::Error)]
+enum ConvertTextureError{
+	#[error("Io error {0:?}")]
+	Io(#[from]std::io::Error),
+	#[error("Image error {0:?}")]
+	Image(#[from]image::ImageError),
+	#[error("DDS create error {0:?}")]
+	DDS(#[from]image_dds::CreateDdsError),
+	#[error("DDS write error {0:?}")]
+	DDSWrite(#[from]image_dds::ddsfile::Error),
+}
+async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
+	let data=match download_result{
+		DownloadResult::Cached(path)=>tokio::fs::read(path).await?,
+		DownloadResult::Data(data)=>data,
+		DownloadResult::Failed=>return Ok(()),
+	};
+	// image::ImageFormat::Png
+	// image::ImageFormat::Jpeg
+	let image=image::load_from_memory(&data)?.to_rgba8();
+
+	// pick format
+	let format=if image.width()%4!=0||image.height()%4!=0{
+		image_dds::ImageFormat::Rgba8UnormSrgb
+	}else{
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
+	};
+
+	//this fails if the image dimensions are not a multiple of 4
+	let dds=image_dds::dds_from_image(
+		&image,
+		format,
+		image_dds::Quality::Slow,
+		image_dds::Mipmaps::GeneratedAutomatic,
+	)?;
+
+	let file_name=format!("textures/{}.dds",asset_id.0);
+	let mut file=std::fs::File::create(file_name)?;
+	dds.write(&mut file)?;
+	Ok(())
+}
+async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
+	tokio::try_join!(
+		tokio::fs::create_dir_all("downloaded_textures"),
+		tokio::fs::create_dir_all("textures"),
+		tokio::fs::create_dir_all("meshes"),
+		tokio::fs::create_dir_all("unions"),
+	)?;
+	// use mpsc
+	let thread_limit=std::thread::available_parallelism()?.get();
+	let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
+	let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+	// map decode dispatcher
+	// read files multithreaded
+	// produce UniqueAssetsResult per file
+	tokio::spawn(async move{
+		// move send so it gets dropped when all maps have been decoded
+		// closing the channel
+		let mut it=paths.into_iter();
+		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+		SEM.add_permits(thread_limit);
+		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+			let send=send_assets.clone();
+			tokio::spawn(async move{
+				let result=unique_assets(path.as_path()).await;
+				_=send.send(result).await;
+				drop(permit);
+			});
+		}
+	});
+	// download manager
+	// insert into global unique assets guy
+	// add to download queue if the asset is globally unique and does not already exist on disk
+	let mut stats=Stats::default();
+	let context=rbx_asset::cookie::CookieContext::new(cookie);
+	let mut globally_unique_assets=UniqueAssets::default();
+	// pop a job = retry_queue.pop_front() or ingest(recv.recv().await)
+	// SLOW MODE:
+	// acquire all permits
+	// drop all permits
+	// pop one job
+	// if it succeeds go into fast mode
+	// FAST MODE:
+	// acquire one permit
+	// pop a job
+	let download_thread=tokio::spawn(async move{
+		while let Some(result)=recv_assets.recv().await{
+			let unique_assets=match result{
+				Ok(unique_assets)=>unique_assets,
+				Err(e)=>{
+					println!("error: {e:?}");
+					continue;
+				},
+			};
+			for texture_id in unique_assets.textures{
+				if globally_unique_assets.textures.insert(texture_id){
+					let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
+					send_texture.send((texture_id,data)).await?;
+				}
+			}
+			for mesh_id in unique_assets.meshes{
+				if globally_unique_assets.meshes.insert(mesh_id){
+					download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
+				}
+			}
+			for union_id in unique_assets.unions{
+				if globally_unique_assets.unions.insert(union_id){
+					download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
+				}
+			}
+		}
+		dbg!(stats);
+		Ok::<(),anyhow::Error>(())
+	});
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+	while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){
+		tokio::spawn(async move{
+			let result=convert_texture(asset_id,download_result).await;
+			drop(permit);
+			result.unwrap();
+		});
+	}
+	download_thread.await??;
+	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
+	Ok(())
+}
+
+#[derive(Debug)]
+#[allow(dead_code)]
+enum ConvertError{
+	IO(std::io::Error),
+	SNFMap(strafesnet_snf::map::Error),
+	RobloxRead(strafesnet_rbx_loader::ReadError),
+	RobloxLoad(strafesnet_rbx_loader::LoadError),
+}
+impl std::fmt::Display for ConvertError{
+	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+		write!(f,"{self:?}")
+	}
+}
+impl std::error::Error for ConvertError{}
+async fn convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
+	let entire_file=tokio::fs::read(path).await?;
+
+	let model=strafesnet_rbx_loader::read(
+		std::io::Cursor::new(entire_file)
+	).map_err(ConvertError::RobloxRead)?;
+
+	let mut place=model.into_place();
+	place.run_scripts();
+
+	let map=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?;
+
+	let mut dest=output_folder;
+	dest.push(path.file_stem().unwrap());
+	dest.set_extension("snfm");
+	let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
+
+	strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
+
+	Ok(())
+}
+
+async fn roblox_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+	let start=std::time::Instant::now();
+
+	let thread_limit=std::thread::available_parallelism()?.get();
+	let mut it=paths.into_iter();
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+
+	while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+		let output_folder=output_folder.clone();
+		tokio::spawn(async move{
+			let result=convert_to_snf(path.as_path(),output_folder).await;
+			drop(permit);
+			match result{
+				Ok(())=>(),
+				Err(e)=>println!("Convert error: {e:?}"),
+			}
+		});
+	}
+	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
+
+	println!("elapsed={:?}", start.elapsed());
+	Ok(())
+}

map-tool/src/source.rs

@@ -0,0 +1,464 @@
+use std::path::{Path,PathBuf};
+use std::borrow::Cow;
+use clap::{Args,Subcommand};
+use anyhow::Result as AResult;
+use futures::StreamExt;
+use strafesnet_bsp_loader::loader::BspFinder;
+use strafesnet_deferred_loader::loader::Loader;
+use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLoader,RenderConfigDeferredLoader};
+
+#[derive(Subcommand)]
+pub enum Commands{
+	SourceToSNF(SourceToSNFSubcommand),
+	ExtractTextures(ExtractTexturesSubcommand),
+	VPKContents(VPKContentsSubcommand),
+	BSPContents(BSPContentsSubcommand),
+}
+
+#[derive(Args)]
+pub struct SourceToSNFSubcommand {
+	#[arg(long)]
+	output_folder:PathBuf,
+	#[arg(required=true)]
+	input_files:Vec<PathBuf>,
+	#[arg(long)]
+	vpks:Vec<PathBuf>,
+}
+#[derive(Args)]
+pub struct ExtractTexturesSubcommand{
+	#[arg(required=true)]
+	bsp_files:Vec<PathBuf>,
+	#[arg(long)]
+	vpks:Vec<PathBuf>,
+}
+#[derive(Args)]
+pub struct VPKContentsSubcommand {
+	#[arg(long)]
+	input_file:PathBuf,
+}
+#[derive(Args)]
+pub struct BSPContentsSubcommand {
+	#[arg(long)]
+	input_file:PathBuf,
+}
+
+impl Commands{
+	pub async fn run(self)->AResult<()>{
+		match self{
+			Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder,subcommand.vpks).await,
+			Commands::ExtractTextures(subcommand)=>extract_textures(subcommand.bsp_files,subcommand.vpks).await,
+			Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
+			Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
+		}
+	}
+}
+
+
+enum VMTContent{
+	VMT(String),
+	VTF(String),
+	Patch(vmt_parser::material::PatchMaterial),
+	Unsupported,//don't want to deal with whatever vmt variant
+	Unresolved,//could not locate a texture because of vmt content
+}
+impl VMTContent{
+	fn vtf(opt:Option<String>)->Self{
+		match opt{
+			Some(s)=>Self::VTF(s),
+			None=>Self::Unresolved,
+		}
+	}
+}
+
+fn get_some_texture(material:vmt_parser::material::Material)->VMTContent{
+	//just grab some texture from somewhere for now
+	match material{
+		vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none
+		vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
+		vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture),
+		vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture),
+		vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture),
+		vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)),
+		vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),//recursive
+		vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture),
+		vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::Refract(mat)=>VMTContent::vtf(mat.base_texture),
+		vmt_parser::material::Material::Modulate(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::DecalModulate(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)),
+		vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported,
+		vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat),
+		_=>unreachable!(),
+	}
+}
+
+#[derive(Debug,thiserror::Error)]
+enum GetVMTError{
+	#[error("Bsp error {0:?}")]
+	Bsp(#[from]vbsp::BspError),
+	#[error("Utf8 error {0:?}")]
+	Utf8(#[from]std::str::Utf8Error),
+	#[error("Vdf error {0:?}")]
+	Vdf(#[from]vmt_parser::VdfError),
+	#[error("Vmt not found")]
+	NotFound,
+}
+
+fn get_vmt(finder:BspFinder,search_name:&str)->Result<vmt_parser::material::Material,GetVMTError>{
+	let vmt_data=finder.find(search_name)?.ok_or(GetVMTError::NotFound)?;
+	//decode vmt and then write
+	let vmt_str=core::str::from_utf8(&vmt_data)?;
+	let material=vmt_parser::from_str(vmt_str)?;
+	//println!("vmt material={:?}",material);
+	Ok(material)
+}
+
+#[derive(Debug,thiserror::Error)]
+enum LoadVMTError{
+	#[error("Bsp error {0:?}")]
+	Bsp(#[from]vbsp::BspError),
+	#[error("GetVMT error {0:?}")]
+	GetVMT(#[from]GetVMTError),
+	#[error("FromUtf8 error {0:?}")]
+	FromUtf8(#[from]std::string::FromUtf8Error),
+	#[error("Vdf error {0:?}")]
+	Vdf(#[from]vmt_parser::VdfError),
+	#[error("Vmt unsupported")]
+	Unsupported,
+	#[error("Vmt unresolved")]
+	Unresolved,
+	#[error("Vmt not found")]
+	NotFound,
+}
+fn recursive_vmt_loader<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,material:vmt_parser::material::Material)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
+	where
+		'bsp:'a,
+		'vpk:'a,
+{
+	match get_some_texture(material){
+		VMTContent::VMT(mut s)=>{
+			s.make_ascii_lowercase();
+			recursive_vmt_loader(finder,get_vmt(finder,&s)?)
+		},
+		VMTContent::VTF(s)=>{
+			let mut texture_file_name=PathBuf::from("materials");
+			texture_file_name.push(s);
+			texture_file_name.set_extension("vtf");
+			texture_file_name.as_mut_os_str().make_ascii_lowercase();
+			Ok(finder.find(texture_file_name.to_str().unwrap())?)
+		},
+		VMTContent::Patch(mat)=>recursive_vmt_loader(finder,
+			mat.resolve(|search_name|{
+				let name_lowercase=search_name.to_lowercase();
+				match finder.find(&name_lowercase)?{
+					Some(bytes)=>Ok(String::from_utf8(bytes.into_owned())?),
+					None=>Err(LoadVMTError::NotFound),
+				}
+			})?
+		),
+		VMTContent::Unsupported=>Err(LoadVMTError::Unsupported),
+		VMTContent::Unresolved=>Err(LoadVMTError::Unresolved),
+	}
+}
+fn load_texture<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,texture_name:&str)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
+	where
+		'bsp:'a,
+		'vpk:'a,
+{
+	let mut texture_file_name=PathBuf::from("materials");
+	//lower case
+	texture_file_name.push(texture_name);
+	texture_file_name.as_mut_os_str().make_ascii_lowercase();
+	//remove stem and search for both vtf and vmt files
+	let stem=texture_file_name.file_stem().unwrap().to_owned();
+	texture_file_name.pop();
+	texture_file_name.push(stem);
+	if let Some(stuff)=finder.find(texture_file_name.to_str().unwrap())?{
+		return Ok(Some(stuff));
+	}
+
+	// search for both vmt,vtf
+	let mut texture_file_name_vmt=texture_file_name.clone();
+	texture_file_name_vmt.set_extension("vmt");
+
+	let get_vmt_result=get_vmt(finder,texture_file_name_vmt.to_str().unwrap());
+	match get_vmt_result{
+		Ok(material)=>{
+			let vmt_result=recursive_vmt_loader(finder,material);
+			match vmt_result{
+				Ok(Some(stuff))=>return Ok(Some(stuff)),
+				Ok(None)
+				|Err(LoadVMTError::NotFound)=>(),
+				|Err(LoadVMTError::GetVMT(GetVMTError::NotFound))=>(),
+				Err(e)=>return Err(e),
+			}
+		}
+		|Err(GetVMTError::NotFound)=>(),
+		Err(e)=>Err(e)?,
+	}
+
+	// try looking for vtf
+	let mut texture_file_name_vtf=texture_file_name.clone();
+	texture_file_name_vtf.set_extension("vtf");
+
+	let get_vtf_result=get_vmt(finder,texture_file_name_vtf.to_str().unwrap());
+	match get_vtf_result{
+		Ok(material)=>{
+			let vtf_result=recursive_vmt_loader(finder,material);
+			match vtf_result{
+				Ok(Some(stuff))=>return Ok(Some(stuff)),
+				Ok(None)
+				|Err(LoadVMTError::NotFound)=>(),
+				|Err(LoadVMTError::GetVMT(GetVMTError::NotFound))=>(),
+				Err(e)=>return Err(e),
+			}
+		}
+		|Err(GetVMTError::NotFound)=>(),
+		Err(e)=>Err(e)?,
+	}
+
+	Ok(None)
+}
+#[derive(Debug,thiserror::Error)]
+enum ExtractTextureError{
+	#[error("Io error {0:?}")]
+	Io(#[from]std::io::Error),
+	#[error("Bsp error {0:?}")]
+	Bsp(#[from]vbsp::BspError),
+	#[error("MeshLoad error {0:?}")]
+	MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError),
+	#[error("Load VMT error {0:?}")]
+	LoadVMT(#[from]LoadVMTError),
+}
+async fn gimme_them_textures(path:&Path,vpk_list:&[strafesnet_bsp_loader::Vpk],send_texture:tokio::sync::mpsc::Sender<(Vec<u8>,String)>)->Result<(),ExtractTextureError>{
+	let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?;
+	let loader_bsp=strafesnet_bsp_loader::Bsp::new(bsp);
+	let bsp=loader_bsp.as_ref();
+
+	let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
+	for texture in bsp.textures(){
+		texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name())));
+	}
+
+	let mut mesh_deferred_loader=MeshDeferredLoader::new();
+	for prop in bsp.static_props(){
+		mesh_deferred_loader.acquire_mesh_id(prop.model());
+	}
+
+	let finder=BspFinder{
+		bsp:&loader_bsp,
+		vpks:vpk_list
+	};
+
+	let mut mesh_loader=strafesnet_bsp_loader::loader::ModelLoader::new(finder);
+	// load models and collect requested textures
+	for model_path in mesh_deferred_loader.into_indices(){
+		let model:vmdl::Model=match mesh_loader.load(model_path){
+			Ok(model)=>model,
+			Err(e)=>{
+				println!("Model={model_path} Load model error: {e}");
+				continue;
+			},
+		};
+		for texture in model.textures(){
+			for search_path in &texture.search_paths{
+				let mut path=PathBuf::from(search_path.as_str());
+				path.push(texture.name.as_str());
+				let path=path.to_str().unwrap().to_owned();
+				texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path)));
+			}
+		}
+	}
+
+	for texture_path in texture_deferred_loader.into_indices(){
+		match load_texture(finder,&texture_path){
+			Ok(Some(texture))=>send_texture.send(
+				(texture.into_owned(),texture_path.into_owned())
+			).await.unwrap(),
+			Ok(None)=>(),
+			Err(e)=>println!("Texture={texture_path} Load error: {e}"),
+		}
+	}
+
+	Ok(())
+}
+
+
+#[derive(Debug,thiserror::Error)]
+enum ConvertTextureError{
+	#[error("Bsp error {0:?}")]
+	Bsp(#[from]vbsp::BspError),
+	#[error("Vtf error {0:?}")]
+	Vtf(#[from]vtf::Error),
+	#[error("DDS create error {0:?}")]
+	DDS(#[from]image_dds::CreateDdsError),
+	#[error("DDS write error {0:?}")]
+	DDSWrite(#[from]image_dds::ddsfile::Error),
+	#[error("Io error {0:?}")]
+	Io(#[from]std::io::Error),
+}
+
+async fn convert_texture(texture:Vec<u8>,write_file_name:impl AsRef<Path>)->Result<(),ConvertTextureError>{
+	let image=vtf::from_bytes(&texture)?.highres_image.decode(0)?.to_rgba8();
+
+	let format=if image.width()%4!=0||image.height()%4!=0{
+		image_dds::ImageFormat::Rgba8UnormSrgb
+	}else{
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
+	};
+	//this fails if the image dimensions are not a multiple of 4
+	let dds = image_dds::dds_from_image(
+		&image,
+		format,
+		image_dds::Quality::Slow,
+		image_dds::Mipmaps::GeneratedAutomatic,
+	)?;
+
+	//write dds
+	let mut dest=PathBuf::from("textures");
+	dest.push(write_file_name);
+	dest.set_extension("dds");
+	std::fs::create_dir_all(dest.parent().unwrap())?;
+	let mut writer=std::io::BufWriter::new(std::fs::File::create(dest)?);
+	dds.write(&mut writer)?;
+
+	Ok(())
+}
+
+async fn read_vpks(vpk_paths:Vec<PathBuf>,thread_limit:usize)->Vec<strafesnet_bsp_loader::Vpk>{
+	futures::stream::iter(vpk_paths).map(|vpk_path|async{
+		// idk why it doesn't want to pass out the errors but this is fatal anyways
+		tokio::task::spawn_blocking(move||Ok::<_,vpk::Error>(strafesnet_bsp_loader::Vpk::new(vpk::VPK::read(&vpk_path)?))).await.unwrap().unwrap()
+	})
+	.buffer_unordered(thread_limit)
+	.collect().await
+}
+
+async fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
+	tokio::try_join!(
+		tokio::fs::create_dir_all("extracted_textures"),
+		tokio::fs::create_dir_all("textures"),
+		tokio::fs::create_dir_all("meshes"),
+	)?;
+	let thread_limit=std::thread::available_parallelism()?.get();
+
+	// load vpk list and leak for static lifetime
+	let vpk_list:&[strafesnet_bsp_loader::Vpk]=read_vpks(vpk_paths,thread_limit).await.leak();
+
+	let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+	let mut it=paths.into_iter();
+	let extract_thread=tokio::spawn(async move{
+		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+		SEM.add_permits(thread_limit);
+		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+			let send=send_texture.clone();
+			tokio::spawn(async move{
+				let result=gimme_them_textures(&path,vpk_list,send).await;
+				drop(permit);
+				match result{
+					Ok(())=>(),
+					Err(e)=>println!("Map={path:?} Decode error: {e:?}"),
+				}
+			});
+		}
+	});
+
+	// convert images
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+	while let (Ok(permit),Some((data,dest)))=(SEM.acquire().await,recv_texture.recv().await){
+		// TODO: dedup dest?
+		tokio::spawn(async move{
+			let result=convert_texture(data,dest).await;
+			drop(permit);
+			match result{
+				Ok(())=>(),
+				Err(e)=>println!("Convert error: {e:?}"),
+			}
+		});
+	}
+	extract_thread.await?;
+	_=SEM.acquire_many(thread_limit as u32).await?;
+	Ok(())
+}
+
+fn vpk_contents(vpk_path:PathBuf)->AResult<()>{
+	let vpk_index=vpk::VPK::read(&vpk_path)?;
+	for (label,entry) in vpk_index.tree.into_iter(){
+		println!("vpk label={} entry={:?}",label,entry);
+	}
+	Ok(())
+}
+
+fn bsp_contents(path:PathBuf)->AResult<()>{
+	let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
+	for file_name in bsp.pack.into_zip().into_inner().unwrap().file_names(){
+		println!("file_name={:?}",file_name);
+	}
+	Ok(())
+}
+
+#[derive(Debug)]
+#[allow(dead_code)]
+enum ConvertError{
+	IO(std::io::Error),
+	SNFMap(strafesnet_snf::map::Error),
+	BspRead(strafesnet_bsp_loader::ReadError),
+	BspLoad(strafesnet_bsp_loader::LoadError),
+}
+impl std::fmt::Display for ConvertError{
+	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+		write!(f,"{self:?}")
+	}
+}
+impl std::error::Error for ConvertError{}
+
+async fn convert_to_snf(path:&Path,vpk_list:&[strafesnet_bsp_loader::Vpk],output_folder:PathBuf)->AResult<()>{
+	let entire_file=tokio::fs::read(path).await?;
+
+	let bsp=strafesnet_bsp_loader::read(
+		std::io::Cursor::new(entire_file)
+	).map_err(ConvertError::BspRead)?;
+
+	let map=bsp.to_snf(LoadFailureMode::DefaultToNone,vpk_list).map_err(ConvertError::BspLoad)?;
+
+	let mut dest=output_folder;
+	dest.push(path.file_stem().unwrap());
+	dest.set_extension("snfm");
+	let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
+
+	strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
+
+	Ok(())
+}
+async fn source_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf,vpk_paths:Vec<PathBuf>)->AResult<()>{
+	let start=std::time::Instant::now();
+
+	let thread_limit=std::thread::available_parallelism()?.get();
+
+	// load vpk list and leak for static lifetime
+	let vpk_list:&[strafesnet_bsp_loader::Vpk]=read_vpks(vpk_paths,thread_limit).await.leak();
+
+	let mut it=paths.into_iter();
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+
+	while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+		let output_folder=output_folder.clone();
+		tokio::spawn(async move{
+			let result=convert_to_snf(path.as_path(),vpk_list,output_folder).await;
+			drop(permit);
+			match result{
+				Ok(())=>(),
+				Err(e)=>println!("Convert error: {e:?}"),
+			}
+		});
+	}
+	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
+
+	println!("elapsed={:?}", start.elapsed());
+	Ok(())
+}