From 5f933407a9d256d2cc5564ce1ce1a976554991d5 Mon Sep 17 00:00:00 2001 From: itzaname Date: Sat, 28 Feb 2026 23:53:58 -0500 Subject: [PATCH] Support library usage --- Cargo.lock | 2 +- Cargo.toml | 16 +- src/lib.rs | 2 + src/main.rs | 7 +- src/roblox.rs | 726 ++++++++++++++++++++++++----------------------- src/source.rs | 766 +++++++++++++++++++++++++------------------------- 6 files changed, 785 insertions(+), 734 deletions(-) create mode 100644 src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 797db5f..9241e7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1568,7 +1568,7 @@ dependencies = [ [[package]] name = "map-tool" -version = "1.7.0" +version = "2.0.0" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index 0f541f5..a9a9f2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,25 @@ [package] name = "map-tool" -version = "1.7.0" +version = "2.0.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "map_tool" +path = "src/lib.rs" + +[[bin]] +name = "map-tool" +path = "src/main.rs" +required-features = ["cli"] + +[features] +cli = ["dep:clap"] + [dependencies] anyhow = "1.0.75" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.2", features = ["derive"], optional = true } flate2 = "1.0.27" futures = "0.3.31" image = "0.25.2" diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..066b8c0 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,2 @@ +pub mod roblox; +pub mod source; diff --git a/src/main.rs b/src/main.rs index da5352b..5d2b0c6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,3 @@ -mod roblox; -mod source; - use clap::{Parser,Subcommand}; use anyhow::Result as AResult; @@ -15,9 +12,9 @@ struct Cli { #[derive(Subcommand)] enum Commands{ #[command(flatten)] - Roblox(roblox::Commands), + Roblox(map_tool::roblox::Commands), #[command(flatten)] - Source(source::Commands), + Source(map_tool::source::Commands), } #[tokio::main] diff --git a/src/roblox.rs b/src/roblox.rs index da2c0d1..e27980e 100644 --- a/src/roblox.rs +++ b/src/roblox.rs @@ -1,60 +1,40 @@ -use std::path::{Path,PathBuf}; use std::io::{Cursor,Read,Seek}; use std::collections::HashSet; -use clap::{Args,Subcommand}; -use anyhow::Result as AResult; use rbx_dom_weak::Instance; use strafesnet_deferred_loader::deferred_loader::LoadFailureMode; use rbxassetid::RobloxAssetId; -use tokio::io::AsyncReadExt; -const DOWNLOAD_LIMIT:usize=16; +// === Public library API === -#[derive(Subcommand)] -pub enum Commands{ - RobloxToSNF(RobloxToSNFSubcommand), - DownloadAssets(DownloadAssetsSubcommand), +/// Unique asset IDs referenced by a Roblox place/model file. +#[derive(Default)] +pub struct UniqueAssets{ + pub meshes:HashSet, + pub unions:HashSet, + pub textures:HashSet, } -#[derive(Args)] -pub struct RobloxToSNFSubcommand { - #[arg(long)] - output_folder:PathBuf, - #[arg(required=true)] - input_files:Vec, -} -#[derive(Args)] -pub struct DownloadAssetsSubcommand{ - #[arg(required=true)] - roblox_files:Vec, - // #[arg(long)] - // cookie_file:Option, -} - -impl Commands{ - pub async fn run(self)->AResult<()>{ - match self{ - Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder).await, - Commands::DownloadAssets(subcommand)=>download_assets( - subcommand.roblox_files, - rbx_asset::cookie::Cookie::new("".to_string()), - ).await, - } - } -} - -#[allow(unused)] -#[derive(Debug)] -enum LoadDomError{ - IO(std::io::Error), +#[derive(Debug,thiserror::Error)] +pub enum LoadDomError{ + #[error("IO error {0:?}")] + IO(#[from]std::io::Error), + #[error("Binary decode error {0:?}")] Binary(rbx_binary::DecodeError), + #[error("XML decode error {0:?}")] Xml(rbx_xml::DecodeError), + #[error("Unknown file format")] UnknownFormat, } -fn load_dom(mut input:R)->Result{ + +/// Parse a Roblox file (binary or XML) from bytes into a WeakDom. +pub fn load_dom(data:&[u8])->Result{ + load_dom_reader(Cursor::new(data)) +} + +fn load_dom_reader(mut input:R)->Result{ let mut first_8=[0u8;8]; - input.read_exact(&mut first_8).map_err(LoadDomError::IO)?; - input.rewind().map_err(LoadDomError::IO)?; + input.read_exact(&mut first_8)?; + input.rewind()?; match &first_8{ b"rbx_binary::from_reader(input).map_err(LoadDomError::Binary), b"rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(LoadDomError::Xml), @@ -62,56 +42,113 @@ fn load_dom(mut input:R)->Result,object:&Instance,property:&str){ - if let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(property){ - let url:&str=content.as_ref(); - if let Ok(asset_id)=url.parse(){ - content_list.insert(asset_id); - }else{ - println!("Content failed to parse into AssetID: {:?}",content); - } - }else{ - println!("property={} does not exist for class={}",property,object.class.as_str()); +/// Scan a parsed DOM and return all unique asset IDs (meshes, textures, unions). +pub fn get_unique_assets(dom:rbx_dom_weak::WeakDom)->UniqueAssets{ + let mut assets=UniqueAssets::default(); + for object in dom.into_raw().1.into_values(){ + assets.collect(&object); } + assets } -async fn read_entire_file(path:impl AsRef)->Result>,std::io::Error>{ - let mut file=tokio::fs::File::open(path).await?; - let mut data=Vec::new(); - file.read_to_end(&mut data).await?; - Ok(Cursor::new(data)) + +/// Scan a Roblox file (bytes) and return all unique asset IDs. +pub fn get_unique_assets_from_file(data:&[u8])->Result{ + let dom=load_dom(data)?; + Ok(get_unique_assets(dom)) } -#[derive(Default)] -struct UniqueAssets{ - meshes:HashSet, - unions:HashSet, - textures:HashSet, + +#[derive(Debug,thiserror::Error)] +pub enum ConvertTextureError{ + #[error("Image error {0:?}")] + Image(#[from]image::ImageError), + #[error("DDS create error {0:?}")] + DDS(#[from]image_dds::CreateDdsError), + #[error("DDS write error {0:?}")] + DDSWrite(#[from]image_dds::ddsfile::Error), } + +/// Convert image bytes (PNG, JPEG, etc.) into DDS texture bytes. +pub fn convert_texture_to_dds(image_data:&[u8])->Result,ConvertTextureError>{ + let image=image::load_from_memory(image_data)?.to_rgba8(); + + let format=if image.width()%4!=0||image.height()%4!=0{ + image_dds::ImageFormat::Rgba8UnormSrgb + }else{ + image_dds::ImageFormat::BC7RgbaUnormSrgb + }; + + let dds=image_dds::dds_from_image( + &image, + format, + image_dds::Quality::Slow, + image_dds::Mipmaps::GeneratedAutomatic, + )?; + + let mut buf=Vec::new(); + dds.write(&mut Cursor::new(&mut buf))?; + Ok(buf) +} + +#[derive(Debug,thiserror::Error)] +pub enum ConvertError{ + #[error("IO error {0:?}")] + IO(#[from]std::io::Error), + #[error("SNF map error {0:?}")] + SNFMap(strafesnet_snf::map::Error), + #[error("Roblox read error {0:?}")] + RobloxRead(strafesnet_rbx_loader::ReadError), + #[error("Roblox load error {0:?}")] + RobloxLoad(strafesnet_rbx_loader::LoadError), +} + +/// Convert a Roblox place/model file (bytes) to SNF map format (bytes). +pub fn convert_to_snf(data:&[u8])->Result,ConvertError>{ + let model=strafesnet_rbx_loader::read( + Cursor::new(data) + ).map_err(ConvertError::RobloxRead)?; + + let mut place=model.into_place(); + place.run_scripts(); + + let map=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?; + + let mut buf=Vec::new(); + strafesnet_snf::map::write_map(Cursor::new(&mut buf),map).map_err(ConvertError::SNFMap)?; + Ok(buf) +} + +/// Download a single asset from Roblox by ID. Returns raw asset bytes. +pub async fn download_asset(context:&rbx_asset::cookie::CookieContext,asset_id:u64)->Result,rbx_asset::cookie::GetError>{ + context.get_asset(rbx_asset::cookie::GetAssetRequest{ + asset_id, + version:None, + }).await +} + +/// Download a single asset with retry and exponential backoff for rate limiting. +/// Returns None if all retries are exhausted or a non-rate-limit error occurs. +pub async fn download_asset_retry(context:&rbx_asset::cookie::CookieContext,asset_id:u64)->Option>{ + const BACKOFF_MUL:f32=1.3956124250860895286; + let mut backoff=1000f32; + for _ in 0..12{ + match download_asset(context,asset_id).await{ + Ok(data)=>return Some(data), + Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{ + if scwuab.status_code.as_u16()==429{ + tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await; + backoff*=BACKOFF_MUL; + }else{ + return None; + } + }, + Err(_)=>return None, + } + } + None +} + +// === Private helpers === + impl UniqueAssets{ fn collect(&mut self,object:&Instance){ match object.class.as_str(){ @@ -141,291 +178,280 @@ impl UniqueAssets{ } } -#[allow(unused)] -#[derive(Debug)] -enum UniqueAssetError{ - IO(std::io::Error), - LoadDom(LoadDomError), -} -async fn unique_assets(path:&Path)->Result{ - // read entire file - let mut assets=UniqueAssets::default(); - let data=read_entire_file(path).await.map_err(UniqueAssetError::IO)?; - let dom=load_dom(data).map_err(UniqueAssetError::LoadDom)?; - for object in dom.into_raw().1.into_values(){ - assets.collect(&object); +fn accumulate_content_id(content_list:&mut HashSet,object:&Instance,property:&str){ + if let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(property){ + let url:&str=content.as_ref(); + if let Ok(asset_id)=url.parse(){ + content_list.insert(asset_id); + }else{ + println!("Content failed to parse into AssetID: {:?}",content); + } + }else{ + println!("property={} does not exist for class={}",property,object.class.as_str()); } - Ok(assets) } -enum DownloadType{ - Texture(RobloxAssetId), - Mesh(RobloxAssetId), - Union(RobloxAssetId), -} -impl DownloadType{ - fn path(&self)->PathBuf{ - match self{ - DownloadType::Texture(asset_id)=>format!("downloaded_textures/{}",asset_id.0.to_string()).into(), - DownloadType::Mesh(asset_id)=>format!("meshes/{}",asset_id.0.to_string()).into(), - DownloadType::Union(asset_id)=>format!("unions/{}",asset_id.0.to_string()).into(), + +// === CLI === + +#[cfg(feature="cli")] +mod cli{ + use super::*; + use std::path::{Path,PathBuf}; + use clap::{Args,Subcommand}; + use anyhow::Result as AResult; + use tokio::io::AsyncReadExt; + + const DOWNLOAD_LIMIT:usize=16; + + #[derive(Subcommand)] + pub enum Commands{ + RobloxToSNF(RobloxToSNFSubcommand), + DownloadAssets(DownloadAssetsSubcommand), + } + + #[derive(Args)] + pub struct RobloxToSNFSubcommand { + #[arg(long)] + output_folder:PathBuf, + #[arg(required=true)] + input_files:Vec, + } + #[derive(Args)] + pub struct DownloadAssetsSubcommand{ + #[arg(required=true)] + roblox_files:Vec, + } + + impl Commands{ + pub async fn run(self)->AResult<()>{ + match self{ + Commands::RobloxToSNF(subcommand)=>cli_roblox_to_snf(subcommand.input_files,subcommand.output_folder).await, + Commands::DownloadAssets(subcommand)=>cli_download_assets( + subcommand.roblox_files, + rbx_asset::cookie::Cookie::new("".to_string()), + ).await, + } } } - fn asset_id(&self)->u64{ - match self{ - DownloadType::Texture(asset_id)=>asset_id.0, - DownloadType::Mesh(asset_id)=>asset_id.0, - DownloadType::Union(asset_id)=>asset_id.0, + + async fn read_entire_file(path:impl AsRef)->Result,std::io::Error>{ + let mut file=tokio::fs::File::open(path).await?; + let mut data=Vec::new(); + file.read_to_end(&mut data).await?; + Ok(data) + } + + enum DownloadType{ + Texture(RobloxAssetId), + Mesh(RobloxAssetId), + Union(RobloxAssetId), + } + impl DownloadType{ + fn path(&self)->PathBuf{ + match self{ + DownloadType::Texture(asset_id)=>format!("downloaded_textures/{}",asset_id.0.to_string()).into(), + DownloadType::Mesh(asset_id)=>format!("meshes/{}",asset_id.0.to_string()).into(), + DownloadType::Union(asset_id)=>format!("unions/{}",asset_id.0.to_string()).into(), + } + } + fn asset_id(&self)->u64{ + match self{ + DownloadType::Texture(asset_id)=>asset_id.0, + DownloadType::Mesh(asset_id)=>asset_id.0, + DownloadType::Union(asset_id)=>asset_id.0, + } } } -} -enum DownloadResult{ - Cached(PathBuf), - Data(Vec), - Failed, -} -#[derive(Default,Debug)] -struct Stats{ - total_assets:u32, - cached_assets:u32, - downloaded_assets:u32, - failed_downloads:u32, - timed_out_downloads:u32, -} -async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result{ - stats.total_assets+=1; - let download_instruction=download_instruction; - // check if file exists on disk - let path=download_instruction.path(); - if tokio::fs::try_exists(path.as_path()).await?{ - stats.cached_assets+=1; - return Ok(DownloadResult::Cached(path)); + enum DownloadResult{ + Cached(PathBuf), + Data(Vec), + Failed, } - let asset_id=download_instruction.asset_id(); - // if not, download file - let mut retry=0; - const BACKOFF_MUL:f32=1.3956124250860895286;//exp(1/3) - let mut backoff=1000f32; - loop{ - let asset_result=context.get_asset(rbx_asset::cookie::GetAssetRequest{ - asset_id, - version:None, - }).await; - match asset_result{ - Ok(asset_result)=>{ - stats.downloaded_assets+=1; - tokio::fs::write(path,&asset_result).await?; - break Ok(DownloadResult::Data(asset_result)); - }, - Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{ - if scwuab.status_code.as_u16()==429{ - if retry==12{ - println!("Giving up asset download {asset_id}"); - stats.timed_out_downloads+=1; + #[derive(Default,Debug)] + struct Stats{ + total_assets:u32, + cached_assets:u32, + downloaded_assets:u32, + failed_downloads:u32, + timed_out_downloads:u32, + } + async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result{ + stats.total_assets+=1; + let path=download_instruction.path(); + if tokio::fs::try_exists(path.as_path()).await?{ + stats.cached_assets+=1; + return Ok(DownloadResult::Cached(path)); + } + let asset_id=download_instruction.asset_id(); + let mut retry=0; + const BACKOFF_MUL:f32=1.3956124250860895286; + let mut backoff=1000f32; + loop{ + let asset_result=context.get_asset(rbx_asset::cookie::GetAssetRequest{ + asset_id, + version:None, + }).await; + match asset_result{ + Ok(asset_result)=>{ + stats.downloaded_assets+=1; + tokio::fs::write(path,&asset_result).await?; + break Ok(DownloadResult::Data(asset_result)); + }, + Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{ + if scwuab.status_code.as_u16()==429{ + if retry==12{ + println!("Giving up asset download {asset_id}"); + stats.timed_out_downloads+=1; + break Ok(DownloadResult::Failed); + } + println!("Hit roblox rate limit, waiting {:.0}ms...",backoff); + tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await; + backoff*=BACKOFF_MUL; + retry+=1; + }else{ + stats.failed_downloads+=1; + println!("weird scuwab error: {scwuab:?}"); break Ok(DownloadResult::Failed); } - println!("Hit roblox rate limit, waiting {:.0}ms...",backoff); - tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await; - backoff*=BACKOFF_MUL; - retry+=1; - }else{ + }, + Err(e)=>{ stats.failed_downloads+=1; - println!("weird scuwab error: {scwuab:?}"); + println!("sadly error: {e}"); break Ok(DownloadResult::Failed); - } - }, - Err(e)=>{ - stats.failed_downloads+=1; - println!("sadly error: {e}"); - break Ok(DownloadResult::Failed); - }, + }, + } } } -} -#[derive(Debug,thiserror::Error)] -enum ConvertTextureError{ - #[error("Io error {0:?}")] - Io(#[from]std::io::Error), - #[error("Image error {0:?}")] - Image(#[from]image::ImageError), - #[error("DDS create error {0:?}")] - DDS(#[from]image_dds::CreateDdsError), - #[error("DDS write error {0:?}")] - DDSWrite(#[from]image_dds::ddsfile::Error), -} -async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{ - let data=match download_result{ - DownloadResult::Cached(path)=>tokio::fs::read(path).await?, - DownloadResult::Data(data)=>data, - DownloadResult::Failed=>return Ok(()), - }; - // image::ImageFormat::Png - // image::ImageFormat::Jpeg - let image=image::load_from_memory(&data)?.to_rgba8(); - // pick format - let format=if image.width()%4!=0||image.height()%4!=0{ - image_dds::ImageFormat::Rgba8UnormSrgb - }else{ - image_dds::ImageFormat::BC7RgbaUnormSrgb - }; + async fn cli_convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),CliConvertTextureError>{ + let data=match download_result{ + DownloadResult::Cached(path)=>tokio::fs::read(path).await?, + DownloadResult::Data(data)=>data, + DownloadResult::Failed=>return Ok(()), + }; + let dds_data=convert_texture_to_dds(&data)?; + let file_name=format!("textures/{}.dds",asset_id.0); + tokio::fs::write(file_name,dds_data).await?; + Ok(()) + } - //this fails if the image dimensions are not a multiple of 4 - let dds=image_dds::dds_from_image( - &image, - format, - image_dds::Quality::Slow, - image_dds::Mipmaps::GeneratedAutomatic, - )?; + #[derive(Debug,thiserror::Error)] + enum CliConvertTextureError{ + #[error("IO error {0:?}")] + Io(#[from]std::io::Error), + #[error("Convert texture error {0:?}")] + Convert(#[from]ConvertTextureError), + } - let file_name=format!("textures/{}.dds",asset_id.0); - let mut file=std::fs::File::create(file_name)?; - dds.write(&mut file)?; - Ok(()) -} -async fn download_assets(paths:Vec,cookie:rbx_asset::cookie::Cookie)->AResult<()>{ - tokio::try_join!( - tokio::fs::create_dir_all("downloaded_textures"), - tokio::fs::create_dir_all("textures"), - tokio::fs::create_dir_all("meshes"), - tokio::fs::create_dir_all("unions"), - )?; - // use mpsc - let thread_limit=std::thread::available_parallelism()?.get(); - let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT); - let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit); - // map decode dispatcher - // read files multithreaded - // produce UniqueAssetsResult per file - tokio::spawn(async move{ - // move send so it gets dropped when all maps have been decoded - // closing the channel + async fn cli_download_assets(paths:Vec,cookie:rbx_asset::cookie::Cookie)->AResult<()>{ + tokio::try_join!( + tokio::fs::create_dir_all("downloaded_textures"), + tokio::fs::create_dir_all("textures"), + tokio::fs::create_dir_all("meshes"), + tokio::fs::create_dir_all("unions"), + )?; + let thread_limit=std::thread::available_parallelism()?.get(); + let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT); + let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit); + tokio::spawn(async move{ + let mut it=paths.into_iter(); + static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0); + SEM.add_permits(thread_limit); + while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){ + let send=send_assets.clone(); + tokio::spawn(async move{ + let data=read_entire_file(path.as_path()).await; + let result=data.map_err(LoadDomError::from).and_then(|d|{ + let dom=load_dom(&d)?; + Ok(get_unique_assets(dom)) + }); + _=send.send(result).await; + drop(permit); + }); + } + }); + let mut stats=Stats::default(); + let context=rbx_asset::cookie::CookieContext::new(cookie); + let mut globally_unique_assets=UniqueAssets::default(); + let download_thread=tokio::spawn(async move{ + while let Some(result)=recv_assets.recv().await{ + let unique_assets=match result{ + Ok(unique_assets)=>unique_assets, + Err(e)=>{ + println!("error: {e:?}"); + continue; + }, + }; + for texture_id in unique_assets.textures{ + if globally_unique_assets.textures.insert(texture_id){ + let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?; + send_texture.send((texture_id,data)).await?; + } + } + for mesh_id in unique_assets.meshes{ + if globally_unique_assets.meshes.insert(mesh_id){ + download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?; + } + } + for union_id in unique_assets.unions{ + if globally_unique_assets.unions.insert(union_id){ + download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?; + } + } + } + dbg!(stats); + Ok::<(),anyhow::Error>(()) + }); + static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0); + SEM.add_permits(thread_limit); + while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){ + tokio::spawn(async move{ + let result=cli_convert_texture(asset_id,download_result).await; + drop(permit); + result.unwrap(); + }); + } + download_thread.await??; + _=SEM.acquire_many(thread_limit as u32).await.unwrap(); + Ok(()) + } + + async fn cli_roblox_to_snf(paths:Vec,output_folder:PathBuf)->AResult<()>{ + let start=std::time::Instant::now(); + + let thread_limit=std::thread::available_parallelism()?.get(); let mut it=paths.into_iter(); static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0); SEM.add_permits(thread_limit); + while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){ - let send=send_assets.clone(); + let output_folder=output_folder.clone(); tokio::spawn(async move{ - let result=unique_assets(path.as_path()).await; - _=send.send(result).await; + let result=cli_convert_to_snf(path.as_path(),output_folder).await; drop(permit); + match result{ + Ok(())=>(), + Err(e)=>println!("Convert error: {e:?}"), + } }); } - }); - // download manager - // insert into global unique assets guy - // add to download queue if the asset is globally unique and does not already exist on disk - let mut stats=Stats::default(); - let context=rbx_asset::cookie::CookieContext::new(cookie); - let mut globally_unique_assets=UniqueAssets::default(); - // pop a job = retry_queue.pop_front() or ingest(recv.recv().await) - // SLOW MODE: - // acquire all permits - // drop all permits - // pop one job - // if it succeeds go into fast mode - // FAST MODE: - // acquire one permit - // pop a job - let download_thread=tokio::spawn(async move{ - while let Some(result)=recv_assets.recv().await{ - let unique_assets=match result{ - Ok(unique_assets)=>unique_assets, - Err(e)=>{ - println!("error: {e:?}"); - continue; - }, - }; - for texture_id in unique_assets.textures{ - if globally_unique_assets.textures.insert(texture_id){ - let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?; - send_texture.send((texture_id,data)).await?; - } - } - for mesh_id in unique_assets.meshes{ - if globally_unique_assets.meshes.insert(mesh_id){ - download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?; - } - } - for union_id in unique_assets.unions{ - if globally_unique_assets.unions.insert(union_id){ - download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?; - } - } - } - dbg!(stats); - Ok::<(),anyhow::Error>(()) - }); - static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0); - SEM.add_permits(thread_limit); - while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){ - tokio::spawn(async move{ - let result=convert_texture(asset_id,download_result).await; - drop(permit); - result.unwrap(); - }); - } - download_thread.await??; - _=SEM.acquire_many(thread_limit as u32).await.unwrap(); - Ok(()) -} + _=SEM.acquire_many(thread_limit as u32).await.unwrap(); -#[derive(Debug)] -#[allow(dead_code)] -enum ConvertError{ - IO(std::io::Error), - SNFMap(strafesnet_snf::map::Error), - RobloxRead(strafesnet_rbx_loader::ReadError), - RobloxLoad(strafesnet_rbx_loader::LoadError), -} -impl std::fmt::Display for ConvertError{ - fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{ - write!(f,"{self:?}") + println!("elapsed={:?}", start.elapsed()); + Ok(()) + } + + async fn cli_convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{ + let entire_file=tokio::fs::read(path).await?; + let snf_data=convert_to_snf(&entire_file)?; + + let mut dest=output_folder; + dest.push(path.file_stem().unwrap()); + dest.set_extension("snfm"); + tokio::fs::write(dest,snf_data).await?; + + Ok(()) } } -impl std::error::Error for ConvertError{} -async fn convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{ - let entire_file=tokio::fs::read(path).await?; - - let model=strafesnet_rbx_loader::read( - std::io::Cursor::new(entire_file) - ).map_err(ConvertError::RobloxRead)?; - - let mut place=model.into_place(); - place.run_scripts(); - - let map=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?; - - let mut dest=output_folder; - dest.push(path.file_stem().unwrap()); - dest.set_extension("snfm"); - let file=std::fs::File::create(dest).map_err(ConvertError::IO)?; - - strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?; - - Ok(()) -} - -async fn roblox_to_snf(paths:Vec,output_folder:PathBuf)->AResult<()>{ - let start=std::time::Instant::now(); - - let thread_limit=std::thread::available_parallelism()?.get(); - let mut it=paths.into_iter(); - static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0); - SEM.add_permits(thread_limit); - - while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){ - let output_folder=output_folder.clone(); - tokio::spawn(async move{ - let result=convert_to_snf(path.as_path(),output_folder).await; - drop(permit); - match result{ - Ok(())=>(), - Err(e)=>println!("Convert error: {e:?}"), - } - }); - } - _=SEM.acquire_many(thread_limit as u32).await.unwrap(); - - println!("elapsed={:?}", start.elapsed()); - Ok(()) -} +#[cfg(feature="cli")] +pub use cli::Commands; diff --git a/src/source.rs b/src/source.rs index 5fcecea..a293d3e 100644 --- a/src/source.rs +++ b/src/source.rs @@ -1,428 +1,442 @@ -use std::path::{Path,PathBuf}; -use std::borrow::Cow; -use clap::{Args,Subcommand}; -use anyhow::Result as AResult; -use futures::StreamExt; -use strafesnet_bsp_loader::loader::BspFinder; -use strafesnet_deferred_loader::loader::Loader; -use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLoader,RenderConfigDeferredLoader}; +use std::io::Cursor; +use strafesnet_deferred_loader::deferred_loader::LoadFailureMode; -#[derive(Subcommand)] -pub enum Commands{ - SourceToSNF(SourceToSNFSubcommand), - ExtractTextures(ExtractTexturesSubcommand), - VPKContents(VPKContentsSubcommand), - BSPContents(BSPContentsSubcommand), -} - -#[derive(Args)] -pub struct SourceToSNFSubcommand { - #[arg(long)] - output_folder:PathBuf, - #[arg(required=true)] - input_files:Vec, - #[arg(long)] - vpks:Vec, -} -#[derive(Args)] -pub struct ExtractTexturesSubcommand{ - #[arg(required=true)] - bsp_files:Vec, - #[arg(long)] - vpks:Vec, -} -#[derive(Args)] -pub struct VPKContentsSubcommand { - #[arg(long)] - input_file:PathBuf, -} -#[derive(Args)] -pub struct BSPContentsSubcommand { - #[arg(long)] - input_file:PathBuf, -} - -impl Commands{ - pub async fn run(self)->AResult<()>{ - match self{ - Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder,subcommand.vpks).await, - Commands::ExtractTextures(subcommand)=>extract_textures(subcommand.bsp_files,subcommand.vpks).await, - Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file), - Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file), - } - } -} - - -enum VMTContent{ - VMT(String), - VTF(String), - Patch(vmt_parser::material::PatchMaterial), - Unsupported,//don't want to deal with whatever vmt variant - Unresolved,//could not locate a texture because of vmt content -} -impl VMTContent{ - fn vtf(opt:Option)->Self{ - match opt{ - Some(s)=>Self::VTF(s), - None=>Self::Unresolved, - } - } -} - -fn get_some_texture(material:vmt_parser::material::Material)->VMTContent{ - //just grab some texture from somewhere for now - match material{ - vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none - vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)), - vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture), - vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture), - vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture), - vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)), - vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),//recursive - vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture), - vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::Refract(mat)=>VMTContent::vtf(mat.base_texture), - vmt_parser::material::Material::Modulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::DecalModulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported, - vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat), - _=>unreachable!(), - } -} +// === Public library API === #[derive(Debug,thiserror::Error)] -enum GetVMTError{ - #[error("Bsp error {0:?}")] - Bsp(#[from]vbsp::BspError), - #[error("Utf8 error {0:?}")] - Utf8(#[from]std::str::Utf8Error), - #[error("Vdf error {0:?}")] - Vdf(#[from]vmt_parser::VdfError), - #[error("Vmt not found")] - NotFound, -} - -fn get_vmt(finder:BspFinder,search_name:&str)->Result{ - let vmt_data=finder.find(search_name)?.ok_or(GetVMTError::NotFound)?; - //decode vmt and then write - let vmt_str=core::str::from_utf8(&vmt_data)?; - let material=vmt_parser::from_str(vmt_str)?; - //println!("vmt material={:?}",material); - Ok(material) -} - -#[derive(Debug,thiserror::Error)] -enum LoadVMTError{ - #[error("Bsp error {0:?}")] - Bsp(#[from]vbsp::BspError), - #[error("GetVMT error {0:?}")] - GetVMT(#[from]GetVMTError), - #[error("FromUtf8 error {0:?}")] - FromUtf8(#[from]std::string::FromUtf8Error), - #[error("Vdf error {0:?}")] - Vdf(#[from]vmt_parser::VdfError), - #[error("Vmt unsupported")] - Unsupported, - #[error("Vmt unresolved")] - Unresolved, - #[error("Vmt not found")] - NotFound, -} -fn recursive_vmt_loader<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,material:vmt_parser::material::Material)->Result>,LoadVMTError> - where - 'bsp:'a, - 'vpk:'a, -{ - match get_some_texture(material){ - VMTContent::VMT(s)=>recursive_vmt_loader(finder,get_vmt(finder,s.as_str())?), - VMTContent::VTF(s)=>{ - let mut texture_file_name=PathBuf::from("materials"); - texture_file_name.push(s); - texture_file_name.set_extension("vtf"); - Ok(finder.find(texture_file_name.to_str().unwrap())?) - }, - VMTContent::Patch(mat)=>recursive_vmt_loader(finder, - mat.resolve(|search_name| - match finder.find(search_name)?{ - Some(bytes)=>Ok(String::from_utf8(bytes.into_owned())?), - None=>Err(LoadVMTError::NotFound), - } - )? - ), - VMTContent::Unsupported=>Err(LoadVMTError::Unsupported), - VMTContent::Unresolved=>Err(LoadVMTError::Unresolved), - } -} -fn load_texture<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,texture_name:&str)->Result>,LoadVMTError> - where - 'bsp:'a, - 'vpk:'a, -{ - let mut texture_file_name=PathBuf::from("materials"); - //lower case - let texture_file_name_lowercase=texture_name.to_lowercase(); - texture_file_name.push(texture_file_name_lowercase.clone()); - //remove stem and search for both vtf and vmt files - let stem=PathBuf::from(texture_file_name.file_stem().unwrap()); - texture_file_name.pop(); - texture_file_name.push(stem); - if let Some(stuff)=finder.find(texture_file_name.to_str().unwrap())?{ - return Ok(Some(stuff)) - } - //somehow search for both files - let mut texture_file_name_vmt=texture_file_name.clone(); - texture_file_name.set_extension("vtf"); - texture_file_name_vmt.set_extension("vmt"); - recursive_vmt_loader(finder,get_vmt(finder,texture_file_name_vmt.to_str().unwrap())?) -} -#[derive(Debug,thiserror::Error)] -enum ExtractTextureError{ - #[error("Io error {0:?}")] - Io(#[from]std::io::Error), - #[error("Bsp error {0:?}")] - Bsp(#[from]vbsp::BspError), - #[error("MeshLoad error {0:?}")] - MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError), - #[error("Load VMT error {0:?}")] - LoadVMT(#[from]LoadVMTError), -} -async fn gimme_them_textures(path:&Path,vpk_list:&[vpk::VPK],send_texture:tokio::sync::mpsc::Sender<(Vec,String)>)->Result<(),ExtractTextureError>{ - let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?; - let loader_bsp=strafesnet_bsp_loader::Bsp::new(bsp); - let bsp=loader_bsp.as_ref(); - - let mut texture_deferred_loader=RenderConfigDeferredLoader::new(); - for texture in bsp.textures(){ - texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name()))); - } - - let mut mesh_deferred_loader=MeshDeferredLoader::new(); - for prop in bsp.static_props(){ - mesh_deferred_loader.acquire_mesh_id(prop.model()); - } - - let finder=BspFinder{ - bsp:&loader_bsp, - vpks:vpk_list - }; - - let mut mesh_loader=strafesnet_bsp_loader::loader::ModelLoader::new(finder); - // load models and collect requested textures - for model_path in mesh_deferred_loader.into_indices(){ - let model:vmdl::Model=match mesh_loader.load(model_path){ - Ok(model)=>model, - Err(e)=>{ - println!("Model={model_path} Load model error: {e}"); - continue; - }, - }; - for texture in model.textures(){ - for search_path in &texture.search_paths{ - let mut path=PathBuf::from(search_path.as_str()); - path.push(texture.name.as_str()); - let path=path.to_str().unwrap().to_owned(); - texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path))); - } - } - } - - for texture_path in texture_deferred_loader.into_indices(){ - match load_texture(finder,&texture_path){ - Ok(Some(texture))=>send_texture.send( - (texture.into_owned(),texture_path.into_owned()) - ).await.unwrap(), - Ok(None)=>(), - Err(e)=>println!("Texture={texture_path} Load error: {e}"), - } - } - - Ok(()) -} - - -#[derive(Debug,thiserror::Error)] -enum ConvertTextureError{ - #[error("Bsp error {0:?}")] - Bsp(#[from]vbsp::BspError), +pub enum ConvertTextureError{ #[error("Vtf error {0:?}")] Vtf(#[from]vtf::Error), #[error("DDS create error {0:?}")] DDS(#[from]image_dds::CreateDdsError), #[error("DDS write error {0:?}")] DDSWrite(#[from]image_dds::ddsfile::Error), - #[error("Io error {0:?}")] - Io(#[from]std::io::Error), } -async fn convert_texture(texture:Vec,write_file_name:impl AsRef)->Result<(),ConvertTextureError>{ - let image=vtf::from_bytes(&texture)?.highres_image.decode(0)?.to_rgba8(); +/// Convert VTF texture bytes to DDS texture bytes. +pub fn convert_texture_to_dds(vtf_data:&[u8])->Result,ConvertTextureError>{ + let vtf_data=vtf_data.to_vec(); + let image=vtf::from_bytes(&vtf_data)?.highres_image.decode(0)?.to_rgba8(); let format=if image.width()%4!=0||image.height()%4!=0{ image_dds::ImageFormat::Rgba8UnormSrgb }else{ image_dds::ImageFormat::BC7RgbaUnormSrgb }; - //this fails if the image dimensions are not a multiple of 4 - let dds = image_dds::dds_from_image( + + let dds=image_dds::dds_from_image( &image, format, image_dds::Quality::Slow, image_dds::Mipmaps::GeneratedAutomatic, )?; - //write dds - let mut dest=PathBuf::from("textures"); - dest.push(write_file_name); - dest.set_extension("dds"); - std::fs::create_dir_all(dest.parent().unwrap())?; - let mut writer=std::io::BufWriter::new(std::fs::File::create(dest)?); - dds.write(&mut writer)?; - - Ok(()) + let mut buf=Vec::new(); + dds.write(&mut Cursor::new(&mut buf))?; + Ok(buf) } -async fn read_vpks(vpk_paths:Vec,thread_limit:usize)->Vec{ +#[derive(Debug,thiserror::Error)] +pub enum ConvertError{ + #[error("BSP read error {0:?}")] + BspRead(strafesnet_bsp_loader::ReadError), + #[error("BSP load error {0:?}")] + BspLoad(strafesnet_bsp_loader::LoadError), + #[error("SNF map error {0:?}")] + SNFMap(strafesnet_snf::map::Error), + #[error("BSP parse error {0:?}")] + BspParse(#[from]vbsp::BspError), +} + +/// Convert a Source BSP file (bytes) to SNF map format (bytes). +pub fn convert_to_snf(bsp_data:&[u8],vpk_list:&[vpk::VPK])->Result,ConvertError>{ + let bsp=strafesnet_bsp_loader::read( + Cursor::new(bsp_data) + ).map_err(ConvertError::BspRead)?; + + let map=bsp.to_snf(LoadFailureMode::DefaultToNone,vpk_list).map_err(ConvertError::BspLoad)?; + + let mut buf=Vec::new(); + strafesnet_snf::map::write_map(Cursor::new(&mut buf),map).map_err(ConvertError::SNFMap)?; + Ok(buf) +} + +/// Read VPK archives from paths. Useful for loading VPKs needed by `convert_to_snf`. +pub async fn read_vpks(vpk_paths:Vec,thread_limit:usize)->Vec{ + use futures::StreamExt; futures::stream::iter(vpk_paths).map(|vpk_path|async{ - // idk why it doesn't want to pass out the errors but this is fatal anyways tokio::task::spawn_blocking(move||vpk::VPK::read(&vpk_path)).await.unwrap().unwrap() }) .buffer_unordered(thread_limit) .collect().await } -async fn extract_textures(paths:Vec,vpk_paths:Vec)->AResult<()>{ - tokio::try_join!( - tokio::fs::create_dir_all("extracted_textures"), - tokio::fs::create_dir_all("textures"), - tokio::fs::create_dir_all("meshes"), - )?; - let thread_limit=std::thread::available_parallelism()?.get(); +// === CLI === - // load vpk list - let vpk_list=read_vpks(vpk_paths,thread_limit).await; +#[cfg(feature="cli")] +mod cli{ + use super::*; + use std::path::{Path,PathBuf}; + use std::borrow::Cow; + use clap::{Args,Subcommand}; + use anyhow::Result as AResult; + use strafesnet_bsp_loader::loader::BspFinder; + use strafesnet_deferred_loader::loader::Loader; + use strafesnet_deferred_loader::deferred_loader::{MeshDeferredLoader,RenderConfigDeferredLoader}; - // leak vpk_list for static lifetime? - let vpk_list:&[vpk::VPK]=vpk_list.leak(); + enum VMTContent{ + VMT(String), + VTF(String), + Patch(vmt_parser::material::PatchMaterial), + Unsupported, + Unresolved, + } + impl VMTContent{ + fn vtf(opt:Option)->Self{ + match opt{ + Some(s)=>Self::VTF(s), + None=>Self::Unresolved, + } + } + } + + fn get_some_texture(material:vmt_parser::material::Material)->VMTContent{ + match material{ + vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)), + vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)), + vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture), + vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture), + vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture), + vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)), + vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material), + vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture), + vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::Refract(mat)=>VMTContent::vtf(mat.base_texture), + vmt_parser::material::Material::Modulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::DecalModulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported, + vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat), + _=>unreachable!(), + } + } + + #[derive(Debug,thiserror::Error)] + enum GetVMTError{ + #[error("Bsp error {0:?}")] + Bsp(#[from]vbsp::BspError), + #[error("Utf8 error {0:?}")] + Utf8(#[from]std::str::Utf8Error), + #[error("Vdf error {0:?}")] + Vdf(#[from]vmt_parser::VdfError), + #[error("Vmt not found")] + NotFound, + } + + fn get_vmt(finder:BspFinder,search_name:&str)->Result{ + let vmt_data=finder.find(search_name)?.ok_or(GetVMTError::NotFound)?; + let vmt_str=core::str::from_utf8(&vmt_data)?; + let material=vmt_parser::from_str(vmt_str)?; + Ok(material) + } + + #[derive(Debug,thiserror::Error)] + enum LoadVMTError{ + #[error("Bsp error {0:?}")] + Bsp(#[from]vbsp::BspError), + #[error("GetVMT error {0:?}")] + GetVMT(#[from]GetVMTError), + #[error("FromUtf8 error {0:?}")] + FromUtf8(#[from]std::string::FromUtf8Error), + #[error("Vdf error {0:?}")] + Vdf(#[from]vmt_parser::VdfError), + #[error("Vmt unsupported")] + Unsupported, + #[error("Vmt unresolved")] + Unresolved, + #[error("Vmt not found")] + NotFound, + } + fn recursive_vmt_loader<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,material:vmt_parser::material::Material)->Result>,LoadVMTError> + where + 'bsp:'a, + 'vpk:'a, + { + match get_some_texture(material){ + VMTContent::VMT(s)=>recursive_vmt_loader(finder,get_vmt(finder,s.as_str())?), + VMTContent::VTF(s)=>{ + let mut texture_file_name=PathBuf::from("materials"); + texture_file_name.push(s); + texture_file_name.set_extension("vtf"); + Ok(finder.find(texture_file_name.to_str().unwrap())?) + }, + VMTContent::Patch(mat)=>recursive_vmt_loader(finder, + mat.resolve(|search_name| + match finder.find(search_name)?{ + Some(bytes)=>Ok(String::from_utf8(bytes.into_owned())?), + None=>Err(LoadVMTError::NotFound), + } + )? + ), + VMTContent::Unsupported=>Err(LoadVMTError::Unsupported), + VMTContent::Unresolved=>Err(LoadVMTError::Unresolved), + } + } + fn load_texture<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,texture_name:&str)->Result>,LoadVMTError> + where + 'bsp:'a, + 'vpk:'a, + { + let mut texture_file_name=PathBuf::from("materials"); + let texture_file_name_lowercase=texture_name.to_lowercase(); + texture_file_name.push(texture_file_name_lowercase.clone()); + let stem=PathBuf::from(texture_file_name.file_stem().unwrap()); + texture_file_name.pop(); + texture_file_name.push(stem); + if let Some(stuff)=finder.find(texture_file_name.to_str().unwrap())?{ + return Ok(Some(stuff)) + } + let mut texture_file_name_vmt=texture_file_name.clone(); + texture_file_name.set_extension("vtf"); + texture_file_name_vmt.set_extension("vmt"); + recursive_vmt_loader(finder,get_vmt(finder,texture_file_name_vmt.to_str().unwrap())?) + } + + #[derive(Subcommand)] + pub enum Commands{ + SourceToSNF(SourceToSNFSubcommand), + ExtractTextures(ExtractTexturesSubcommand), + VPKContents(VPKContentsSubcommand), + BSPContents(BSPContentsSubcommand), + } + + #[derive(Args)] + pub struct SourceToSNFSubcommand { + #[arg(long)] + output_folder:PathBuf, + #[arg(required=true)] + input_files:Vec, + #[arg(long)] + vpks:Vec, + } + #[derive(Args)] + pub struct ExtractTexturesSubcommand{ + #[arg(required=true)] + bsp_files:Vec, + #[arg(long)] + vpks:Vec, + } + #[derive(Args)] + pub struct VPKContentsSubcommand { + #[arg(long)] + input_file:PathBuf, + } + #[derive(Args)] + pub struct BSPContentsSubcommand { + #[arg(long)] + input_file:PathBuf, + } + + impl Commands{ + pub async fn run(self)->AResult<()>{ + match self{ + Commands::SourceToSNF(subcommand)=>cli_source_to_snf(subcommand.input_files,subcommand.output_folder,subcommand.vpks).await, + Commands::ExtractTextures(subcommand)=>cli_extract_textures(subcommand.bsp_files,subcommand.vpks).await, + Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file), + Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file), + } + } + } + + #[derive(Debug,thiserror::Error)] + enum ExtractTextureError{ + #[error("Io error {0:?}")] + Io(#[from]std::io::Error), + #[error("Bsp error {0:?}")] + Bsp(#[from]vbsp::BspError), + #[error("MeshLoad error {0:?}")] + MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError), + #[error("Load VMT error {0:?}")] + LoadVMT(#[from]LoadVMTError), + } + async fn gimme_them_textures(path:&Path,vpk_list:&[vpk::VPK],send_texture:tokio::sync::mpsc::Sender<(Vec,String)>)->Result<(),ExtractTextureError>{ + let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?; + let loader_bsp=strafesnet_bsp_loader::Bsp::new(bsp); + let bsp=loader_bsp.as_ref(); + + let mut texture_deferred_loader=RenderConfigDeferredLoader::new(); + for texture in bsp.textures(){ + texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name()))); + } + + let mut mesh_deferred_loader=MeshDeferredLoader::new(); + for prop in bsp.static_props(){ + mesh_deferred_loader.acquire_mesh_id(prop.model()); + } + + let finder=BspFinder{ + bsp:&loader_bsp, + vpks:vpk_list + }; + + let mut mesh_loader=strafesnet_bsp_loader::loader::ModelLoader::new(finder); + for model_path in mesh_deferred_loader.into_indices(){ + let model:vmdl::Model=match mesh_loader.load(model_path){ + Ok(model)=>model, + Err(e)=>{ + println!("Model={model_path} Load model error: {e}"); + continue; + }, + }; + for texture in model.textures(){ + for search_path in &texture.search_paths{ + let mut path=PathBuf::from(search_path.as_str()); + path.push(texture.name.as_str()); + let path=path.to_str().unwrap().to_owned(); + texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path))); + } + } + } + + for texture_path in texture_deferred_loader.into_indices(){ + match load_texture(finder,&texture_path){ + Ok(Some(texture))=>send_texture.send( + (texture.into_owned(),texture_path.into_owned()) + ).await.unwrap(), + Ok(None)=>(), + Err(e)=>println!("Texture={texture_path} Load error: {e}"), + } + } + + Ok(()) + } + + #[derive(Debug,thiserror::Error)] + enum CliConvertTextureError{ + #[error("IO error {0:?}")] + Io(#[from]std::io::Error), + #[error("Convert texture error {0:?}")] + Convert(#[from]ConvertTextureError), + } + + async fn cli_convert_texture(texture:Vec,write_file_name:impl AsRef)->Result<(),CliConvertTextureError>{ + let dds_data=convert_texture_to_dds(&texture)?; + + let mut dest=PathBuf::from("textures"); + dest.push(write_file_name); + dest.set_extension("dds"); + std::fs::create_dir_all(dest.parent().unwrap())?; + let mut writer=std::io::BufWriter::new(std::fs::File::create(dest)?); + std::io::Write::write_all(&mut writer,&dds_data)?; + + Ok(()) + } + + async fn cli_extract_textures(paths:Vec,vpk_paths:Vec)->AResult<()>{ + tokio::try_join!( + tokio::fs::create_dir_all("extracted_textures"), + tokio::fs::create_dir_all("textures"), + tokio::fs::create_dir_all("meshes"), + )?; + let thread_limit=std::thread::available_parallelism()?.get(); + + let vpk_list=read_vpks(vpk_paths,thread_limit).await; + let vpk_list:&[vpk::VPK]=vpk_list.leak(); + + let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit); + let mut it=paths.into_iter(); + let extract_thread=tokio::spawn(async move{ + static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0); + SEM.add_permits(thread_limit); + while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){ + let send=send_texture.clone(); + tokio::spawn(async move{ + let result=gimme_them_textures(&path,vpk_list,send).await; + drop(permit); + match result{ + Ok(())=>(), + Err(e)=>println!("Map={path:?} Decode error: {e:?}"), + } + }); + } + }); - let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit); - let mut it=paths.into_iter(); - let extract_thread=tokio::spawn(async move{ static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0); SEM.add_permits(thread_limit); - while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){ - let send=send_texture.clone(); + while let (Ok(permit),Some((data,dest)))=(SEM.acquire().await,recv_texture.recv().await){ tokio::spawn(async move{ - let result=gimme_them_textures(&path,vpk_list,send).await; + let result=cli_convert_texture(data,dest).await; drop(permit); match result{ Ok(())=>(), - Err(e)=>println!("Map={path:?} Decode error: {e:?}"), + Err(e)=>println!("Convert error: {e:?}"), } }); } - }); - - // convert images - static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0); - SEM.add_permits(thread_limit); - while let (Ok(permit),Some((data,dest)))=(SEM.acquire().await,recv_texture.recv().await){ - // TODO: dedup dest? - tokio::spawn(async move{ - let result=convert_texture(data,dest).await; - drop(permit); - match result{ - Ok(())=>(), - Err(e)=>println!("Convert error: {e:?}"), - } - }); + extract_thread.await?; + _=SEM.acquire_many(thread_limit as u32).await?; + Ok(()) } - extract_thread.await?; - _=SEM.acquire_many(thread_limit as u32).await?; - Ok(()) -} -fn vpk_contents(vpk_path:PathBuf)->AResult<()>{ - let vpk_index=vpk::VPK::read(&vpk_path)?; - for (label,entry) in vpk_index.tree.into_iter(){ - println!("vpk label={} entry={:?}",label,entry); + fn vpk_contents(vpk_path:PathBuf)->AResult<()>{ + let vpk_index=vpk::VPK::read(&vpk_path)?; + for (label,entry) in vpk_index.tree.into_iter(){ + println!("vpk label={} entry={:?}",label,entry); + } + Ok(()) } - Ok(()) -} -fn bsp_contents(path:PathBuf)->AResult<()>{ - let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?; - for file_name in bsp.pack.into_zip().into_inner().unwrap().file_names(){ - println!("file_name={:?}",file_name); + fn bsp_contents(path:PathBuf)->AResult<()>{ + let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?; + for file_name in bsp.pack.into_zip().into_inner().unwrap().file_names(){ + println!("file_name={:?}",file_name); + } + Ok(()) } - Ok(()) -} -#[derive(Debug)] -#[allow(dead_code)] -enum ConvertError{ - IO(std::io::Error), - SNFMap(strafesnet_snf::map::Error), - BspRead(strafesnet_bsp_loader::ReadError), - BspLoad(strafesnet_bsp_loader::LoadError), -} -impl std::fmt::Display for ConvertError{ - fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{ - write!(f,"{self:?}") + async fn cli_convert_to_snf(path:&Path,vpk_list:&[vpk::VPK],output_folder:PathBuf)->AResult<()>{ + let entire_file=tokio::fs::read(path).await?; + let snf_data=convert_to_snf(&entire_file,vpk_list)?; + + let mut dest=output_folder; + dest.push(path.file_stem().unwrap()); + dest.set_extension("snfm"); + tokio::fs::write(dest,snf_data).await?; + + Ok(()) + } + + async fn cli_source_to_snf(paths:Vec,output_folder:PathBuf,vpk_paths:Vec)->AResult<()>{ + let start=std::time::Instant::now(); + + let thread_limit=std::thread::available_parallelism()?.get(); + + let vpk_list=read_vpks(vpk_paths,thread_limit).await; + let vpk_list:&[vpk::VPK]=vpk_list.leak(); + + let mut it=paths.into_iter(); + static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0); + SEM.add_permits(thread_limit); + + while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){ + let output_folder=output_folder.clone(); + tokio::spawn(async move{ + let result=cli_convert_to_snf(path.as_path(),vpk_list,output_folder).await; + drop(permit); + match result{ + Ok(())=>(), + Err(e)=>println!("Convert error: {e:?}"), + } + }); + } + _=SEM.acquire_many(thread_limit as u32).await.unwrap(); + + println!("elapsed={:?}", start.elapsed()); + Ok(()) } } -impl std::error::Error for ConvertError{} - -async fn convert_to_snf(path:&Path,vpk_list:&[vpk::VPK],output_folder:PathBuf)->AResult<()>{ - let entire_file=tokio::fs::read(path).await?; - - let bsp=strafesnet_bsp_loader::read( - std::io::Cursor::new(entire_file) - ).map_err(ConvertError::BspRead)?; - - let map=bsp.to_snf(LoadFailureMode::DefaultToNone,vpk_list).map_err(ConvertError::BspLoad)?; - - let mut dest=output_folder; - dest.push(path.file_stem().unwrap()); - dest.set_extension("snfm"); - let file=std::fs::File::create(dest).map_err(ConvertError::IO)?; - - strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?; - - Ok(()) -} -async fn source_to_snf(paths:Vec,output_folder:PathBuf,vpk_paths:Vec)->AResult<()>{ - let start=std::time::Instant::now(); - - let thread_limit=std::thread::available_parallelism()?.get(); - - // load vpk list - let vpk_list=read_vpks(vpk_paths,thread_limit).await; - - // leak vpk_list for static lifetime? - let vpk_list:&[vpk::VPK]=vpk_list.leak(); - - let mut it=paths.into_iter(); - static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0); - SEM.add_permits(thread_limit); - - while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){ - let output_folder=output_folder.clone(); - tokio::spawn(async move{ - let result=convert_to_snf(path.as_path(),vpk_list,output_folder).await; - drop(permit); - match result{ - Ok(())=>(), - Err(e)=>println!("Convert error: {e:?}"), - } - }); - } - _=SEM.acquire_many(thread_limit as u32).await.unwrap(); - - println!("elapsed={:?}", start.elapsed()); - Ok(()) -} +#[cfg(feature="cli")] +pub use cli::Commands;