split code into modules

src/inference.rs

@@ -0,0 +1,219 @@
+#[derive(clap::Subcommand)]
+pub enum Commands {
+	Simulate(SimulateSubcommand),
+}
+impl Commands {
+	pub fn run(self) {
+		match self {
+			Commands::Simulate(subcommand) => subcommand.run(),
+		}
+	}
+}
+
+#[derive(clap::Args)]
+pub struct SimulateSubcommand {}
+
+impl SimulateSubcommand {
+	fn run(self) {
+		inference();
+	}
+}
+
+use burn::prelude::*;
+
+use crate::inputs::GraphicsState;
+use crate::net::{INPUT, InferenceBackend, Net};
+
+use strafesnet_common::instruction::TimedInstruction;
+use strafesnet_common::mouse::MouseState;
+use strafesnet_common::physics::{
+	Instruction as PhysicsInputInstruction, MiscInstruction, ModeInstruction, MouseInstruction,
+	SetControlInstruction, Time as PhysicsTime,
+};
+use strafesnet_physics::physics::{PhysicsContext, PhysicsData, PhysicsState};
+
+pub struct Recording {
+	instructions: Vec<TimedInstruction<PhysicsInputInstruction, PhysicsTime>>,
+}
+struct FrameState {
+	trajectory: strafesnet_physics::physics::Trajectory,
+	camera: strafesnet_physics::physics::PhysicsCamera,
+}
+impl FrameState {
+	fn pos(&self, time: PhysicsTime) -> glam::Vec3 {
+		self.trajectory
+			.extrapolated_position(time)
+			.map(Into::<f32>::into)
+			.to_array()
+			.into()
+	}
+	fn angles(&self) -> glam::Vec2 {
+		self.camera.simulate_move_angles(glam::IVec2::ZERO)
+	}
+}
+struct Session {
+	geometry_shared: PhysicsData,
+	simulation: PhysicsState,
+	recording: Recording,
+}
+impl Session {
+	fn get_frame_state(&self) -> FrameState {
+		FrameState {
+			trajectory: self.simulation.camera_trajectory(&self.geometry_shared),
+			camera: self.simulation.camera(),
+		}
+	}
+	fn run(&mut self, time: PhysicsTime, instruction: PhysicsInputInstruction) {
+		let instruction = TimedInstruction { time, instruction };
+		self.recording.instructions.push(instruction.clone());
+		PhysicsContext::run_input_instruction(
+			&mut self.simulation,
+			&self.geometry_shared,
+			instruction,
+		);
+	}
+}
+
+fn inference() {
+	let mut args = std::env::args().skip(1);
+
+	// pick device
+	let gpu_id: usize = args
+		.next()
+		.map(|id| id.parse().unwrap())
+		.unwrap_or_default();
+	let device = burn::backend::cuda::CudaDevice::new(gpu_id);
+
+	// load model
+	let path: std::path::PathBuf = args.next().unwrap().parse().unwrap();
+	let mut model: Net<InferenceBackend> = Net::init(&device);
+	model = model
+		.load_file(
+			path,
+			&burn::record::BinFileRecorder::<burn::record::FullPrecisionSettings>::new(),
+			&device,
+		)
+		.unwrap();
+
+	// load map
+	let map_file = include_bytes!("../files/bhop_marble_5692093612.snfm");
+	let map = strafesnet_snf::read_map(std::io::Cursor::new(map_file))
+		.unwrap()
+		.into_complete_map()
+		.unwrap();
+	let modes = map.modes.clone().denormalize();
+	let mode = modes
+		.get_mode(strafesnet_common::gameplay_modes::ModeId::MAIN)
+		.unwrap();
+	let start_zone = map.models.get(mode.get_start().get() as usize).unwrap();
+	let start_offset = glam::Vec3::from_array(
+		start_zone
+			.transform
+			.translation
+			.map(|f| f.into())
+			.to_array(),
+	);
+
+	// setup graphics
+	let mut g = GraphicsState::new(&map);
+
+	// setup simulation
+	let mut session = Session {
+		geometry_shared: PhysicsData::new(&map),
+		simulation: PhysicsState::default(),
+		recording: Recording {
+			instructions: Vec::new(),
+		},
+	};
+
+	let mut time = PhysicsTime::ZERO;
+
+	// reset to start zone
+	session.run(time, PhysicsInputInstruction::Mode(ModeInstruction::Reset));
+	// session.run(
+	// 	time,
+	// 	PhysicsInputInstruction::Misc(MiscInstruction::SetSensitivity(?)),
+	// );
+	session.run(
+		time,
+		PhysicsInputInstruction::Mode(ModeInstruction::Restart(
+			strafesnet_common::gameplay_modes::ModeId::MAIN,
+		)),
+	);
+
+	// TEMP: turn mouse left
+	let mut mouse_pos = glam::ivec2(-5300, 0);
+
+	const STEP: PhysicsTime = PhysicsTime::from_millis(10);
+	let mut input_floats = Vec::new();
+	// setup agent-simulation feedback loop
+	for _ in 0..20 * 100 {
+		// generate inputs
+		let frame_state = session.get_frame_state();
+		g.generate_inputs(
+			frame_state.pos(time) - start_offset,
+			frame_state.angles(),
+			&mut input_floats,
+		);
+
+		// inference
+		let inputs = Tensor::from_data(
+			TensorData::new(input_floats.clone(), Shape::new([1, INPUT])),
+			&device,
+		);
+		let outputs = model.forward(inputs).into_data().into_vec::<f32>().unwrap();
+
+		let &[
+			move_forward,
+			move_left,
+			move_back,
+			move_right,
+			jump,
+			mouse_dx,
+			mouse_dy,
+		] = outputs.as_slice()
+		else {
+			panic!()
+		};
+
+		macro_rules! set_control {
+			($control:ident,$output:expr) => {
+				session.run(
+					time,
+					PhysicsInputInstruction::SetControl(SetControlInstruction::$control(
+						0.5 < $output,
+					)),
+				);
+			};
+		}
+		set_control!(SetMoveForward, move_forward);
+		set_control!(SetMoveLeft, move_left);
+		set_control!(SetMoveBack, move_back);
+		set_control!(SetMoveRight, move_right);
+		set_control!(SetJump, jump);
+
+		mouse_pos += glam::vec2(mouse_dx, mouse_dy).round().as_ivec2();
+		let next_time = time + STEP;
+		session.run(
+			time,
+			PhysicsInputInstruction::Mouse(MouseInstruction::SetNextMouse(MouseState {
+				pos: mouse_pos,
+				time: next_time,
+			})),
+		);
+
+		time = next_time;
+
+		// clear
+		input_floats.clear();
+	}
+
+	let date_string = format!("{}.snfb", chrono::Utc::now());
+	let file = std::fs::File::create(date_string).unwrap();
+	strafesnet_snf::bot::write_bot(
+		std::io::BufWriter::new(file),
+		strafesnet_physics::VERSION.get(),
+		core::mem::take(&mut session.recording.instructions),
+	)
+	.unwrap();
+}

src/inputs.rs

@@ -0,0 +1,166 @@
+const LIMITS: wgpu::Limits = wgpu::Limits::defaults();
+const FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8UnormSrgb;
+use strafesnet_graphics::setup;
+
+use crate::net::{POSITION_HISTORY, SIZE};
+
+// bytes_per_row needs to be a multiple of 256.
+const STRIDE_SIZE: u32 = (SIZE.x * size_of::<f32>() as u32).next_multiple_of(256);
+
+pub struct GraphicsState {
+	device: wgpu::Device,
+	queue: wgpu::Queue,
+	graphics: strafesnet_roblox_bot_player::graphics::Graphics,
+	graphics_texture_view: wgpu::TextureView,
+	output_staging_buffer: wgpu::Buffer,
+	texture_data: Vec<u8>,
+	position_history: Vec<glam::Vec3>,
+}
+impl GraphicsState {
+	pub fn new(map: &strafesnet_common::map::CompleteMap) -> Self {
+		let desc = wgpu::InstanceDescriptor::new_without_display_handle_from_env();
+		let instance = wgpu::Instance::new(desc);
+		let (device, queue) = pollster::block_on(async {
+			let adapter = instance
+				.request_adapter(&wgpu::RequestAdapterOptions {
+					power_preference: wgpu::PowerPreference::HighPerformance,
+					force_fallback_adapter: false,
+					compatible_surface: None,
+				})
+				.await
+				.unwrap();
+			setup::step4::request_device(&adapter, LIMITS)
+				.await
+				.unwrap()
+		});
+		let mut graphics = strafesnet_roblox_bot_player::graphics::Graphics::new(
+			&device, &queue, SIZE, FORMAT, LIMITS,
+		);
+		graphics.change_map(&device, &queue, map).unwrap();
+		let graphics_texture = device.create_texture(&wgpu::TextureDescriptor {
+			label: Some("RGB texture"),
+			format: FORMAT,
+			size: wgpu::Extent3d {
+				width: SIZE.x,
+				height: SIZE.y,
+				depth_or_array_layers: 1,
+			},
+			mip_level_count: 1,
+			sample_count: 1,
+			dimension: wgpu::TextureDimension::D2,
+			usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING,
+			view_formats: &[],
+		});
+		let graphics_texture_view = graphics_texture.create_view(&wgpu::TextureViewDescriptor {
+			label: Some("RGB texture view"),
+			aspect: wgpu::TextureAspect::All,
+			usage: Some(
+				wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING,
+			),
+			..Default::default()
+		});
+		let texture_data = Vec::<u8>::with_capacity((STRIDE_SIZE * SIZE.y) as usize);
+		let output_staging_buffer = device.create_buffer(&wgpu::BufferDescriptor {
+			label: Some("Output staging buffer"),
+			size: texture_data.capacity() as u64,
+			usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
+			mapped_at_creation: false,
+		});
+		let position_history = Vec::with_capacity(POSITION_HISTORY);
+		Self {
+			device,
+			queue,
+			graphics,
+			graphics_texture_view,
+			output_staging_buffer,
+			texture_data,
+			position_history,
+		}
+	}
+	pub fn generate_inputs(&mut self, pos: glam::Vec3, angles: glam::Vec2, inputs: &mut Vec<f32>) {
+		// write position history to model inputs
+		if !self.position_history.is_empty() {
+			let camera = strafesnet_graphics::graphics::view_inv(pos, angles).inverse();
+			for &pos in self.position_history.iter().rev() {
+				let relative_pos = camera.transform_vector3(pos);
+				inputs.extend_from_slice(&relative_pos.to_array());
+			}
+		}
+		// fill remaining history with zeroes
+		for _ in self.position_history.len()..POSITION_HISTORY {
+			inputs.extend_from_slice(&[0.0, 0.0, 0.0]);
+		}
+
+		// track position history
+		if self.position_history.len() < POSITION_HISTORY {
+			self.position_history.push(pos);
+		} else {
+			self.position_history.rotate_left(1);
+			*self.position_history.last_mut().unwrap() = pos;
+		}
+
+		let mut encoder = self
+			.device
+			.create_command_encoder(&wgpu::CommandEncoderDescriptor {
+				label: Some("wgpu encoder"),
+			});
+
+		// render!
+		self.graphics
+			.encode_commands(&mut encoder, &self.graphics_texture_view, pos, angles);
+
+		// copy the depth texture into ram
+		encoder.copy_texture_to_buffer(
+			wgpu::TexelCopyTextureInfo {
+				texture: self.graphics.depth_texture(),
+				mip_level: 0,
+				origin: wgpu::Origin3d::ZERO,
+				aspect: wgpu::TextureAspect::All,
+			},
+			wgpu::TexelCopyBufferInfo {
+				buffer: &self.output_staging_buffer,
+				layout: wgpu::TexelCopyBufferLayout {
+					offset: 0,
+					// This needs to be a multiple of 256.
+					bytes_per_row: Some(STRIDE_SIZE),
+					rows_per_image: Some(SIZE.y),
+				},
+			},
+			wgpu::Extent3d {
+				width: SIZE.x,
+				height: SIZE.y,
+				depth_or_array_layers: 1,
+			},
+		);
+
+		self.queue.submit([encoder.finish()]);
+
+		// map buffer
+		let buffer_slice = self.output_staging_buffer.slice(..);
+		let (sender, receiver) = std::sync::mpsc::channel();
+		buffer_slice.map_async(wgpu::MapMode::Read, move |r| sender.send(r).unwrap());
+		self.device
+			.poll(wgpu::PollType::wait_indefinitely())
+			.unwrap();
+		receiver.recv().unwrap().unwrap();
+
+		// copy texture inside a scope so the mapped view gets dropped
+		{
+			let view = buffer_slice.get_mapped_range();
+			self.texture_data.extend_from_slice(&view[..]);
+		}
+		self.output_staging_buffer.unmap();
+
+		// discombolulate stride
+		for y in 0..SIZE.y {
+			inputs.extend(
+				self.texture_data[(STRIDE_SIZE * y) as usize
+					..(STRIDE_SIZE * y + SIZE.x * size_of::<f32>() as u32) as usize]
+					.chunks_exact(4)
+					.map(|b| 1.0 - 2.0 * f32::from_le_bytes(b.try_into().unwrap())),
+			)
+		}
+
+		self.texture_data.clear();
+	}
+}

src/main.rs

@@ -1,611 +1,30 @@
-use burn::backend::Autodiff;
+use clap::{Parser,Subcommand};
-use burn::nn::loss::{MseLoss, Reduction};
-use burn::nn::{Dropout, DropoutConfig, Linear, LinearConfig, Relu};
-use burn::optim::{AdamConfig, GradientsParams, Optimizer};
-use burn::prelude::*;
 
-type InferenceBackend = burn::backend::Cuda<f32>;
+mod net;
-type TrainingBackend = Autodiff<InferenceBackend>;
+mod inputs;
+mod inference;
+mod training;
 
-const LIMITS: wgpu::Limits = wgpu::Limits::defaults();
+#[derive(Parser)]
-const FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8UnormSrgb;
+#[command(author,version,about,long_about=None)]
-use strafesnet_graphics::setup;
+#[command(propagate_version=true)]
-use strafesnet_roblox_bot_file::v0;
+struct Cli{
-
+	#[command(subcommand)]
-const SIZE: glam::UVec2 = glam::uvec2(64, 36);
+	command:Commands,
-const POSITION_HISTORY: usize = 4;
-const INPUT: usize = (SIZE.x * SIZE.y) as usize + POSITION_HISTORY * 3;
-const HIDDEN: [usize; 2] = [INPUT >> 3, INPUT >> 7];
-// MoveForward
-// MoveLeft
-// MoveBack
-// MoveRight
-// Jump
-// mouse_dx
-// mouse_dy
-const OUTPUT: usize = 7;
-
-// bytes_per_row needs to be a multiple of 256.
-const STRIDE_SIZE: u32 = (SIZE.x * size_of::<f32>() as u32).next_multiple_of(256);
-
-#[derive(Module, Debug)]
-struct Net<B: Backend> {
-	input: Linear<B>,
-	dropout: Dropout,
-	hidden: [Linear<B>; HIDDEN.len() - 1],
-	output: Linear<B>,
-	activation: Relu,
-}
-impl<B: Backend> Net<B> {
-	fn init(device: &B::Device) -> Self {
-		let mut it = HIDDEN.into_iter();
-		let mut last_size = it.next().unwrap();
-		let input = LinearConfig::new(INPUT, last_size).init(device);
-		let hidden = core::array::from_fn(|_| {
-			let size = it.next().unwrap();
-			let layer = LinearConfig::new(last_size, size).init(device);
-			last_size = size;
-			layer
-		});
-		let output = LinearConfig::new(last_size, OUTPUT).init(device);
-		let dropout = DropoutConfig::new(0.1).init();
-		Self {
-			input,
-			dropout,
-			hidden,
-			output,
-			activation: Relu::new(),
-		}
-	}
-	fn forward(&self, input: Tensor<B, 2>) -> Tensor<B, 2> {
-		let x = self.input.forward(input);
-		let x = self.dropout.forward(x);
-		let mut x = self.activation.forward(x);
-		for layer in &self.hidden {
-			x = layer.forward(x);
-			x = self.activation.forward(x);
-		}
-		self.output.forward(x)
-	}
 }
 
-struct GraphicsState {
+#[derive(Subcommand)]
-	device: wgpu::Device,
+enum Commands{
-	queue: wgpu::Queue,
+	#[command(flatten)]
-	graphics: strafesnet_roblox_bot_player::graphics::Graphics,
+	Roblox(inference::Commands),
-	graphics_texture_view: wgpu::TextureView,
+	#[command(flatten)]
-	output_staging_buffer: wgpu::Buffer,
+	Source(training::Commands),
-	texture_data: Vec<u8>,
-	position_history: Vec<glam::Vec3>,
-}
-impl GraphicsState {
-	fn new(map: &strafesnet_common::map::CompleteMap) -> Self {
-		let desc = wgpu::InstanceDescriptor::new_without_display_handle_from_env();
-		let instance = wgpu::Instance::new(desc);
-		let (device, queue) = pollster::block_on(async {
-			let adapter = instance
-				.request_adapter(&wgpu::RequestAdapterOptions {
-					power_preference: wgpu::PowerPreference::HighPerformance,
-					force_fallback_adapter: false,
-					compatible_surface: None,
-				})
-				.await
-				.unwrap();
-			setup::step4::request_device(&adapter, LIMITS)
-				.await
-				.unwrap()
-		});
-		let mut graphics = strafesnet_roblox_bot_player::graphics::Graphics::new(
-			&device, &queue, SIZE, FORMAT, LIMITS,
-		);
-		graphics.change_map(&device, &queue, map).unwrap();
-		let graphics_texture = device.create_texture(&wgpu::TextureDescriptor {
-			label: Some("RGB texture"),
-			format: FORMAT,
-			size: wgpu::Extent3d {
-				width: SIZE.x,
-				height: SIZE.y,
-				depth_or_array_layers: 1,
-			},
-			mip_level_count: 1,
-			sample_count: 1,
-			dimension: wgpu::TextureDimension::D2,
-			usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING,
-			view_formats: &[],
-		});
-		let graphics_texture_view = graphics_texture.create_view(&wgpu::TextureViewDescriptor {
-			label: Some("RGB texture view"),
-			aspect: wgpu::TextureAspect::All,
-			usage: Some(
-				wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING,
-			),
-			..Default::default()
-		});
-		let texture_data = Vec::<u8>::with_capacity((STRIDE_SIZE * SIZE.y) as usize);
-		let output_staging_buffer = device.create_buffer(&wgpu::BufferDescriptor {
-			label: Some("Output staging buffer"),
-			size: texture_data.capacity() as u64,
-			usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
-			mapped_at_creation: false,
-		});
-		let position_history = Vec::with_capacity(POSITION_HISTORY);
-		Self {
-			device,
-			queue,
-			graphics,
-			graphics_texture_view,
-			output_staging_buffer,
-			texture_data,
-			position_history,
-		}
-	}
-	fn generate_inputs(&mut self, pos: glam::Vec3, angles: glam::Vec2, inputs: &mut Vec<f32>) {
-		// write position history to model inputs
-		if !self.position_history.is_empty() {
-			let camera = strafesnet_graphics::graphics::view_inv(pos, angles).inverse();
-			for &pos in self.position_history.iter().rev() {
-				let relative_pos = camera.transform_vector3(pos);
-				inputs.extend_from_slice(&relative_pos.to_array());
-			}
-		}
-		// fill remaining history with zeroes
-		for _ in self.position_history.len()..POSITION_HISTORY {
-			inputs.extend_from_slice(&[0.0, 0.0, 0.0]);
-		}
-
-		// track position history
-		if self.position_history.len() < POSITION_HISTORY {
-			self.position_history.push(pos);
-		} else {
-			self.position_history.rotate_left(1);
-			*self.position_history.last_mut().unwrap() = pos;
-		}
-
-		let mut encoder = self
-			.device
-			.create_command_encoder(&wgpu::CommandEncoderDescriptor {
-				label: Some("wgpu encoder"),
-			});
-
-		// render!
-		self.graphics
-			.encode_commands(&mut encoder, &self.graphics_texture_view, pos, angles);
-
-		// copy the depth texture into ram
-		encoder.copy_texture_to_buffer(
-			wgpu::TexelCopyTextureInfo {
-				texture: self.graphics.depth_texture(),
-				mip_level: 0,
-				origin: wgpu::Origin3d::ZERO,
-				aspect: wgpu::TextureAspect::All,
-			},
-			wgpu::TexelCopyBufferInfo {
-				buffer: &self.output_staging_buffer,
-				layout: wgpu::TexelCopyBufferLayout {
-					offset: 0,
-					// This needs to be a multiple of 256.
-					bytes_per_row: Some(STRIDE_SIZE),
-					rows_per_image: Some(SIZE.y),
-				},
-			},
-			wgpu::Extent3d {
-				width: SIZE.x,
-				height: SIZE.y,
-				depth_or_array_layers: 1,
-			},
-		);
-
-		self.queue.submit([encoder.finish()]);
-
-		// map buffer
-		let buffer_slice = self.output_staging_buffer.slice(..);
-		let (sender, receiver) = std::sync::mpsc::channel();
-		buffer_slice.map_async(wgpu::MapMode::Read, move |r| sender.send(r).unwrap());
-		self.device
-			.poll(wgpu::PollType::wait_indefinitely())
-			.unwrap();
-		receiver.recv().unwrap().unwrap();
-
-		// copy texture inside a scope so the mapped view gets dropped
-		{
-			let view = buffer_slice.get_mapped_range();
-			self.texture_data.extend_from_slice(&view[..]);
-		}
-		self.output_staging_buffer.unmap();
-
-		// discombolulate stride
-		for y in 0..SIZE.y {
-			inputs.extend(
-				self.texture_data[(STRIDE_SIZE * y) as usize
-					..(STRIDE_SIZE * y + SIZE.x * size_of::<f32>() as u32) as usize]
-					.chunks_exact(4)
-					.map(|b| 1.0 - 2.0 * f32::from_le_bytes(b.try_into().unwrap())),
-			)
-		}
-
-		self.texture_data.clear();
-	}
-}
-
-fn training() {
-	let gpu_id: usize = std::env::args()
-		.skip(1)
-		.next()
-		.map(|id| id.parse().unwrap())
-		.unwrap_or_default();
-	// load map
-	// load replay
-	// setup player
-
-	let map_file = include_bytes!("../files/bhop_marble_5692093612.snfm");
-	let bot_file = include_bytes!("../files/bhop_marble_7cf33a64-7120-4514-b9fa-4fe29d9523d.qbot");
-
-	// read files
-	let map = strafesnet_snf::read_map(std::io::Cursor::new(map_file))
-		.unwrap()
-		.into_complete_map()
-		.unwrap();
-	let timelines =
-		strafesnet_roblox_bot_file::v0::read_all_to_block(std::io::Cursor::new(bot_file)).unwrap();
-	let bot = strafesnet_roblox_bot_player::bot::CompleteBot::new(timelines).unwrap();
-	let world_offset = bot.world_offset();
-	let timelines = bot.timelines();
-
-	// setup simulation
-	// run progressively longer segments of the map, starting very close to the end of the run and working the starting time backwards until the ai can run the whole map
-
-	// set up graphics
-	let mut g = GraphicsState::new(&map);
-
-	// training data
-	let training_samples = timelines.input_events.len() - 1;
-
-	let input_size = INPUT * size_of::<f32>();
-	let mut inputs = Vec::with_capacity(input_size * training_samples);
-	let mut targets = Vec::with_capacity(OUTPUT * training_samples);
-
-	// generate all frames
-	println!("Generating {training_samples} frames of depth textures...");
-	let mut it = timelines.input_events.iter();
-
-	// grab mouse position from first frame, omitting one frame from the training data
-	let first = it.next().unwrap();
-	let mut last_mx = first.event.mouse_pos.x;
-	let mut last_my = first.event.mouse_pos.y;
-
-	for input_event in it {
-		let mouse_dx = input_event.event.mouse_pos.x - last_mx;
-		let mouse_dy = input_event.event.mouse_pos.y - last_my;
-		last_mx = input_event.event.mouse_pos.x;
-		last_my = input_event.event.mouse_pos.y;
-
-		// set targets
-		targets.extend([
-			// MoveForward
-			input_event
-				.event
-				.game_controls
-				.contains(v0::GameControls::MoveForward) as i32 as f32,
-			// MoveLeft
-			input_event
-				.event
-				.game_controls
-				.contains(v0::GameControls::MoveLeft) as i32 as f32,
-			// MoveBack
-			input_event
-				.event
-				.game_controls
-				.contains(v0::GameControls::MoveBack) as i32 as f32,
-			// MoveRight
-			input_event
-				.event
-				.game_controls
-				.contains(v0::GameControls::MoveRight) as i32 as f32,
-			// Jump
-			input_event
-				.event
-				.game_controls
-				.contains(v0::GameControls::Jump) as i32 as f32,
-			mouse_dx,
-			mouse_dy,
-		]);
-
-		// find the closest output event to the input event time
-		let output_event_index = timelines
-			.output_events
-			.binary_search_by(|event| event.time.partial_cmp(&input_event.time).unwrap());
-
-		let output_event = match output_event_index {
-			// found the exact same timestamp
-			Ok(output_event_index) => &timelines.output_events[output_event_index],
-			// found first index greater than the time.
-			// check this index and the one before and return the closest one
-			Err(insert_index) => timelines
-				.output_events
-				.get(insert_index)
-				.into_iter()
-				.chain(
-					insert_index
-						.checked_sub(1)
-						.and_then(|index| timelines.output_events.get(index)),
-				)
-				.min_by(|&e0, &e1| {
-					(e0.time - input_event.time)
-						.abs()
-						.partial_cmp(&(e1.time - input_event.time).abs())
-						.unwrap()
-				})
-				.unwrap(),
-		};
-
-		fn vec3(v: v0::Vector3) -> glam::Vec3 {
-			glam::vec3(v.x, v.y, v.z)
-		}
-		fn angles(a: v0::Vector3) -> glam::Vec2 {
-			glam::vec2(a.y, a.x)
-		}
-
-		let pos = vec3(output_event.event.position) - world_offset;
-		let angles = angles(output_event.event.angles);
-
-		g.generate_inputs(pos, angles, &mut inputs);
-	}
-
-	let device = burn::backend::cuda::CudaDevice::new(gpu_id);
-
-	let mut model: Net<TrainingBackend> = Net::init(&device);
-	println!("Training model ({} parameters)", model.num_params());
-
-	let mut optim = AdamConfig::new().init();
-
-	let inputs = Tensor::from_data(
-		TensorData::new(inputs, Shape::new([training_samples, INPUT])),
-		&device,
-	);
-	let targets = Tensor::from_data(
-		TensorData::new(targets, Shape::new([training_samples, OUTPUT])),
-		&device,
-	);
-
-	const LEARNING_RATE: f64 = 0.001;
-	const EPOCHS: usize = 100000;
-
-	let mut best_model = model.clone();
-	let mut best_loss = f32::INFINITY;
-
-	for epoch in 0..EPOCHS {
-		let predictions = model.forward(inputs.clone());
-
-		let loss = MseLoss::new().forward(predictions, targets.clone(), Reduction::Mean);
-
-		let loss_scalar = loss.clone().into_scalar();
-
-		if epoch == 0 {
-			// kinda a fake print, but that's what is happening after this point
-			println!("Compiling optimized GPU kernels...");
-		}
-
-		let grads = loss.backward();
-		let grads = GradientsParams::from_grads(grads, &model);
-
-		// get the best model
-		if loss_scalar < best_loss {
-			best_loss = loss_scalar;
-			best_model = model.clone();
-		}
-
-		model = optim.step(LEARNING_RATE, model, grads);
-
-		if epoch % (EPOCHS >> 4) == 0 || epoch == EPOCHS - 1 {
-			// .clone().into_scalar() extracts the f32 value from a 1-element tensor.
-			println!("  epoch {:>5} | loss = {:.8}", epoch, loss_scalar);
-		}
-	}
-
-	let date_string = format!("{}_{}.model", chrono::Utc::now(), best_loss);
-	best_model
-		.save_file(
-			date_string,
-			&burn::record::BinFileRecorder::<burn::record::FullPrecisionSettings>::new(),
-		)
-		.unwrap();
-}
-
-use strafesnet_common::instruction::TimedInstruction;
-use strafesnet_common::mouse::MouseState;
-use strafesnet_common::physics::{
-	Instruction as PhysicsInputInstruction, MiscInstruction, ModeInstruction, MouseInstruction,
-	SetControlInstruction, Time as PhysicsTime,
-};
-use strafesnet_physics::physics::{PhysicsContext, PhysicsData, PhysicsState};
-
-pub struct Recording {
-	instructions: Vec<TimedInstruction<PhysicsInputInstruction, PhysicsTime>>,
-}
-struct FrameState {
-	trajectory: strafesnet_physics::physics::Trajectory,
-	camera: strafesnet_physics::physics::PhysicsCamera,
-}
-impl FrameState {
-	fn pos(&self, time: PhysicsTime) -> glam::Vec3 {
-		self.trajectory
-			.extrapolated_position(time)
-			.map(Into::<f32>::into)
-			.to_array()
-			.into()
-	}
-	fn angles(&self) -> glam::Vec2 {
-		self.camera.simulate_move_angles(glam::IVec2::ZERO)
-	}
-}
-struct Session {
-	geometry_shared: PhysicsData,
-	simulation: PhysicsState,
-	recording: Recording,
-}
-impl Session {
-	fn get_frame_state(&self) -> FrameState {
-		FrameState {
-			trajectory: self.simulation.camera_trajectory(&self.geometry_shared),
-			camera: self.simulation.camera(),
-		}
-	}
-	fn run(&mut self, time: PhysicsTime, instruction: PhysicsInputInstruction) {
-		let instruction = TimedInstruction { time, instruction };
-		self.recording.instructions.push(instruction.clone());
-		PhysicsContext::run_input_instruction(
-			&mut self.simulation,
-			&self.geometry_shared,
-			instruction,
-		);
-	}
-}
-
-fn inference() {
-	let mut args = std::env::args().skip(1);
-
-	// pick device
-	let gpu_id: usize = args
-		.next()
-		.map(|id| id.parse().unwrap())
-		.unwrap_or_default();
-	let device = burn::backend::cuda::CudaDevice::new(gpu_id);
-
-	// load model
-	let path: std::path::PathBuf = args.next().unwrap().parse().unwrap();
-	let mut model: Net<InferenceBackend> = Net::init(&device);
-	model = model
-		.load_file(
-			path,
-			&burn::record::BinFileRecorder::<burn::record::FullPrecisionSettings>::new(),
-			&device,
-		)
-		.unwrap();
-
-	// load map
-	let map_file = include_bytes!("../files/bhop_marble_5692093612.snfm");
-	let map = strafesnet_snf::read_map(std::io::Cursor::new(map_file))
-		.unwrap()
-		.into_complete_map()
-		.unwrap();
-	let modes = map.modes.clone().denormalize();
-	let mode = modes
-		.get_mode(strafesnet_common::gameplay_modes::ModeId::MAIN)
-		.unwrap();
-	let start_zone = map.models.get(mode.get_start().get() as usize).unwrap();
-	let start_offset = glam::Vec3::from_array(
-		start_zone
-			.transform
-			.translation
-			.map(|f| f.into())
-			.to_array(),
-	);
-
-	// setup graphics
-	let mut g = GraphicsState::new(&map);
-
-	// setup simulation
-	let mut session = Session {
-		geometry_shared: PhysicsData::new(&map),
-		simulation: PhysicsState::default(),
-		recording: Recording {
-			instructions: Vec::new(),
-		},
-	};
-
-	let mut time = PhysicsTime::ZERO;
-
-	// reset to start zone
-	session.run(time, PhysicsInputInstruction::Mode(ModeInstruction::Reset));
-	// session.run(
-	// 	time,
-	// 	PhysicsInputInstruction::Misc(MiscInstruction::SetSensitivity(?)),
-	// );
-	session.run(
-		time,
-		PhysicsInputInstruction::Mode(ModeInstruction::Restart(
-			strafesnet_common::gameplay_modes::ModeId::MAIN,
-		)),
-	);
-
-	// TEMP: turn mouse left
-	let mut mouse_pos = glam::ivec2(-5300, 0);
-
-	const STEP: PhysicsTime = PhysicsTime::from_millis(10);
-	let mut input_floats = Vec::new();
-	// setup agent-simulation feedback loop
-	for _ in 0..20 * 100 {
-		// generate inputs
-		let frame_state = session.get_frame_state();
-		g.generate_inputs(
-			frame_state.pos(time) - start_offset,
-			frame_state.angles(),
-			&mut input_floats,
-		);
-
-		// inference
-		let inputs = Tensor::from_data(
-			TensorData::new(input_floats.clone(), Shape::new([1, INPUT])),
-			&device,
-		);
-		let outputs = model.forward(inputs).into_data().into_vec::<f32>().unwrap();
-
-		let &[
-			move_forward,
-			move_left,
-			move_back,
-			move_right,
-			jump,
-			mouse_dx,
-			mouse_dy,
-		] = outputs.as_slice()
-		else {
-			panic!()
-		};
-
-		macro_rules! set_control {
-			($control:ident,$output:expr) => {
-				session.run(
-					time,
-					PhysicsInputInstruction::SetControl(SetControlInstruction::$control(
-						0.5 < $output,
-					)),
-				);
-			};
-		}
-		set_control!(SetMoveForward, move_forward);
-		set_control!(SetMoveLeft, move_left);
-		set_control!(SetMoveBack, move_back);
-		set_control!(SetMoveRight, move_right);
-		set_control!(SetJump, jump);
-
-		mouse_pos += glam::vec2(mouse_dx, mouse_dy).round().as_ivec2();
-		let next_time = time + STEP;
-		session.run(
-			time,
-			PhysicsInputInstruction::Mouse(MouseInstruction::SetNextMouse(MouseState {
-				pos: mouse_pos,
-				time: next_time,
-			})),
-		);
-
-		time = next_time;
-
-		// clear
-		input_floats.clear();
-	}
-
-	let date_string = format!("{}.snfb", chrono::Utc::now());
-	let file = std::fs::File::create(date_string).unwrap();
-	strafesnet_snf::bot::write_bot(
-		std::io::BufWriter::new(file),
-		strafesnet_physics::VERSION.get(),
-		core::mem::take(&mut session.recording.instructions),
-	)
-	.unwrap();
 }
 
 fn main(){
-	// training();
+	let cli=Cli::parse();
-	inference();
+	match cli.command{
+		Commands::Roblox(commands)=>commands.run(),
+		Commands::Source(commands)=>commands.run(),
+	}
 }

src/net.rs

@@ -0,0 +1,60 @@
+use burn::backend::Autodiff;
+use burn::nn::{Dropout, DropoutConfig, Linear, LinearConfig, Relu};
+use burn::prelude::*;
+
+pub type InferenceBackend = burn::backend::Cuda<f32>;
+pub type TrainingBackend = Autodiff<InferenceBackend>;
+
+pub const SIZE: glam::UVec2 = glam::uvec2(64, 36);
+pub const POSITION_HISTORY: usize = 4;
+pub const INPUT: usize = (SIZE.x * SIZE.y) as usize + POSITION_HISTORY * 3;
+pub const HIDDEN: [usize; 2] = [INPUT >> 3, INPUT >> 7];
+// MoveForward
+// MoveLeft
+// MoveBack
+// MoveRight
+// Jump
+// mouse_dx
+// mouse_dy
+pub const OUTPUT: usize = 7;
+
+#[derive(Module, Debug)]
+pub struct Net<B: Backend> {
+	input: Linear<B>,
+	dropout: Dropout,
+	hidden: [Linear<B>; HIDDEN.len() - 1],
+	output: Linear<B>,
+	activation: Relu,
+}
+impl<B: Backend> Net<B> {
+	pub fn init(device: &B::Device) -> Self {
+		let mut it = HIDDEN.into_iter();
+		let mut last_size = it.next().unwrap();
+		let input = LinearConfig::new(INPUT, last_size).init(device);
+		let hidden = core::array::from_fn(|_| {
+			let size = it.next().unwrap();
+			let layer = LinearConfig::new(last_size, size).init(device);
+			last_size = size;
+			layer
+		});
+		let output = LinearConfig::new(last_size, OUTPUT).init(device);
+		let dropout = DropoutConfig::new(0.1).init();
+		Self {
+			input,
+			dropout,
+			hidden,
+			output,
+			activation: Relu::new(),
+		}
+	}
+	pub fn forward(&self, input: Tensor<B, 2>) -> Tensor<B, 2> {
+		let x = self.input.forward(input);
+		let x = self.dropout.forward(x);
+		let mut x = self.activation.forward(x);
+		for layer in &self.hidden {
+			x = layer.forward(x);
+			x = self.activation.forward(x);
+		}
+		self.output.forward(x)
+	}
+}

src/training.rs

@@ -0,0 +1,212 @@
+#[derive(clap::Subcommand)]
+pub enum Commands {
+	Train(TrainSubcommand),
+}
+impl Commands {
+	pub fn run(self) {
+		match self {
+			Commands::Train(subcommand) => subcommand.run(),
+		}
+	}
+}
+
+#[derive(clap::Args)]
+pub struct TrainSubcommand {}
+impl TrainSubcommand {
+	fn run(self) {
+		training();
+	}
+}
+
+use burn::nn::loss::{MseLoss, Reduction};
+use burn::optim::{AdamConfig, GradientsParams, Optimizer};
+use burn::prelude::*;
+
+use crate::inputs::GraphicsState;
+use crate::net::{INPUT, Net, OUTPUT, TrainingBackend};
+
+use strafesnet_roblox_bot_file::v0;
+
+fn training() {
+	let gpu_id: usize = std::env::args()
+		.skip(1)
+		.next()
+		.map(|id| id.parse().unwrap())
+		.unwrap_or_default();
+	// load map
+	// load replay
+	// setup player
+
+	let map_file = include_bytes!("../files/bhop_marble_5692093612.snfm");
+	let bot_file = include_bytes!("../files/bhop_marble_7cf33a64-7120-4514-b9fa-4fe29d9523d.qbot");
+
+	// read files
+	let map = strafesnet_snf::read_map(std::io::Cursor::new(map_file))
+		.unwrap()
+		.into_complete_map()
+		.unwrap();
+	let timelines =
+		strafesnet_roblox_bot_file::v0::read_all_to_block(std::io::Cursor::new(bot_file)).unwrap();
+	let bot = strafesnet_roblox_bot_player::bot::CompleteBot::new(timelines).unwrap();
+	let world_offset = bot.world_offset();
+	let timelines = bot.timelines();
+
+	// setup simulation
+	// run progressively longer segments of the map, starting very close to the end of the run and working the starting time backwards until the ai can run the whole map
+
+	// set up graphics
+	let mut g = GraphicsState::new(&map);
+
+	// training data
+	let training_samples = timelines.input_events.len() - 1;
+
+	let input_size = INPUT * size_of::<f32>();
+	let mut inputs = Vec::with_capacity(input_size * training_samples);
+	let mut targets = Vec::with_capacity(OUTPUT * training_samples);
+
+	// generate all frames
+	println!("Generating {training_samples} frames of depth textures...");
+	let mut it = timelines.input_events.iter();
+
+	// grab mouse position from first frame, omitting one frame from the training data
+	let first = it.next().unwrap();
+	let mut last_mx = first.event.mouse_pos.x;
+	let mut last_my = first.event.mouse_pos.y;
+
+	for input_event in it {
+		let mouse_dx = input_event.event.mouse_pos.x - last_mx;
+		let mouse_dy = input_event.event.mouse_pos.y - last_my;
+		last_mx = input_event.event.mouse_pos.x;
+		last_my = input_event.event.mouse_pos.y;
+
+		// set targets
+		targets.extend([
+			// MoveForward
+			input_event
+				.event
+				.game_controls
+				.contains(v0::GameControls::MoveForward) as i32 as f32,
+			// MoveLeft
+			input_event
+				.event
+				.game_controls
+				.contains(v0::GameControls::MoveLeft) as i32 as f32,
+			// MoveBack
+			input_event
+				.event
+				.game_controls
+				.contains(v0::GameControls::MoveBack) as i32 as f32,
+			// MoveRight
+			input_event
+				.event
+				.game_controls
+				.contains(v0::GameControls::MoveRight) as i32 as f32,
+			// Jump
+			input_event
+				.event
+				.game_controls
+				.contains(v0::GameControls::Jump) as i32 as f32,
+			mouse_dx,
+			mouse_dy,
+		]);
+
+		// find the closest output event to the input event time
+		let output_event_index = timelines
+			.output_events
+			.binary_search_by(|event| event.time.partial_cmp(&input_event.time).unwrap());
+
+		let output_event = match output_event_index {
+			// found the exact same timestamp
+			Ok(output_event_index) => &timelines.output_events[output_event_index],
+			// found first index greater than the time.
+			// check this index and the one before and return the closest one
+			Err(insert_index) => timelines
+				.output_events
+				.get(insert_index)
+				.into_iter()
+				.chain(
+					insert_index
+						.checked_sub(1)
+						.and_then(|index| timelines.output_events.get(index)),
+				)
+				.min_by(|&e0, &e1| {
+					(e0.time - input_event.time)
+						.abs()
+						.partial_cmp(&(e1.time - input_event.time).abs())
+						.unwrap()
+				})
+				.unwrap(),
+		};
+
+		fn vec3(v: v0::Vector3) -> glam::Vec3 {
+			glam::vec3(v.x, v.y, v.z)
+		}
+		fn angles(a: v0::Vector3) -> glam::Vec2 {
+			glam::vec2(a.y, a.x)
+		}
+
+		let pos = vec3(output_event.event.position) - world_offset;
+		let angles = angles(output_event.event.angles);
+
+		g.generate_inputs(pos, angles, &mut inputs);
+	}
+
+	let device = burn::backend::cuda::CudaDevice::new(gpu_id);
+
+	let mut model: Net<TrainingBackend> = Net::init(&device);
+	println!("Training model ({} parameters)", model.num_params());
+
+	let mut optim = AdamConfig::new().init();
+
+	let inputs = Tensor::from_data(
+		TensorData::new(inputs, Shape::new([training_samples, INPUT])),
+		&device,
+	);
+	let targets = Tensor::from_data(
+		TensorData::new(targets, Shape::new([training_samples, OUTPUT])),
+		&device,
+	);
+
+	const LEARNING_RATE: f64 = 0.001;
+	const EPOCHS: usize = 100000;
+
+	let mut best_model = model.clone();
+	let mut best_loss = f32::INFINITY;
+
+	for epoch in 0..EPOCHS {
+		let predictions = model.forward(inputs.clone());
+
+		let loss = MseLoss::new().forward(predictions, targets.clone(), Reduction::Mean);
+
+		let loss_scalar = loss.clone().into_scalar();
+
+		if epoch == 0 {
+			// kinda a fake print, but that's what is happening after this point
+			println!("Compiling optimized GPU kernels...");
+		}
+
+		let grads = loss.backward();
+		let grads = GradientsParams::from_grads(grads, &model);
+
+		// get the best model
+		if loss_scalar < best_loss {
+			best_loss = loss_scalar;
+			best_model = model.clone();
+		}
+
+		model = optim.step(LEARNING_RATE, model, grads);
+
+		if epoch % (EPOCHS >> 4) == 0 || epoch == EPOCHS - 1 {
+			// .clone().into_scalar() extracts the f32 value from a 1-element tensor.
+			println!("  epoch {:>5} | loss = {:.8}", epoch, loss_scalar);
+		}
+	}
+
+	let date_string = format!("{}_{}.model", chrono::Utc::now(), best_loss);
+	best_model
+		.save_file(
+			date_string,
+			&burn::record::BinFileRecorder::<burn::record::FullPrecisionSettings>::new(),
+		)
+		.unwrap();
+}