Rectify and reproject image

demo/UltraleapCamera.gd

@@ -0,0 +1,20 @@
+extends Camera3D
+
+@export var materials : Array[ShaderMaterial]
+
+func _ready():
+	var view_size = get_viewport().size
+	_update_projection(view_size)
+
+
+func _on_view_size_changed(view_size):
+	_update_projection(view_size)
+
+
+func _update_projection(view_size):
+	var aspect = view_size.x / float(view_size.y)
+	var fovy = rad_to_deg(2 * atan(tan(deg_to_rad(fov) * 0.5) * aspect))
+	var _projection : Projection = Projection.create_perspective(fovy, aspect, near, far, keep_aspect == KEEP_HEIGHT)
+	for material in materials:
+		material.set_shader_parameter("projection", _projection)
+		pass

demo/main.tscn

@@ -1,4 +1,4 @@
-[gd_scene load_steps=97 format=3 uid="uid://2vmsr5t51l5c"]
+[gd_scene load_steps=103 format=3 uid="uid://2vmsr5t51l5c"]
 
 [ext_resource type="Script" path="res://scripts/Visualiser.gd" id="1_i4o6k"]
 [ext_resource type="Script" path="res://scripts/Starter.gd" id="1_pwqvd"]
@@ -7,10 +7,14 @@
 [ext_resource type="Texture2D" uid="uid://64qjvjymo7oj" path="res://textures/LeftImage.tres" id="3_48g1p"]
 [ext_resource type="Sky" uid="uid://dlont1iyrrl3g" path="res://materials/SkyImages.tres" id="3_il4lh"]
 [ext_resource type="Texture2D" uid="uid://t5jb84gwhfx4" path="res://textures/RightImage.tres" id="4_antfi"]
+[ext_resource type="Script" path="res://UltraleapCamera.gd" id="4_gr4o3"]
 [ext_resource type="Script" path="res://scripts/ImagePuller.gd" id="4_sgil3"]
 [ext_resource type="Texture2D" uid="uid://7h3wom5btmhp" path="res://textures/LeftDistortion.tres" id="7_vascd"]
 [ext_resource type="Script" path="res://scripts/DebugHands.gd" id="8_uah46"]
 [ext_resource type="Texture2D" uid="uid://dwx41gu1dkwgt" path="res://textures/RightDistortion.tres" id="9_douml"]
+[ext_resource type="Texture2D" uid="uid://bupl2rdcvjd1w" path="res://textures/RawLeftImage.tres" id="9_h7pse"]
+[ext_resource type="Material" uid="uid://cgbf7yeymk5ch" path="res://materials/canvas_image.tres" id="12_vmkw8"]
+[ext_resource type="Material" uid="uid://ewxi4ovsa7p1" path="res://materials/spatial_image.tres" id="15_3ed8b"]
 
 [sub_resource type="Environment" id="Environment_3k2hr"]
 background_mode = 2
@@ -394,6 +398,11 @@ height = 0.0141687
 radial_segments = 8
 rings = 4
 
+[sub_resource type="Gradient" id="Gradient_n8bxu"]
+
+[sub_resource type="GradientTexture2D" id="GradientTexture2D_x2m6l"]
+gradient = SubResource("Gradient_n8bxu")
+
 [node name="Visualiser" type="Node3D" node_paths=PackedStringArray("tracking", "camera", "hands")]
 script = ExtResource("1_i4o6k")
 tracking = NodePath("UltraleapHandTracking")
@@ -401,17 +410,26 @@ camera = NodePath("Camera3D")
 hands = NodePath("Hands")
 hud = NodePath("HUD")
 sky = ExtResource("2_4y0uk")
+materials_to_rectify = Array[Material]([ExtResource("15_3ed8b"), ExtResource("12_vmkw8")])
 
 [node name="UltraleapHandTracking" type="UltraleapHandTracking" parent="."]
-transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, -0.0572538, 0.0773736, 0.281333)
 script = ExtResource("1_pwqvd")
 
 [node name="Camera3D" type="Camera3D" parent="."]
-transform = Transform3D(1, 0, 0, 0, 0.866025, 0.5, 0, -0.5, 0.866025, 0, 0.4, 0.6)
+transform = Transform3D(-1, -8.74227e-08, 3.82137e-15, 0, -4.37114e-08, -1, 8.74228e-08, -1, 4.37114e-08, -0.033, 0, 0)
+fov = 106.0
 near = 0.001
+script = ExtResource("4_gr4o3")
+materials = Array[ShaderMaterial]([ExtResource("12_vmkw8")])
 
-[node name="DirectionalLight3D" type="DirectionalLight3D" parent="."]
-transform = Transform3D(0.818401, 0.410311, -0.402325, 0, 0.700125, 0.714021, 0.574648, -0.584355, 0.572982, 0, 1.81848, 0)
+[node name="DirectionalLight3D" type="DirectionalLight3D" parent="Camera3D"]
+
+[node name="Camera3D2" type="Camera3D" parent="."]
+transform = Transform3D(-1, -8.74227e-08, 3.82137e-15, 0, -4.37114e-08, -1, 8.74228e-08, -1, 4.37114e-08, -0.033, 0, 0)
+fov = 106.0
+near = 0.001
+script = ExtResource("4_gr4o3")
+materials = Array[ShaderMaterial]([ExtResource("12_vmkw8")])
 
 [node name="WorldEnvironment" type="WorldEnvironment" parent="."]
 environment = SubResource("Environment_3k2hr")
@@ -442,6 +460,7 @@ stretch_mode = 5
 script = ExtResource("4_sgil3")
 tracker = NodePath("../../../UltraleapHandTracking")
 distortion_texture = ExtResource("7_vascd")
+raw_image = ExtResource("9_h7pse")
 
 [node name="TextureRect2" type="TextureRect" parent="CanvasLayer/HBoxContainer" node_paths=PackedStringArray("tracker")]
 layout_mode = 2
@@ -634,6 +653,20 @@ mesh = SubResource("CapsuleMesh_dnlrb")
 transform = Transform3D(0.999972, 0.00737919, 0.00080738, -0.00737915, 0.976307, 0.216265, 0.000807624, -0.216265, 0.976334, -5.24223e-05, -0.0163794, 0.00153229)
 mesh = SubResource("CapsuleMesh_mtm4i")
 
+[node name="CanvasLayer2" type="CanvasLayer" parent="."]
+visible = false
+
+[node name="Sprite2D" type="Sprite2D" parent="CanvasLayer2"]
+material = ExtResource("12_vmkw8")
+position = Vector2(500, 500)
+scale = Vector2(10, 10)
+texture = SubResource("GradientTexture2D_x2m6l")
+
+[node name="CSGBox3D" type="CSGBox3D" parent="."]
+transform = Transform3D(100, 0, 0, 0, -4.37114e-06, -1, 0, 100, -4.37114e-08, 0, 10, 0)
+material = ExtResource("15_3ed8b")
+
+[connection signal="view_size_changed" from="." to="Camera3D" method="_on_view_size_changed"]
 [connection signal="view_size_changed" from="." to="CanvasLayer/HBoxContainer/TextureRect" method="_on_view_size_changed"]
 [connection signal="view_size_changed" from="." to="CanvasLayer/HBoxContainer/TextureRect2" method="_on_view_size_changed"]
 [connection signal="device_added" from="UltraleapHandTracking" to="." method="_on_hand_tracking_device_added"]

demo/materials/canvas_image.tres

@@ -0,0 +1,12 @@
+[gd_resource type="ShaderMaterial" load_steps=4 format=3 uid="uid://cgbf7yeymk5ch"]
+
+[ext_resource type="Shader" path="res://shaders/UltraleapCanvas.gdshader" id="1_0fobb"]
+[ext_resource type="Texture2D" uid="uid://7h3wom5btmhp" path="res://textures/LeftDistortion.tres" id="2_t20d2"]
+[ext_resource type="Texture2D" uid="uid://bupl2rdcvjd1w" path="res://textures/RawLeftImage.tres" id="3_4rm8k"]
+
+[resource]
+shader = ExtResource("1_0fobb")
+shader_parameter/undistort = true
+shader_parameter/projection = Projection(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1)
+shader_parameter/image = ExtResource("3_4rm8k")
+shader_parameter/distortion = ExtResource("2_t20d2")

demo/materials/spatial_image.tres

@@ -0,0 +1,12 @@
+[gd_resource type="ShaderMaterial" load_steps=4 format=3 uid="uid://ewxi4ovsa7p1"]
+
+[ext_resource type="Shader" path="res://shaders/UltraleapSpatial.gdshader" id="1_y1a3l"]
+[ext_resource type="Texture2D" uid="uid://7h3wom5btmhp" path="res://textures/LeftDistortion.tres" id="2_6ary6"]
+[ext_resource type="Texture2D" uid="uid://bupl2rdcvjd1w" path="res://textures/RawLeftImage.tres" id="3_0jrcn"]
+
+[resource]
+render_priority = 0
+shader = ExtResource("1_y1a3l")
+shader_parameter/undistort = true
+shader_parameter/image = ExtResource("3_0jrcn")
+shader_parameter/distortion = ExtResource("2_6ary6")

demo/scripts/ImagePuller.gd

@@ -6,6 +6,7 @@ enum Side { Left, Right }
 @export var side : ImagePuller.Side = Side.Left
 @export var tracker : UltraleapHandTracking
 @export var distortion_texture : Texture2D
+@export var raw_image : Texture2D
 
 var img : Image = Image.new()
 var data_img : Image = Image.new()
@@ -21,7 +22,6 @@ var device_index : int = 0
 var view_size : Vector2i
 var image_size : Vector2i
 
-
 func _process(_delta):
 	var length = min(view_size.x, view_size.y)
 	
@@ -64,6 +64,8 @@ func _process(_delta):
 			
 		last_size = image_size
 		texture.set_image(img)
+		if raw_image != null:
+			raw_image.set_image(data_img)
 
 
 func on_hud_device_selection(serial):
@@ -90,3 +92,4 @@ func _on_view_size_changed(new_view_size : Vector2i):
 func _paint_black(surf_size : Vector2i):
 	bg_data.resize(surf_size.x * surf_size.y)
 	img.set_data(surf_size.x, surf_size.y, false, Image.FORMAT_L8, bg_data)
+	#img.convert(Image.FORMAT_RGBA8)

demo/scripts/Visualiser.gd

@@ -5,6 +5,18 @@ extends Node3D
 @export var hands : DebugHands
 @export_node_path("CanvasLayer") var hud
 @export var sky : ShaderMaterial
+@export var materials_to_rectify : Array[Material]
+
+@export_group("Camera positions")
+@export_subgroup("Desktop")
+@export var desktop_position : Vector3
+@export var desktop_rotation : Vector3
+@export_subgroup("HMD")
+@export var hmd_position : Vector3
+@export var hmd_rotation : Vector3
+@export_subgroup("Screentop")
+@export var screentop_position : Vector3
+@export var screentop_rotation : Vector3
 
 signal view_size_changed(view_size : Vector2i)
 
@@ -71,6 +83,7 @@ func _on_tracking_mode_changed(tracking_mode):
 
 
 func move_camera_based_on_tracking_mode(tracking_mode : UltraleapTypes.TrackingMode):
+	return
 	if (tracking_mode == UltraleapTypes.TrackingMode.HMD):
 		hands.position = Vector3(0, 0.4, 0.6)
 		hands.rotation = Vector3(deg_to_rad(90), deg_to_rad(180), 0)
@@ -90,3 +103,9 @@ func move_camera_based_on_tracking_mode(tracking_mode : UltraleapTypes.TrackingM
 
 func on_undistort_toggled(enabled):
 	sky.set_shader_parameter("undistort", enabled)
+	
+	for material in materials_to_rectify:
+		material.set_shader_parameter("undistort", enabled)
+
+func deg_to_rad(vector : Vector3):
+	return Vector3(deg_to_rad(vector.x), deg_to_rad(vector.y), deg_to_rad(vector.z))

demo/shaders/Ultraleap.gdshaderinc

@@ -0,0 +1,25 @@
+vec2 get_projected_uv(vec2 uv, mat4 projection_matrix) {
+	// Center uv around (0, 0), between (-1, 1)
+	uv = (uv * 2.0) - 1.0;
+	
+	// Get projection parameters
+	vec4 _projection = vec4(
+		projection_matrix[0][2],		// u0
+		-projection_matrix[1][2],		// v0
+		projection_matrix[0][0],		// alphax
+		projection_matrix[1][1]			// alphay
+	);
+
+	vec2 tangent = (uv + _projection.xy) / _projection.zw;
+	
+	// Add vec2(0.5, 0.5) to recenter the image (we offseted it earlier)
+	// 0.125 == 1/8 is due to the fact that the distortion ray angles
+	// in the distortion texture are also multiplied by 1/8
+	vec2 distortion_uv = clamp(0.125 * tangent + vec2(0.5, 0.5), 0, 1);
+	
+	return distortion_uv;
+}
+
+vec2 get_rectified_uv(vec2 uv, sampler2D distortion_matrix) {
+	return texture(distortion_matrix, uv).xy;
+}

demo/shaders/UltraleapCanvas.gdshader

@@ -0,0 +1,20 @@
+shader_type canvas_item;
+
+#include "res://shaders/Ultraleap.gdshaderinc"
+
+uniform sampler2D image : hint_default_black;
+uniform sampler2D distortion : hint_default_black;
+
+uniform bool undistort = true;
+
+uniform mat4 projection;
+
+void fragment() {
+	float undistort_float = float(undistort);
+	vec2 uv = SCREEN_UV;
+	uv.y = 1.0 - uv.y;
+	uv = get_projected_uv(uv, projection);
+	uv = get_rectified_uv(uv, distortion);
+	vec3 pixel = undistort_float * texture(image, uv).rgb + (1.0 - undistort_float) * texture(image, SCREEN_UV).rgb;
+	COLOR.rgb = pow(pixel, vec3(1.0));
+}

demo/shaders/UltraleapSpatial.gdshader

@@ -0,0 +1,17 @@
+shader_type spatial;
+render_mode unshaded;
+
+#include "res://shaders/Ultraleap.gdshaderinc"
+
+uniform sampler2D image : hint_default_black;
+uniform sampler2D distortion : hint_default_black;
+
+uniform bool undistort = true;
+
+void fragment() {
+	float undistort_float = float(undistort);
+	vec2 uv = get_projected_uv(SCREEN_UV, PROJECTION_MATRIX);
+	uv = get_rectified_uv(uv, distortion);
+	vec3 pixel = undistort_float * texture(image, uv).rgb + (1.0 - undistort_float) * texture(image, SCREEN_UV).rgb;
+	ALBEDO.rgb = pow(pixel, vec3(1.6));
+}

demo/textures/RawLeftImage.tres

@@ -0,0 +1,3 @@
+[gd_resource type="ImageTexture" format=3 uid="uid://bupl2rdcvjd1w"]
+
+[resource]