Use RayMap and RenderLayers in bevy_sprite/picking_backend (#21959)

# Credit
- @bytemunch for the original PR and the brunt of the work
- @akimakinai for reviewing and introducing an important change to the
RenderLayers logic

# Objective

- Adoption of the PR found
[here](#18069) (as was
recommended to me on the bevy discord)
- I upgraded the changes proposed there to work with bevy/main
- This likely fixes a bunch of unintuitive picking behavior around
Viewports and RenderLayers. See the original PR for details.

## Solution

- Rewrote parts of `bevy_sprite/picking_backend.rs`. It now iterates
over bevy_picking's `RayMap` to generate `PointerHit`s for each viable
`Camera`, rather than just the first viable `Camera` found.
- Introduced checking if the `Camera`'s and `Sprite`'s `RenderLayers`
intersect before generating a hit.

## Testing

- I tested it with my own game, running on bevy 0.17.3
- I don't expect the behavior to be different for a game built with
bevy/main, as it's just the picking backend. However, that is still open
for testing.

## What reviewers can test
- Before this change, `PointerHit`s would not generate on viewports of
higher order, if their area overlapped with a viewport of lower order.
For example, if you had a `Camera` render to the entire window, and then
a second `Camera` render a smaller viewport on top, the smaller viewport
would not detect `PointerHit`s. This should be fixed now.
- Before this change, a `Camera` not on the default `RenderLayers`
layer, would generate and consume `PointerHit`s, even if it did not
share any layers with the picked `Sprite`s. This could also block
`Sprite`s from being picked at all. This should be fixed now.

## Extra note
The behavior of the picking backend is now to produce `PointerHit`s for
each viable viewport under the cursor. This can lead to the unintuitive
result, that you can pick a `Sprite` you can not see, due to being
obscured by another viewport.
I have a solution for this and it would be a flag in
`SpritePickingSettings`. Should I open a new PR for this or enter a
discussion?

Author: bloopyboop

crates/bevy_sprite/src/picking_backend.rs

@@ -13,7 +13,10 @@
 use crate::{Anchor, Sprite};
 use bevy_app::prelude::*;
 use bevy_asset::prelude::*;
-use bevy_camera::{visibility::ViewVisibility, Camera, Projection};
+use bevy_camera::{
+    visibility::{RenderLayers, ViewVisibility},
+    Camera, Projection,
+};
 use bevy_color::Alpha;
 use bevy_ecs::prelude::*;
 use bevy_image::prelude::*;
@@ -88,6 +91,7 @@ fn sprite_picking(
         &GlobalTransform,
         &Projection,
         Has<SpritePickingCamera>,
+        Option<&RenderLayers>,
     )>,
     primary_window: Query<Entity, With<PrimaryWindow>>,
     images: Res<Assets<Image>>,
@@ -100,157 +104,188 @@ fn sprite_picking(
         &Anchor,
         &Pickable,
         &ViewVisibility,
+        Option<&RenderLayers>,
     )>,
     mut pointer_hits_writer: MessageWriter<PointerHits>,
+    ray_map: Res<RayMap>,
 ) {
     let mut sorted_sprites: Vec<_> = sprite_query
         .iter()
-        .filter_map(|(entity, sprite, transform, anchor, pickable, vis)| {
-            if !transform.affine().is_nan() && vis.get() {
-                Some((entity, sprite, transform, anchor, pickable))
-            } else {
-                None
-            }
-        })
+        .filter_map(
+            |(entity, sprite, transform, anchor, pickable, vis, render_layers)| {
+                if !transform.affine().is_nan() && vis.get() {
+                    Some((entity, sprite, transform, anchor, pickable, render_layers))
+                } else {
+                    None
+                }
+            },
+        )
         .collect();
 
     // radsort is a stable radix sort that performed better than `slice::sort_by_key`
-    radsort::sort_by_key(&mut sorted_sprites, |(_, _, transform, _, _)| {
+    radsort::sort_by_key(&mut sorted_sprites, |(_, _, transform, _, _, _)| {
         -transform.translation().z
     });
 
     let primary_window = primary_window.single().ok();
 
-    for (pointer, location) in pointers.iter().filter_map(|(pointer, pointer_location)| {
-        pointer_location.location().map(|loc| (pointer, loc))
-    }) {
+    let pick_sets = ray_map.iter().flat_map(|(ray_id, ray)| {
         let mut blocked = false;
-        let Some((cam_entity, camera, cam_transform, Projection::Orthographic(cam_ortho), _)) =
-            cameras
-                .iter()
-                .filter(|(_, camera, _, _, cam_can_pick)| {
-                    let marker_requirement = !settings.require_markers || *cam_can_pick;
-                    camera.is_active && marker_requirement
-                })
-                .find(|(_, camera, _, _, _)| {
-                    camera
-                        .target
-                        .normalize(primary_window)
-                        .is_some_and(|x| x == location.target)
-                })
+
+        let Ok((
+            cam_entity,
+            camera,
+            cam_transform,
+            Projection::Orthographic(cam_ortho),
+            cam_can_pick,
+            cam_render_layers,
+        )) = cameras.get(ray_id.camera)
         else {
-            continue;
+            return None;
         };
 
+        let marker_requirement = !settings.require_markers || cam_can_pick;
+        if !camera.is_active || !marker_requirement {
+            return None;
+        }
+
+        let location = pointers.iter().find_map(|(id, loc)| {
+            if *id == ray_id.pointer {
+                return loc.location.as_ref();
+            }
+            None
+        })?;
+
+        if camera
+            .target
+            .normalize(primary_window)
+            .is_none_or(|x| x != location.target)
+        {
+            return None;
+        }
+
         let viewport_pos = location.position;
         if let Some(viewport) = camera.logical_viewport_rect()
             && !viewport.contains(viewport_pos)
         {
             // The pointer is outside the viewport, skip it
-            continue;
+            return None;
         }
 
-        let Ok(cursor_ray_world) = camera.viewport_to_world(cam_transform, viewport_pos) else {
-            continue;
-        };
         let cursor_ray_len = cam_ortho.far - cam_ortho.near;
-        let cursor_ray_end = cursor_ray_world.origin + cursor_ray_world.direction * cursor_ray_len;
+        let cursor_ray_end = ray.origin + ray.direction * cursor_ray_len;
 
         let picks: Vec<(Entity, HitData)> = sorted_sprites
             .iter()
             .copied()
-            .filter_map(|(entity, sprite, sprite_transform, anchor, pickable)| {
-                if blocked {
-                    return None;
-                }
+            .filter_map(
+                |(entity, sprite, sprite_transform, anchor, pickable, sprite_render_layers)| {
+                    if blocked {
+                        return None;
+                    }
 
-                // Transform cursor line segment to sprite coordinate system
-                let world_to_sprite = sprite_transform.affine().inverse();
-                let cursor_start_sprite = world_to_sprite.transform_point3(cursor_ray_world.origin);
-                let cursor_end_sprite = world_to_sprite.transform_point3(cursor_ray_end);
+                    // Filter out sprites based on whether they share RenderLayers with the current
+                    // ray's associated camera.
+                    // Any entity without a RenderLayers component will by default be
+                    // on RenderLayers::layer(0) only.
+                    if !cam_render_layers
+                        .unwrap_or_default()
+                        .intersects(sprite_render_layers.unwrap_or_default())
+                    {
+                        return None;
+                    }
 
-                // Find where the cursor segment intersects the plane Z=0 (which is the sprite's
-                // plane in sprite-local space). It may not intersect if, for example, we're
-                // viewing the sprite side-on
-                if cursor_start_sprite.z == cursor_end_sprite.z {
-                    // Cursor ray is parallel to the sprite and misses it
-                    return None;
-                }
-                let lerp_factor =
-                    f32::inverse_lerp(cursor_start_sprite.z, cursor_end_sprite.z, 0.0);
-                if !(0.0..=1.0).contains(&lerp_factor) {
-                    // Lerp factor is out of range, meaning that while an infinite line cast by
-                    // the cursor would intersect the sprite, the sprite is not between the
-                    // camera's near and far planes
-                    return None;
-                }
-                // Otherwise we can interpolate the xy of the start and end positions by the
-                // lerp factor to get the cursor position in sprite space!
-                let cursor_pos_sprite = cursor_start_sprite
-                    .lerp(cursor_end_sprite, lerp_factor)
-                    .xy();
+                    // Transform cursor line segment to sprite coordinate system
+                    let world_to_sprite = sprite_transform.affine().inverse();
+                    let cursor_start_sprite = world_to_sprite.transform_point3(ray.origin);
+                    let cursor_end_sprite = world_to_sprite.transform_point3(cursor_ray_end);
+
+                    // Find where the cursor segment intersects the plane Z=0 (which is the sprite's
+                    // plane in sprite-local space). It may not intersect if, for example, we're
+                    // viewing the sprite side-on
+                    if cursor_start_sprite.z == cursor_end_sprite.z {
+                        // Cursor ray is parallel to the sprite and misses it
+                        return None;
+                    }
+                    let lerp_factor =
+                        f32::inverse_lerp(cursor_start_sprite.z, cursor_end_sprite.z, 0.0);
+                    if !(0.0..=1.0).contains(&lerp_factor) {
+                        // Lerp factor is out of range, meaning that while an infinite line cast by
+                        // the cursor would intersect the sprite, the sprite is not between the
+                        // camera's near and far planes
+                        return None;
+                    }
+                    // Otherwise we can interpolate the xy of the start and end positions by the
+                    // lerp factor to get the cursor position in sprite space!
+                    let cursor_pos_sprite = cursor_start_sprite
+                        .lerp(cursor_end_sprite, lerp_factor)
+                        .xy();
 
-                let Ok(cursor_pixel_space) = sprite.compute_pixel_space_point(
-                    cursor_pos_sprite,
-                    *anchor,
-                    &images,
-                    &texture_atlas_layout,
-                ) else {
-                    return None;
-                };
+                    let Ok(cursor_pixel_space) = sprite.compute_pixel_space_point(
+                        cursor_pos_sprite,
+                        *anchor,
+                        &images,
+                        &texture_atlas_layout,
+                    ) else {
+                        return None;
+                    };
 
-                // Since the pixel space coordinate is `Ok`, we know the cursor is in the bounds of
-                // the sprite.
+                    // Since the pixel space coordinate is `Ok`, we know the cursor is in the bounds of
+                    // the sprite.
 
-                let cursor_in_valid_pixels_of_sprite = 'valid_pixel: {
-                    match settings.picking_mode {
-                        SpritePickingMode::AlphaThreshold(cutoff) => {
-                            let Some(image) = images.get(&sprite.image) else {
-                                // [`Sprite::from_color`] returns a defaulted handle.
-                                // This handle doesn't return a valid image, so returning false here would make picking "color sprites" impossible
-                                break 'valid_pixel true;
-                            };
-                            // grab pixel and check alpha
-                            let Ok(color) = image.get_color_at(
-                                cursor_pixel_space.x as u32,
-                                cursor_pixel_space.y as u32,
-                            ) else {
-                                // We don't know how to interpret the pixel.
-                                break 'valid_pixel false;
-                            };
-                            // Check the alpha is above the cutoff.
-                            color.alpha() > cutoff
+                    let cursor_in_valid_pixels_of_sprite = 'valid_pixel: {
+                        match settings.picking_mode {
+                            SpritePickingMode::AlphaThreshold(cutoff) => {
+                                let Some(image) = images.get(&sprite.image) else {
+                                    // [`Sprite::from_color`] returns a defaulted handle.
+                                    // This handle doesn't return a valid image, so returning false here would make picking "color sprites" impossible
+                                    break 'valid_pixel true;
+                                };
+                                // grab pixel and check alpha
+                                let Ok(color) = image.get_color_at(
+                                    cursor_pixel_space.x as u32,
+                                    cursor_pixel_space.y as u32,
+                                ) else {
+                                    // We don't know how to interpret the pixel.
+                                    break 'valid_pixel false;
+                                };
+                                // Check the alpha is above the cutoff.
+                                color.alpha() > cutoff
+                            }
+                            SpritePickingMode::BoundingBox => true,
                         }
-                        SpritePickingMode::BoundingBox => true,
-                    }
-                };
+                    };
 
-                blocked = cursor_in_valid_pixels_of_sprite && pickable.should_block_lower;
+                    blocked = cursor_in_valid_pixels_of_sprite && pickable.should_block_lower;
 
-                cursor_in_valid_pixels_of_sprite.then(|| {
-                    let hit_pos_world =
-                        sprite_transform.transform_point(cursor_pos_sprite.extend(0.0));
-                    // Transform point from world to camera space to get the Z distance
-                    let hit_pos_cam = cam_transform
-                        .affine()
-                        .inverse()
-                        .transform_point3(hit_pos_world);
-                    // HitData requires a depth as calculated from the camera's near clipping plane
-                    let depth = -cam_ortho.near - hit_pos_cam.z;
-                    (
-                        entity,
-                        HitData::new(
-                            cam_entity,
-                            depth,
-                            Some(hit_pos_world),
-                            Some(*sprite_transform.back()),
-                        ),
-                    )
-                })
-            })
+                    cursor_in_valid_pixels_of_sprite.then(|| {
+                        let hit_pos_world =
+                            sprite_transform.transform_point(cursor_pos_sprite.extend(0.0));
+                        // Transform point from world to camera space to get the Z distance
+                        let hit_pos_cam = cam_transform
+                            .affine()
+                            .inverse()
+                            .transform_point3(hit_pos_world);
+                        // HitData requires a depth as calculated from the camera's near clipping plane
+                        let depth = -cam_ortho.near - hit_pos_cam.z;
+                        (
+                            entity,
+                            HitData::new(
+                                cam_entity,
+                                depth,
+                                Some(hit_pos_world),
+                                Some(*sprite_transform.back()),
+                            ),
+                        )
+                    })
+                },
+            )
             .collect();
 
-        let order = camera.order as f32;
-        pointer_hits_writer.write(PointerHits::new(*pointer, picks, order));
-    }
+        Some((ray_id.pointer, picks, camera.order))
+    });
+
+    pick_sets.for_each(|(pointer, picks, order)| {
+        pointer_hits_writer.write(PointerHits::new(pointer, picks, order as f32));
+    });
 }