From 488ec0d3177b05ef11ee96b5b14cde4dae39a72c Mon Sep 17 00:00:00 2001 From: Ian Douglas Scott Date: Mon, 24 Oct 2022 17:06:02 -0700 Subject: [PATCH] WIP glow image rendering support https://github.com/iced-rs/iced/issues/674 This works, but duplicates code from `iced_wgpu` that should ideally be shared, and the cache never evicts. The next step here is to work on an API design to move some of the image loading and caching logic from the backend to `iced_graphics` (or elsewhere?). --- Cargo.toml | 2 +- glow/Cargo.toml | 21 +- glow/src/backend.rs | 22 +- glow/src/image.rs | 367 ++++++++++++++++++++++++++++++ glow/src/lib.rs | 1 + glow/src/shader/common/image.frag | 22 ++ glow/src/shader/common/image.vert | 9 + 7 files changed, 440 insertions(+), 4 deletions(-) create mode 100644 glow/src/image.rs create mode 100644 glow/src/shader/common/image.frag create mode 100644 glow/src/shader/common/image.vert diff --git a/Cargo.toml b/Cargo.toml index 9c6a435a99..3ad7c69843 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ resolver = "2" [features] default = ["wgpu"] # Enables the `Image` widget -image = ["iced_wgpu/image", "image_rs"] +image = ["iced_wgpu/image", "iced_glow/image", "image_rs"] # Enables the `Svg` widget svg = ["iced_wgpu/svg"] # Enables the `Canvas` widget diff --git a/glow/Cargo.toml b/glow/Cargo.toml index 18215e9b6f..63171eec7a 100644 --- a/glow/Cargo.toml +++ b/glow/Cargo.toml @@ -8,11 +8,22 @@ license = "MIT AND OFL-1.1" repository = "https://github.com/iced-rs/iced" [features] +image = ["png", "jpeg", "jpeg_rayon", "gif", "webp", "bmp"] +png = ["image_rs/png"] +jpeg = ["image_rs/jpeg"] +jpeg_rayon = ["image_rs/jpeg_rayon"] +gif = ["image_rs/gif"] +webp = ["image_rs/webp"] +pnm = ["image_rs/pnm"] +ico = ["image_rs/ico"] +bmp = ["image_rs/bmp"] +hdr = ["image_rs/hdr"] +dds = ["image_rs/dds"] +farbfeld = ["image_rs/farbfeld"] canvas = ["iced_graphics/canvas"] qr_code = ["iced_graphics/qr_code"] default_system_font = ["iced_graphics/font-source"] # Not supported yet! -image = [] svg = [] [dependencies] @@ -22,6 +33,8 @@ glyph_brush = "0.7" euclid = "0.22" bytemuck = "1.4" log = "0.4" +kamadak-exif = "0.5" +bitflags = "1.2" [dependencies.iced_native] version = "0.5" @@ -32,6 +45,12 @@ version = "0.3" path = "../graphics" features = ["font-fallback", "font-icons", "opengl"] +[dependencies.image_rs] +version = "0.23" +package = "image" +default-features = false +optional = true + [package.metadata.docs.rs] rustdoc-args = ["--cfg", "docsrs"] all-features = true diff --git a/glow/src/backend.rs b/glow/src/backend.rs index 78d4229e58..6841a1147b 100644 --- a/glow/src/backend.rs +++ b/glow/src/backend.rs @@ -1,3 +1,4 @@ +use crate::image; use crate::program; use crate::quad; use crate::text; @@ -16,6 +17,7 @@ use iced_native::{Font, Size}; /// [`iced`]: https://github.com/iced-rs/iced #[derive(Debug)] pub struct Backend { + image_pipeline: image::Pipeline, quad_pipeline: quad::Pipeline, text_pipeline: text::Pipeline, triangle_pipeline: triangle::Pipeline, @@ -33,10 +35,12 @@ impl Backend { let shader_version = program::Version::new(gl); + let image_pipeline = image::Pipeline::new(gl, &shader_version); let quad_pipeline = quad::Pipeline::new(gl, &shader_version); let triangle_pipeline = triangle::Pipeline::new(gl, &shader_version); Self { + image_pipeline, quad_pipeline, text_pipeline, triangle_pipeline, @@ -113,6 +117,20 @@ impl Backend { ); } + #[cfg(feature = "image")] + if !layer.images.is_empty() { + let scaled = transformation + * Transformation::scale(scale_factor, scale_factor); + + self.image_pipeline.draw( + gl, + target_height, + scaled, + scale_factor, + &layer.images, + ); + } + if !layer.text.is_empty() { for text in layer.text.iter() { // Target physical coordinates directly to avoid blurry text @@ -239,8 +257,8 @@ impl backend::Text for Backend { #[cfg(feature = "image")] impl backend::Image for Backend { - fn dimensions(&self, _handle: &iced_native::image::Handle) -> (u32, u32) { - (50, 50) + fn dimensions(&self, handle: &iced_native::image::Handle) -> (u32, u32) { + self.image_pipeline.dimensions(handle) } } diff --git a/glow/src/image.rs b/glow/src/image.rs new file mode 100644 index 0000000000..47be709437 --- /dev/null +++ b/glow/src/image.rs @@ -0,0 +1,367 @@ +use crate::program::{self, Shader}; +use crate::Transformation; +use glow::HasContext; +use iced_graphics::layer; +use iced_native::image; +use iced_native::Rectangle; +use std::cell::RefCell; +use std::collections::HashMap; +use std::marker::PhantomData; + +use bitflags::bitflags; + +pub use iced_graphics::triangle::{Mesh2D, Vertex2D}; + +#[derive(Debug)] +pub(crate) struct Pipeline { + program: ::Program, + vertex_array: ::VertexArray, + vertex_buffer: ::Buffer, + transform_location: ::UniformLocation, + cache: RefCell, + textures: HashMap::Texture>, +} + +impl Pipeline { + pub fn new( + gl: &glow::Context, + shader_version: &program::Version, + ) -> Pipeline { + let program = unsafe { + let vertex_shader = Shader::vertex( + gl, + shader_version, + include_str!("shader/common/image.vert"), + ); + let fragment_shader = Shader::fragment( + gl, + shader_version, + include_str!("shader/common/image.frag"), + ); + + program::create( + gl, + &[vertex_shader, fragment_shader], + &[(0, "i_Position")], + ) + }; + + let transform_location = + unsafe { gl.get_uniform_location(program, "u_Transform") } + .expect("Get transform location"); + + unsafe { + gl.use_program(Some(program)); + + let transform: [f32; 16] = Transformation::identity().into(); + gl.uniform_matrix_4_f32_slice( + Some(&transform_location), + false, + &transform, + ); + + gl.use_program(None); + } + + let vertex_buffer = + unsafe { gl.create_buffer().expect("Create vertex buffer") }; + let vertex_array = + unsafe { gl.create_vertex_array().expect("Create vertex array") }; + + unsafe { + gl.bind_vertex_array(Some(vertex_array)); + gl.bind_buffer(glow::ARRAY_BUFFER, Some(vertex_buffer)); + + let vertices = &[0u8, 0, 1, 0, 0, 1, 1, 1]; + gl.buffer_data_size( + glow::ARRAY_BUFFER, + vertices.len() as i32, + glow::STATIC_DRAW, + ); + gl.buffer_sub_data_u8_slice( + glow::ARRAY_BUFFER, + 0, + bytemuck::cast_slice(vertices), + ); + + gl.enable_vertex_attrib_array(0); + gl.vertex_attrib_pointer_f32( + 0, + 2, + glow::UNSIGNED_BYTE, + false, + 0, + 0, + ); + + gl.bind_buffer(glow::ARRAY_BUFFER, None); + gl.bind_vertex_array(None); + } + + Pipeline { + program, + vertex_array, + vertex_buffer, + transform_location, + cache: Default::default(), + textures: HashMap::new(), + } + } + + #[cfg(feature = "image")] + pub fn dimensions(&self, handle: &image::Handle) -> (u32, u32) { + match self.cache.borrow_mut().load(handle) { + Some(image) => image.dimensions(), + None => (1, 1), + } + } + + pub fn draw( + &mut self, + gl: &glow::Context, + target_height: u32, + transformation: Transformation, + scale_factor: f32, + images: &[layer::Image], + ) { + unsafe { + gl.use_program(Some(self.program)); + gl.bind_vertex_array(Some(self.vertex_array)); + gl.bind_buffer(glow::ARRAY_BUFFER, Some(self.vertex_buffer)); + } + + for image in images { + match &image { + #[cfg(feature = "image")] + layer::Image::Raster { handle, bounds } => { + if !self.textures.contains_key(&handle.id()) { + if let Some(image) = self.cache.get_mut().load(&handle) + { + let (width, height) = image.dimensions(); + unsafe { + let texture = gl + .create_texture() + .expect("create texture"); + gl.bind_texture( + glow::TEXTURE_2D, + Some(texture), + ); + gl.tex_image_2d( + glow::TEXTURE_2D, + 0, + glow::RGBA as i32, + width as i32, + height as i32, + 0, + glow::BGRA, + glow::UNSIGNED_BYTE, + Some(image), + ); + gl.tex_parameter_i32( + glow::TEXTURE_2D, + glow::TEXTURE_WRAP_S, + glow::CLAMP_TO_EDGE as _, + ); + gl.tex_parameter_i32( + glow::TEXTURE_2D, + glow::TEXTURE_WRAP_T, + glow::CLAMP_TO_EDGE as _, + ); + gl.tex_parameter_i32( + glow::TEXTURE_2D, + glow::TEXTURE_MIN_FILTER, + glow::LINEAR as _, + ); + gl.tex_parameter_i32( + glow::TEXTURE_2D, + glow::TEXTURE_MAG_FILTER, + glow::LINEAR as _, + ); + gl.bind_texture( + glow::TEXTURE_2D, + None, + ); + let _ = self.textures.insert(handle.id(), texture); + } + } + } + + if let Some(texture) = self.textures.get(&handle.id()) { + unsafe { gl.bind_texture(glow::TEXTURE_2D, Some(*texture)) }; + } else { + continue; + } + + unsafe { + let translate = + Transformation::translate(bounds.x, bounds.y); + let scale = + Transformation::scale(bounds.width, bounds.height); + let transformation = transformation * translate * scale; + let matrix: [f32; 16] = transformation.into(); + gl.uniform_matrix_4_f32_slice( + Some(&self.transform_location), + false, + &matrix, + ); + + gl.draw_arrays(glow::TRIANGLE_STRIP, 0, 4); + + gl.bind_texture( + glow::TEXTURE_2D, + None, + ); + } + } + #[cfg(not(feature = "image"))] + layer::Image::Raster { .. } => {} + + layer::Image::Vector { .. } => {} + } + } + + unsafe { + gl.bind_buffer(glow::ARRAY_BUFFER, None); + gl.bind_vertex_array(None); + gl.use_program(None); + } + } +} + +#[derive(Debug, Default)] +pub struct Cache { + map: HashMap, Vec>>, +} + +impl Cache { + fn load( + &mut self, + handle: &image::Handle, + ) -> Option<&mut ::image_rs::ImageBuffer<::image_rs::Bgra, Vec>> + { + if self.map.contains_key(&handle.id()) { + return Some(self.map.get_mut(&handle.id()).unwrap()); + } + + let image = match handle.data() { + image::Data::Path(path) => { + if let Ok(image) = image_rs::open(path) { + let operation = std::fs::File::open(path) + .ok() + .map(std::io::BufReader::new) + .and_then(|mut reader| { + Operation::from_exif(&mut reader).ok() + }) + .unwrap_or_else(Operation::empty); + + Some(operation.perform(image.to_bgra8())) + } else { + None + } + } + image::Data::Bytes(bytes) => { + if let Ok(image) = image_rs::load_from_memory(bytes) { + let operation = + Operation::from_exif(&mut std::io::Cursor::new(bytes)) + .ok() + .unwrap_or_else(Operation::empty); + + Some(operation.perform(image.to_bgra8())) + } else { + None + } + } + image::Data::Pixels { + width, + height, + pixels, + } => { + if let Some(image) = image_rs::ImageBuffer::from_vec( + *width, + *height, + pixels.to_vec(), + ) { + Some(image) + } else { + None + } + } + }?; + + let _ = self.map.insert(handle.id(), image); + self.map.get_mut(&handle.id()) + } +} + +bitflags! { + struct Operation: u8 { + const FLIP_HORIZONTALLY = 0b001; + const ROTATE_180 = 0b010; + const FLIP_DIAGONALLY = 0b100; + } +} + +impl Operation { + // Meaning of the returned value is described e.g. at: + // https://magnushoff.com/articles/jpeg-orientation/ + fn from_exif(reader: &mut R) -> Result + where + R: std::io::BufRead + std::io::Seek, + { + let exif = exif::Reader::new().read_from_container(reader)?; + + Ok(exif + .get_field(exif::Tag::Orientation, exif::In::PRIMARY) + .and_then(|field| field.value.get_uint(0)) + .and_then(|value| u8::try_from(value).ok()) + .and_then(|value| Self::from_bits(value.saturating_sub(1))) + .unwrap_or_else(Self::empty)) + } + + fn perform

( + self, + image: image_rs::ImageBuffer>, + ) -> image_rs::ImageBuffer> + where + P: image_rs::Pixel + 'static, + { + use image_rs::imageops; + + let mut image = if self.contains(Self::FLIP_DIAGONALLY) { + flip_diagonally(image) + } else { + image + }; + + if self.contains(Self::ROTATE_180) { + imageops::rotate180_in_place(&mut image); + } + + if self.contains(Self::FLIP_HORIZONTALLY) { + imageops::flip_horizontal_in_place(&mut image); + } + + image + } +} + +fn flip_diagonally( + image: I, +) -> image_rs::ImageBuffer::Subpixel>> +where + I: image_rs::GenericImage, + I::Pixel: 'static, +{ + let (width, height) = image.dimensions(); + let mut out = image_rs::ImageBuffer::new(height, width); + + for x in 0..width { + for y in 0..height { + let p = image.get_pixel(x, y); + + out.put_pixel(y, x, p); + } + } + + out +} diff --git a/glow/src/lib.rs b/glow/src/lib.rs index de9c000214..0fba452df0 100644 --- a/glow/src/lib.rs +++ b/glow/src/lib.rs @@ -24,6 +24,7 @@ pub use glow; mod backend; +mod image; mod program; mod quad; mod text; diff --git a/glow/src/shader/common/image.frag b/glow/src/shader/common/image.frag new file mode 100644 index 0000000000..5e05abdf2c --- /dev/null +++ b/glow/src/shader/common/image.frag @@ -0,0 +1,22 @@ +#ifdef GL_ES +#ifdef GL_FRAGMENT_PRECISION_HIGH +precision highp float; +#else +precision mediump float; +#endif +#endif + +uniform sampler2D tex; +in vec2 tex_pos; + +#ifdef HIGHER_THAN_300 +out vec4 fragColor; +#define gl_FragColor fragColor +#endif +#ifdef GL_ES +#define texture texture2D +#endif + +void main() { + gl_FragColor = texture(tex, tex_pos); +} diff --git a/glow/src/shader/common/image.vert b/glow/src/shader/common/image.vert new file mode 100644 index 0000000000..93e541f2b3 --- /dev/null +++ b/glow/src/shader/common/image.vert @@ -0,0 +1,9 @@ +uniform mat4 u_Transform; + +in vec2 i_Position; +out vec2 tex_pos; + +void main() { + gl_Position = u_Transform * vec4(i_Position, 0.0, 1.0); + tex_pos = i_Position; +}