1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 use api::{DirtyRect, ExternalImageType, ImageFormat, ImageBufferKind}; 6 use api::{DebugFlags, ImageDescriptor}; 7 use api::units::*; 8 #[cfg(test)] 9 use api::{DocumentId, IdNamespace}; 10 use crate::device::{TextureFilter, TextureFormatPair}; 11 use crate::freelist::{FreeList, FreeListHandle, WeakFreeListHandle}; 12 use crate::gpu_cache::{GpuCache, GpuCacheHandle}; 13 use crate::gpu_types::{ImageSource, UvRectKind}; 14 use crate::internal_types::{ 15 CacheTextureId, Swizzle, SwizzleSettings, FrameStamp, FrameId, 16 TextureUpdateList, TextureUpdateSource, TextureSource, 17 TextureCacheAllocInfo, TextureCacheUpdate, TextureCacheCategory, 18 }; 19 use crate::lru_cache::LRUCache; 20 use crate::profiler::{self, TransactionProfile}; 21 use crate::resource_cache::{CacheItem, CachedImageData}; 22 use crate::texture_pack::{ 23 AllocatorList, AllocId, AtlasAllocatorList, ShelfAllocator, ShelfAllocatorOptions, 24 }; 25 use std::cell::Cell; 26 use std::mem; 27 use std::rc::Rc; 28 use euclid::size2; 29 use malloc_size_of::{MallocSizeOf, MallocSizeOfOps}; 30 31 /// Information about which shader will use the entry. 32 /// 33 /// For batching purposes, it's beneficial to group some items in their 34 /// own textures if we know that they are used by a specific shader. 35 #[derive(Copy, Clone, Debug, PartialEq, Eq)] 36 #[cfg_attr(feature = "capture", derive(Serialize))] 37 #[cfg_attr(feature = "replay", derive(Deserialize))] 38 pub enum TargetShader { 39 Default, 40 Text, 41 } 42 43 /// The size of each region in shared cache texture arrays. 44 pub const TEXTURE_REGION_DIMENSIONS: i32 = 512; 45 46 /// Items in the texture cache can either be standalone textures, 47 /// or a sub-rect inside the shared cache. 48 #[derive(Clone, Debug)] 49 #[cfg_attr(feature = "capture", derive(Serialize))] 50 #[cfg_attr(feature = "replay", derive(Deserialize))] 51 pub enum EntryDetails { 52 Standalone { 53 /// Number of bytes this entry allocates 54 size_in_bytes: usize, 55 }, 56 Cache { 57 /// Origin within the texture layer where this item exists. 58 origin: DeviceIntPoint, 59 /// ID of the allocation specific to its allocator. 60 alloc_id: AllocId, 61 /// The allocated size in bytes for this entry. 62 allocated_size_in_bytes: usize, 63 }, 64 } 65 66 impl EntryDetails { describe(&self) -> DeviceIntPoint67 fn describe(&self) -> DeviceIntPoint { 68 match *self { 69 EntryDetails::Standalone { .. } => DeviceIntPoint::zero(), 70 EntryDetails::Cache { origin, .. } => origin, 71 } 72 } 73 } 74 75 #[derive(Debug, PartialEq)] 76 #[cfg_attr(feature = "capture", derive(Serialize))] 77 #[cfg_attr(feature = "replay", derive(Deserialize))] 78 pub enum AutoCacheEntryMarker {} 79 80 #[derive(Debug, PartialEq)] 81 #[cfg_attr(feature = "capture", derive(Serialize))] 82 #[cfg_attr(feature = "replay", derive(Deserialize))] 83 pub enum ManualCacheEntryMarker {} 84 85 // Stores information related to a single entry in the texture 86 // cache. This is stored for each item whether it's in the shared 87 // cache or a standalone texture. 88 #[derive(Debug)] 89 #[cfg_attr(feature = "capture", derive(Serialize))] 90 #[cfg_attr(feature = "replay", derive(Deserialize))] 91 pub struct CacheEntry { 92 /// Size of the requested item, in device pixels. Does not include any 93 /// padding for alignment that the allocator may have added to this entry's 94 /// allocation. 95 pub size: DeviceIntSize, 96 /// Details specific to standalone or shared items. 97 pub details: EntryDetails, 98 /// Arbitrary user data associated with this item. 99 pub user_data: [f32; 4], 100 /// The last frame this item was requested for rendering. 101 // TODO(gw): This stamp is only used for picture cache tiles, and some checks 102 // in the glyph cache eviction code. We could probably remove it 103 // entirely in future (or move to PictureCacheEntry). 104 pub last_access: FrameStamp, 105 /// Handle to the resource rect in the GPU cache. 106 pub uv_rect_handle: GpuCacheHandle, 107 /// Image format of the data that the entry expects. 108 pub input_format: ImageFormat, 109 pub filter: TextureFilter, 110 pub swizzle: Swizzle, 111 /// The actual device texture ID this is part of. 112 pub texture_id: CacheTextureId, 113 /// Optional notice when the entry is evicted from the cache. 114 pub eviction_notice: Option<EvictionNotice>, 115 /// The type of UV rect this entry specifies. 116 pub uv_rect_kind: UvRectKind, 117 118 pub shader: TargetShader, 119 } 120 121 malloc_size_of::malloc_size_of_is_0!( 122 CacheEntry, 123 AutoCacheEntryMarker, ManualCacheEntryMarker 124 ); 125 126 impl CacheEntry { 127 // Create a new entry for a standalone texture. new_standalone( texture_id: CacheTextureId, last_access: FrameStamp, params: &CacheAllocParams, swizzle: Swizzle, size_in_bytes: usize, ) -> Self128 fn new_standalone( 129 texture_id: CacheTextureId, 130 last_access: FrameStamp, 131 params: &CacheAllocParams, 132 swizzle: Swizzle, 133 size_in_bytes: usize, 134 ) -> Self { 135 CacheEntry { 136 size: params.descriptor.size, 137 user_data: params.user_data, 138 last_access, 139 details: EntryDetails::Standalone { 140 size_in_bytes, 141 }, 142 texture_id, 143 input_format: params.descriptor.format, 144 filter: params.filter, 145 swizzle, 146 uv_rect_handle: GpuCacheHandle::new(), 147 eviction_notice: None, 148 uv_rect_kind: params.uv_rect_kind, 149 shader: TargetShader::Default, 150 } 151 } 152 153 // Update the GPU cache for this texture cache entry. 154 // This ensures that the UV rect, and texture layer index 155 // are up to date in the GPU cache for vertex shaders 156 // to fetch from. update_gpu_cache(&mut self, gpu_cache: &mut GpuCache)157 fn update_gpu_cache(&mut self, gpu_cache: &mut GpuCache) { 158 if let Some(mut request) = gpu_cache.request(&mut self.uv_rect_handle) { 159 let origin = self.details.describe(); 160 let image_source = ImageSource { 161 p0: origin.to_f32(), 162 p1: (origin + self.size).to_f32(), 163 user_data: self.user_data, 164 uv_rect_kind: self.uv_rect_kind, 165 }; 166 image_source.write_gpu_blocks(&mut request); 167 } 168 } 169 evict(&self)170 fn evict(&self) { 171 if let Some(eviction_notice) = self.eviction_notice.as_ref() { 172 eviction_notice.notify(); 173 } 174 } 175 alternative_input_format(&self) -> ImageFormat176 fn alternative_input_format(&self) -> ImageFormat { 177 match self.input_format { 178 ImageFormat::RGBA8 => ImageFormat::BGRA8, 179 ImageFormat::BGRA8 => ImageFormat::RGBA8, 180 other => other, 181 } 182 } 183 } 184 185 186 /// A texture cache handle is a weak reference to a cache entry. 187 /// 188 /// If the handle has not been inserted into the cache yet, or if the entry was 189 /// previously inserted and then evicted, lookup of the handle will fail, and 190 /// the cache handle needs to re-upload this item to the texture cache (see 191 /// request() below). 192 193 #[derive(MallocSizeOf,Clone,PartialEq,Debug)] 194 #[cfg_attr(feature = "capture", derive(Serialize))] 195 #[cfg_attr(feature = "replay", derive(Deserialize))] 196 pub enum TextureCacheHandle { 197 /// A fresh handle. 198 Empty, 199 200 /// A handle for an entry with automatic eviction. 201 Auto(WeakFreeListHandle<AutoCacheEntryMarker>), 202 203 /// A handle for an entry with manual eviction. 204 Manual(WeakFreeListHandle<ManualCacheEntryMarker>) 205 } 206 207 impl TextureCacheHandle { invalid() -> Self208 pub fn invalid() -> Self { 209 TextureCacheHandle::Empty 210 } 211 } 212 213 /// Describes the eviction policy for a given entry in the texture cache. 214 #[derive(Copy, Clone, Debug, PartialEq, Eq)] 215 #[cfg_attr(feature = "capture", derive(Serialize))] 216 #[cfg_attr(feature = "replay", derive(Deserialize))] 217 pub enum Eviction { 218 /// The entry will be evicted under the normal rules (which differ between 219 /// standalone and shared entries). 220 Auto, 221 /// The entry will not be evicted until the policy is explicitly set to a 222 /// different value. 223 Manual, 224 } 225 226 // An eviction notice is a shared condition useful for detecting 227 // when a TextureCacheHandle gets evicted from the TextureCache. 228 // It is optionally installed to the TextureCache when an update() 229 // is scheduled. A single notice may be shared among any number of 230 // TextureCacheHandle updates. The notice may then be subsequently 231 // checked to see if any of the updates using it have been evicted. 232 #[derive(Clone, Debug, Default)] 233 #[cfg_attr(feature = "capture", derive(Serialize))] 234 #[cfg_attr(feature = "replay", derive(Deserialize))] 235 pub struct EvictionNotice { 236 evicted: Rc<Cell<bool>>, 237 } 238 239 impl EvictionNotice { notify(&self)240 fn notify(&self) { 241 self.evicted.set(true); 242 } 243 check(&self) -> bool244 pub fn check(&self) -> bool { 245 if self.evicted.get() { 246 self.evicted.set(false); 247 true 248 } else { 249 false 250 } 251 } 252 } 253 254 /// The different budget types for the texture cache. Each type has its own 255 /// memory budget. Once the budget is exceeded, entries with automatic eviction 256 /// are evicted. Entries with manual eviction share the same budget but are not 257 /// evicted once the budget is exceeded. 258 /// Keeping separate budgets ensures that we don't evict entries from unrelated 259 /// textures if one texture gets full. 260 #[derive(Copy, Clone, Debug, PartialEq, Eq)] 261 #[repr(u8)] 262 #[cfg_attr(feature = "capture", derive(Serialize))] 263 #[cfg_attr(feature = "replay", derive(Deserialize))] 264 enum BudgetType { 265 SharedColor8Linear, 266 SharedColor8Nearest, 267 SharedColor8Glyphs, 268 SharedAlpha8, 269 SharedAlpha8Glyphs, 270 SharedAlpha16, 271 Standalone, 272 } 273 274 impl BudgetType { 275 pub const COUNT: usize = 7; 276 277 pub const VALUES: [BudgetType; BudgetType::COUNT] = [ 278 BudgetType::SharedColor8Linear, 279 BudgetType::SharedColor8Nearest, 280 BudgetType::SharedColor8Glyphs, 281 BudgetType::SharedAlpha8, 282 BudgetType::SharedAlpha8Glyphs, 283 BudgetType::SharedAlpha16, 284 BudgetType::Standalone, 285 ]; 286 287 pub const PRESSURE_COUNTERS: [usize; BudgetType::COUNT] = [ 288 profiler::ATLAS_COLOR8_LINEAR_PRESSURE, 289 profiler::ATLAS_COLOR8_NEAREST_PRESSURE, 290 profiler::ATLAS_COLOR8_GLYPHS_PRESSURE, 291 profiler::ATLAS_ALPHA8_PRESSURE, 292 profiler::ATLAS_ALPHA8_GLYPHS_PRESSURE, 293 profiler::ATLAS_ALPHA16_PRESSURE, 294 profiler::ATLAS_STANDALONE_PRESSURE, 295 ]; 296 iter() -> impl Iterator<Item = BudgetType>297 pub fn iter() -> impl Iterator<Item = BudgetType> { 298 BudgetType::VALUES.iter().cloned() 299 } 300 } 301 302 /// A set of lazily allocated, fixed size, texture arrays for each format the 303 /// texture cache supports. 304 #[cfg_attr(feature = "capture", derive(Serialize))] 305 #[cfg_attr(feature = "replay", derive(Deserialize))] 306 struct SharedTextures { 307 color8_nearest: AllocatorList<ShelfAllocator, TextureParameters>, 308 alpha8_linear: AllocatorList<ShelfAllocator, TextureParameters>, 309 alpha8_glyphs: AllocatorList<ShelfAllocator, TextureParameters>, 310 alpha16_linear: AllocatorList<ShelfAllocator, TextureParameters>, 311 color8_linear: AllocatorList<ShelfAllocator, TextureParameters>, 312 color8_glyphs: AllocatorList<ShelfAllocator, TextureParameters>, 313 bytes_per_texture_of_type: [i32 ; BudgetType::COUNT], 314 next_compaction_idx: usize, 315 } 316 317 impl SharedTextures { 318 /// Mints a new set of shared textures. new(color_formats: TextureFormatPair<ImageFormat>, config: &TextureCacheConfig) -> Self319 fn new(color_formats: TextureFormatPair<ImageFormat>, config: &TextureCacheConfig) -> Self { 320 let mut bytes_per_texture_of_type = [0 ; BudgetType::COUNT]; 321 322 // Used primarily for cached shadow masks. There can be lots of 323 // these on some pages like francine, but most pages don't use it 324 // much. 325 // Most content tends to fit into two 512x512 textures. We are 326 // conservatively using 1024x1024 to fit everything in a single 327 // texture and avoid breaking batches, but it's worth checking 328 // whether it would actually lead to a lot of batch breaks in 329 // practice. 330 let alpha8_linear = AllocatorList::new( 331 config.alpha8_texture_size, 332 ShelfAllocatorOptions { 333 num_columns: 1, 334 alignment: size2(8, 8), 335 .. ShelfAllocatorOptions::default() 336 }, 337 TextureParameters { 338 formats: TextureFormatPair::from(ImageFormat::R8), 339 filter: TextureFilter::Linear, 340 }, 341 ); 342 bytes_per_texture_of_type[BudgetType::SharedAlpha8 as usize] = 343 config.alpha8_texture_size * config.alpha8_texture_size; 344 345 // The cache for alpha glyphs (separate to help with batching). 346 let alpha8_glyphs = AllocatorList::new( 347 config.alpha8_glyph_texture_size, 348 ShelfAllocatorOptions { 349 num_columns: if config.alpha8_glyph_texture_size >= 1024 { 2 } else { 1 }, 350 alignment: size2(4, 8), 351 .. ShelfAllocatorOptions::default() 352 }, 353 TextureParameters { 354 formats: TextureFormatPair::from(ImageFormat::R8), 355 filter: TextureFilter::Linear, 356 }, 357 ); 358 bytes_per_texture_of_type[BudgetType::SharedAlpha8Glyphs as usize] = 359 config.alpha8_glyph_texture_size * config.alpha8_glyph_texture_size; 360 361 // Used for experimental hdr yuv texture support, but not used in 362 // production Firefox. 363 let alpha16_linear = AllocatorList::new( 364 config.alpha16_texture_size, 365 ShelfAllocatorOptions { 366 num_columns: if config.alpha16_texture_size >= 1024 { 2 } else { 1 }, 367 alignment: size2(8, 8), 368 .. ShelfAllocatorOptions::default() 369 }, 370 TextureParameters { 371 formats: TextureFormatPair::from(ImageFormat::R16), 372 filter: TextureFilter::Linear, 373 }, 374 ); 375 bytes_per_texture_of_type[BudgetType::SharedAlpha16 as usize] = 376 ImageFormat::R16.bytes_per_pixel() * 377 config.alpha16_texture_size * config.alpha16_texture_size; 378 379 // The primary cache for images, etc. 380 let color8_linear = AllocatorList::new( 381 config.color8_linear_texture_size, 382 ShelfAllocatorOptions { 383 num_columns: if config.color8_linear_texture_size >= 1024 { 2 } else { 1 }, 384 alignment: size2(16, 16), 385 .. ShelfAllocatorOptions::default() 386 }, 387 TextureParameters { 388 formats: color_formats.clone(), 389 filter: TextureFilter::Linear, 390 }, 391 ); 392 bytes_per_texture_of_type[BudgetType::SharedColor8Linear as usize] = 393 color_formats.internal.bytes_per_pixel() * 394 config.color8_linear_texture_size * config.color8_linear_texture_size; 395 396 // The cache for subpixel-AA and bitmap glyphs (separate to help with batching). 397 let color8_glyphs = AllocatorList::new( 398 config.color8_glyph_texture_size, 399 ShelfAllocatorOptions { 400 num_columns: if config.color8_glyph_texture_size >= 1024 { 2 } else { 1 }, 401 alignment: size2(4, 8), 402 .. ShelfAllocatorOptions::default() 403 }, 404 TextureParameters { 405 formats: color_formats.clone(), 406 filter: TextureFilter::Linear, 407 }, 408 ); 409 bytes_per_texture_of_type[BudgetType::SharedColor8Glyphs as usize] = 410 color_formats.internal.bytes_per_pixel() * 411 config.color8_glyph_texture_size * config.color8_glyph_texture_size; 412 413 // Used for image-rendering: crisp. This is mostly favicons, which 414 // are small. Some other images use it too, but those tend to be 415 // larger than 512x512 and thus don't use the shared cache anyway. 416 let color8_nearest = AllocatorList::new( 417 config.color8_nearest_texture_size, 418 ShelfAllocatorOptions::default(), 419 TextureParameters { 420 formats: color_formats.clone(), 421 filter: TextureFilter::Nearest, 422 } 423 ); 424 bytes_per_texture_of_type[BudgetType::SharedColor8Nearest as usize] = 425 color_formats.internal.bytes_per_pixel() * 426 config.color8_nearest_texture_size * config.color8_nearest_texture_size; 427 428 Self { 429 alpha8_linear, 430 alpha8_glyphs, 431 alpha16_linear, 432 color8_linear, 433 color8_glyphs, 434 color8_nearest, 435 bytes_per_texture_of_type, 436 next_compaction_idx: 0, 437 } 438 } 439 440 /// Clears each texture in the set, with the given set of pending updates. clear(&mut self, updates: &mut TextureUpdateList)441 fn clear(&mut self, updates: &mut TextureUpdateList) { 442 let texture_dealloc_cb = &mut |texture_id| { 443 updates.push_free(texture_id); 444 }; 445 446 self.alpha8_linear.clear(texture_dealloc_cb); 447 self.alpha8_glyphs.clear(texture_dealloc_cb); 448 self.alpha16_linear.clear(texture_dealloc_cb); 449 self.color8_linear.clear(texture_dealloc_cb); 450 self.color8_nearest.clear(texture_dealloc_cb); 451 self.color8_glyphs.clear(texture_dealloc_cb); 452 } 453 454 /// Returns a mutable borrow for the shared texture array matching the parameters. select( &mut self, external_format: ImageFormat, filter: TextureFilter, shader: TargetShader, ) -> (&mut dyn AtlasAllocatorList<TextureParameters>, BudgetType)455 fn select( 456 &mut self, external_format: ImageFormat, filter: TextureFilter, shader: TargetShader, 457 ) -> (&mut dyn AtlasAllocatorList<TextureParameters>, BudgetType) { 458 match external_format { 459 ImageFormat::R8 => { 460 assert_eq!(filter, TextureFilter::Linear); 461 match shader { 462 TargetShader::Text => { 463 (&mut self.alpha8_glyphs, BudgetType::SharedAlpha8Glyphs) 464 }, 465 _ => (&mut self.alpha8_linear, BudgetType::SharedAlpha8), 466 } 467 } 468 ImageFormat::R16 => { 469 assert_eq!(filter, TextureFilter::Linear); 470 (&mut self.alpha16_linear, BudgetType::SharedAlpha16) 471 } 472 ImageFormat::RGBA8 | 473 ImageFormat::BGRA8 => { 474 match (filter, shader) { 475 (TextureFilter::Linear, TargetShader::Text) => { 476 (&mut self.color8_glyphs, BudgetType::SharedColor8Glyphs) 477 }, 478 (TextureFilter::Linear, _) => { 479 (&mut self.color8_linear, BudgetType::SharedColor8Linear) 480 }, 481 (TextureFilter::Nearest, _) => { 482 (&mut self.color8_nearest, BudgetType::SharedColor8Nearest) 483 }, 484 _ => panic!("Unexpected filter {:?}", filter), 485 } 486 } 487 _ => panic!("Unexpected format {:?}", external_format), 488 } 489 } 490 491 /// How many bytes a single texture of the given type takes up, for the 492 /// configured texture sizes. bytes_per_shared_texture(&self, budget_type: BudgetType) -> usize493 fn bytes_per_shared_texture(&self, budget_type: BudgetType) -> usize { 494 self.bytes_per_texture_of_type[budget_type as usize] as usize 495 } 496 has_multiple_textures(&self, budget_type: BudgetType) -> bool497 fn has_multiple_textures(&self, budget_type: BudgetType) -> bool { 498 match budget_type { 499 BudgetType::SharedColor8Linear => self.color8_linear.allocated_textures() > 1, 500 BudgetType::SharedColor8Nearest => self.color8_nearest.allocated_textures() > 1, 501 BudgetType::SharedColor8Glyphs => self.color8_glyphs.allocated_textures() > 1, 502 BudgetType::SharedAlpha8 => self.alpha8_linear.allocated_textures() > 1, 503 BudgetType::SharedAlpha8Glyphs => self.alpha8_glyphs.allocated_textures() > 1, 504 BudgetType::SharedAlpha16 => self.alpha16_linear.allocated_textures() > 1, 505 BudgetType::Standalone => false, 506 } 507 } 508 } 509 510 /// Container struct for the various parameters used in cache allocation. 511 struct CacheAllocParams { 512 descriptor: ImageDescriptor, 513 filter: TextureFilter, 514 user_data: [f32; 4], 515 uv_rect_kind: UvRectKind, 516 shader: TargetShader, 517 } 518 519 /// Startup parameters for the texture cache. 520 /// 521 /// Texture sizes must be at least 512. 522 #[derive(Clone)] 523 pub struct TextureCacheConfig { 524 pub color8_linear_texture_size: i32, 525 pub color8_nearest_texture_size: i32, 526 pub color8_glyph_texture_size: i32, 527 pub alpha8_texture_size: i32, 528 pub alpha8_glyph_texture_size: i32, 529 pub alpha16_texture_size: i32, 530 } 531 532 impl TextureCacheConfig { 533 pub const DEFAULT: Self = TextureCacheConfig { 534 color8_linear_texture_size: 2048, 535 color8_nearest_texture_size: 512, 536 color8_glyph_texture_size: 2048, 537 alpha8_texture_size: 1024, 538 alpha8_glyph_texture_size: 2048, 539 alpha16_texture_size: 512, 540 }; 541 } 542 543 /// General-purpose manager for images in GPU memory. This includes images, 544 /// rasterized glyphs, rasterized blobs, cached render tasks, etc. 545 /// 546 /// The texture cache is owned and managed by the RenderBackend thread, and 547 /// produces a series of commands to manipulate the textures on the Renderer 548 /// thread. These commands are executed before any rendering is performed for 549 /// a given frame. 550 /// 551 /// Entries in the texture cache are not guaranteed to live past the end of the 552 /// frame in which they are requested, and may be evicted. The API supports 553 /// querying whether an entry is still available. 554 /// 555 /// The TextureCache is different from the GpuCache in that the former stores 556 /// images, whereas the latter stores data and parameters for use in the shaders. 557 /// This means that the texture cache can be visualized, which is a good way to 558 /// understand how it works. Enabling gfx.webrender.debug.texture-cache shows a 559 /// live view of its contents in Firefox. 560 #[cfg_attr(feature = "capture", derive(Serialize))] 561 #[cfg_attr(feature = "replay", derive(Deserialize))] 562 pub struct TextureCache { 563 /// Set of texture arrays in different formats used for the shared cache. 564 shared_textures: SharedTextures, 565 566 /// Maximum texture size supported by hardware. 567 max_texture_size: i32, 568 569 /// Maximum texture size before it is considered preferable to break the 570 /// texture into tiles. 571 tiling_threshold: i32, 572 573 /// Settings on using texture unit swizzling. 574 swizzle: Option<SwizzleSettings>, 575 576 /// The current set of debug flags. 577 debug_flags: DebugFlags, 578 579 /// The next unused virtual texture ID. Monotonically increasing. 580 pub next_id: CacheTextureId, 581 582 /// A list of allocations and updates that need to be applied to the texture 583 /// cache in the rendering thread this frame. 584 #[cfg_attr(all(feature = "serde", any(feature = "capture", feature = "replay")), serde(skip))] 585 pub pending_updates: TextureUpdateList, 586 587 /// The current `FrameStamp`. Used for cache eviction policies. 588 now: FrameStamp, 589 590 /// Cache of texture cache handles with automatic lifetime management, evicted 591 /// in a least-recently-used order. 592 lru_cache: LRUCache<CacheEntry, AutoCacheEntryMarker>, 593 594 /// Cache of texture cache entries with manual liftime management. 595 manual_entries: FreeList<CacheEntry, ManualCacheEntryMarker>, 596 597 /// Strong handles for the manual_entries FreeList. 598 manual_handles: Vec<FreeListHandle<ManualCacheEntryMarker>>, 599 600 /// Memory usage of allocated entries in all of the shared or standalone 601 /// textures. Includes both manually and automatically evicted entries. 602 bytes_allocated: [usize ; BudgetType::COUNT], 603 } 604 605 impl TextureCache { 606 /// The maximum number of items that will be evicted per frame. This limit helps avoid jank 607 /// on frames where we want to evict a large number of items. Instead, we'd prefer to drop 608 /// the items incrementally over a number of frames, even if that means the total allocated 609 /// size of the cache is above the desired threshold for a small number of frames. 610 const MAX_EVICTIONS_PER_FRAME: usize = 32; 611 new( max_texture_size: i32, tiling_threshold: i32, color_formats: TextureFormatPair<ImageFormat>, swizzle: Option<SwizzleSettings>, config: &TextureCacheConfig, ) -> Self612 pub fn new( 613 max_texture_size: i32, 614 tiling_threshold: i32, 615 color_formats: TextureFormatPair<ImageFormat>, 616 swizzle: Option<SwizzleSettings>, 617 config: &TextureCacheConfig, 618 ) -> Self { 619 let pending_updates = TextureUpdateList::new(); 620 621 // Shared texture cache controls swizzling on a per-entry basis, assuming that 622 // the texture as a whole doesn't need to be swizzled (but only some entries do). 623 // It would be possible to support this, but not needed at the moment. 624 assert!(color_formats.internal != ImageFormat::BGRA8 || 625 swizzle.map_or(true, |s| s.bgra8_sampling_swizzle == Swizzle::default()) 626 ); 627 628 let next_texture_id = CacheTextureId(1); 629 630 TextureCache { 631 shared_textures: SharedTextures::new(color_formats, config), 632 max_texture_size, 633 tiling_threshold, 634 swizzle, 635 debug_flags: DebugFlags::empty(), 636 next_id: next_texture_id, 637 pending_updates, 638 now: FrameStamp::INVALID, 639 lru_cache: LRUCache::new(BudgetType::COUNT), 640 manual_entries: FreeList::new(), 641 manual_handles: Vec::new(), 642 bytes_allocated: [0 ; BudgetType::COUNT], 643 } 644 } 645 646 /// Creates a TextureCache and sets it up with a valid `FrameStamp`, which 647 /// is useful for avoiding panics when instantiating the `TextureCache` 648 /// directly from unit test code. 649 #[cfg(test)] new_for_testing( max_texture_size: i32, image_format: ImageFormat, ) -> Self650 pub fn new_for_testing( 651 max_texture_size: i32, 652 image_format: ImageFormat, 653 ) -> Self { 654 let mut cache = Self::new( 655 max_texture_size, 656 max_texture_size, 657 TextureFormatPair::from(image_format), 658 None, 659 &TextureCacheConfig::DEFAULT, 660 ); 661 let mut now = FrameStamp::first(DocumentId::new(IdNamespace(1), 1)); 662 now.advance(); 663 cache.begin_frame(now, &mut TransactionProfile::new()); 664 cache 665 } 666 set_debug_flags(&mut self, flags: DebugFlags)667 pub fn set_debug_flags(&mut self, flags: DebugFlags) { 668 self.debug_flags = flags; 669 } 670 671 /// Clear all entries in the texture cache. This is a fairly drastic 672 /// step that should only be called very rarely. clear_all(&mut self)673 pub fn clear_all(&mut self) { 674 // Evict all manual eviction handles 675 let manual_handles = mem::replace( 676 &mut self.manual_handles, 677 Vec::new(), 678 ); 679 for handle in manual_handles { 680 let entry = self.manual_entries.free(handle); 681 self.evict_impl(entry); 682 } 683 684 // Evict all auto (LRU) cache handles 685 for budget_type in BudgetType::iter() { 686 while let Some(entry) = self.lru_cache.pop_oldest(budget_type as u8) { 687 entry.evict(); 688 self.free(&entry); 689 } 690 } 691 692 // Free the picture and shared textures 693 self.shared_textures.clear(&mut self.pending_updates); 694 self.pending_updates.note_clear(); 695 } 696 697 /// Called at the beginning of each frame. begin_frame(&mut self, stamp: FrameStamp, profile: &mut TransactionProfile)698 pub fn begin_frame(&mut self, stamp: FrameStamp, profile: &mut TransactionProfile) { 699 debug_assert!(!self.now.is_valid()); 700 profile_scope!("begin_frame"); 701 self.now = stamp; 702 703 // Texture cache eviction is done at the start of the frame. This ensures that 704 // we won't evict items that have been requested on this frame. 705 // It also frees up space in the cache for items allocated later in the frame 706 // potentially reducing texture allocations and fragmentation. 707 self.evict_items_from_cache_if_required(profile); 708 } 709 end_frame(&mut self, profile: &mut TransactionProfile)710 pub fn end_frame(&mut self, profile: &mut TransactionProfile) { 711 debug_assert!(self.now.is_valid()); 712 713 let updates = &mut self.pending_updates; // To avoid referring to self in the closure. 714 let callback = &mut|texture_id| { updates.push_free(texture_id); }; 715 716 // Release of empty shared textures is done at the end of the frame. That way, if the 717 // eviction at the start of the frame frees up a texture, that is then subsequently 718 // used during the frame, we avoid doing a free/alloc for it. 719 self.shared_textures.alpha8_linear.release_empty_textures(callback); 720 self.shared_textures.alpha8_glyphs.release_empty_textures(callback); 721 self.shared_textures.alpha16_linear.release_empty_textures(callback); 722 self.shared_textures.color8_linear.release_empty_textures(callback); 723 self.shared_textures.color8_nearest.release_empty_textures(callback); 724 self.shared_textures.color8_glyphs.release_empty_textures(callback); 725 726 for budget in BudgetType::iter() { 727 let threshold = self.get_eviction_threshold(budget); 728 let pressure = self.bytes_allocated[budget as usize] as f32 / threshold as f32; 729 profile.set(BudgetType::PRESSURE_COUNTERS[budget as usize], pressure); 730 } 731 732 profile.set(profiler::ATLAS_A8_PIXELS, self.shared_textures.alpha8_linear.allocated_space()); 733 profile.set(profiler::ATLAS_A8_TEXTURES, self.shared_textures.alpha8_linear.allocated_textures()); 734 profile.set(profiler::ATLAS_A8_GLYPHS_PIXELS, self.shared_textures.alpha8_glyphs.allocated_space()); 735 profile.set(profiler::ATLAS_A8_GLYPHS_TEXTURES, self.shared_textures.alpha8_glyphs.allocated_textures()); 736 profile.set(profiler::ATLAS_A16_PIXELS, self.shared_textures.alpha16_linear.allocated_space()); 737 profile.set(profiler::ATLAS_A16_TEXTURES, self.shared_textures.alpha16_linear.allocated_textures()); 738 profile.set(profiler::ATLAS_RGBA8_LINEAR_PIXELS, self.shared_textures.color8_linear.allocated_space()); 739 profile.set(profiler::ATLAS_RGBA8_LINEAR_TEXTURES, self.shared_textures.color8_linear.allocated_textures()); 740 profile.set(profiler::ATLAS_RGBA8_NEAREST_PIXELS, self.shared_textures.color8_nearest.allocated_space()); 741 profile.set(profiler::ATLAS_RGBA8_NEAREST_TEXTURES, self.shared_textures.color8_nearest.allocated_textures()); 742 profile.set(profiler::ATLAS_RGBA8_GLYPHS_PIXELS, self.shared_textures.color8_glyphs.allocated_space()); 743 profile.set(profiler::ATLAS_RGBA8_GLYPHS_TEXTURES, self.shared_textures.color8_glyphs.allocated_textures()); 744 745 let shared_bytes = [ 746 BudgetType::SharedColor8Linear, 747 BudgetType::SharedColor8Nearest, 748 BudgetType::SharedColor8Glyphs, 749 BudgetType::SharedAlpha8, 750 BudgetType::SharedAlpha8Glyphs, 751 BudgetType::SharedAlpha16, 752 ].iter().map(|b| self.bytes_allocated[*b as usize]).sum(); 753 754 profile.set(profiler::ATLAS_ITEMS_MEM, profiler::bytes_to_mb(shared_bytes)); 755 756 self.now = FrameStamp::INVALID; 757 } 758 run_compaction(&mut self, gpu_cache: &mut GpuCache)759 pub fn run_compaction(&mut self, gpu_cache: &mut GpuCache) { 760 // Use the same order as BudgetType::VALUES so that we can index self.bytes_allocated 761 // with the same index. 762 let allocator_lists = [ 763 &mut self.shared_textures.color8_linear, 764 &mut self.shared_textures.color8_nearest, 765 &mut self.shared_textures.color8_glyphs, 766 &mut self.shared_textures.alpha8_linear, 767 &mut self.shared_textures.alpha8_glyphs, 768 &mut self.shared_textures.alpha16_linear, 769 ]; 770 771 // Pick a texture type on which to try to run the compaction logic this frame. 772 let idx = self.shared_textures.next_compaction_idx; 773 774 // Number of moved pixels after which we stop attempting to move more items for this frame. 775 // The constant is up for adjustment, the main goal is to avoid causing frame spikes on 776 // low end GPUs. 777 let area_threshold = 512*512; 778 779 let mut changes = Vec::new(); 780 allocator_lists[idx].try_compaction(area_threshold, &mut changes); 781 782 if changes.is_empty() { 783 // Nothing to do, we'll try another texture type next frame. 784 self.shared_textures.next_compaction_idx = (self.shared_textures.next_compaction_idx + 1) % allocator_lists.len(); 785 } 786 787 for change in changes { 788 let bpp = allocator_lists[idx].texture_parameters().formats.internal.bytes_per_pixel(); 789 790 // While the area of the image does not change, the area it occupies in the texture 791 // atlas may (in other words the number of wasted pixels can change), so we have 792 // to keep track of that. 793 let old_bytes = (change.old_rect.area() * bpp) as usize; 794 let new_bytes = (change.new_rect.area() * bpp) as usize; 795 self.bytes_allocated[idx] -= old_bytes; 796 self.bytes_allocated[idx] += new_bytes; 797 798 let entry = match change.handle { 799 TextureCacheHandle::Auto(handle) => self.lru_cache.get_opt_mut(&handle).unwrap(), 800 TextureCacheHandle::Manual(handle) => self.manual_entries.get_opt_mut(&handle).unwrap(), 801 TextureCacheHandle::Empty => { panic!("invalid handle"); } 802 }; 803 entry.texture_id = change.new_tex; 804 entry.details = EntryDetails::Cache { 805 origin: change.new_rect.min, 806 alloc_id: change.new_id, 807 allocated_size_in_bytes: new_bytes, 808 }; 809 810 gpu_cache.invalidate(&entry.uv_rect_handle); 811 entry.uv_rect_handle = GpuCacheHandle::new(); 812 813 let src_rect = DeviceIntRect::from_origin_and_size(change.old_rect.min, entry.size); 814 let dst_rect = DeviceIntRect::from_origin_and_size(change.new_rect.min, entry.size); 815 816 self.pending_updates.push_copy(change.old_tex, &src_rect, change.new_tex, &dst_rect); 817 818 if self.debug_flags.contains( 819 DebugFlags::TEXTURE_CACHE_DBG | 820 DebugFlags::TEXTURE_CACHE_DBG_CLEAR_EVICTED) 821 { 822 self.pending_updates.push_debug_clear( 823 change.old_tex, 824 src_rect.min, 825 src_rect.width(), 826 src_rect.height(), 827 ); 828 } 829 } 830 } 831 832 // Request an item in the texture cache. All images that will 833 // be used on a frame *must* have request() called on their 834 // handle, to update the last used timestamp and ensure 835 // that resources are not flushed from the cache too early. 836 // 837 // Returns true if the image needs to be uploaded to the 838 // texture cache (either never uploaded, or has been 839 // evicted on a previous frame). request(&mut self, handle: &TextureCacheHandle, gpu_cache: &mut GpuCache) -> bool840 pub fn request(&mut self, handle: &TextureCacheHandle, gpu_cache: &mut GpuCache) -> bool { 841 let now = self.now; 842 let entry = match handle { 843 TextureCacheHandle::Empty => None, 844 TextureCacheHandle::Auto(handle) => { 845 // Call touch rather than get_opt_mut so that the LRU index 846 // knows that the entry has been used. 847 self.lru_cache.touch(handle) 848 }, 849 TextureCacheHandle::Manual(handle) => { 850 self.manual_entries.get_opt_mut(handle) 851 }, 852 }; 853 entry.map_or(true, |entry| { 854 // If an image is requested that is already in the cache, 855 // refresh the GPU cache data associated with this item. 856 entry.last_access = now; 857 entry.update_gpu_cache(gpu_cache); 858 false 859 }) 860 } 861 get_entry_opt(&self, handle: &TextureCacheHandle) -> Option<&CacheEntry>862 fn get_entry_opt(&self, handle: &TextureCacheHandle) -> Option<&CacheEntry> { 863 match handle { 864 TextureCacheHandle::Empty => None, 865 TextureCacheHandle::Auto(handle) => self.lru_cache.get_opt(handle), 866 TextureCacheHandle::Manual(handle) => self.manual_entries.get_opt(handle), 867 } 868 } 869 get_entry_opt_mut(&mut self, handle: &TextureCacheHandle) -> Option<&mut CacheEntry>870 fn get_entry_opt_mut(&mut self, handle: &TextureCacheHandle) -> Option<&mut CacheEntry> { 871 match handle { 872 TextureCacheHandle::Empty => None, 873 TextureCacheHandle::Auto(handle) => self.lru_cache.get_opt_mut(handle), 874 TextureCacheHandle::Manual(handle) => self.manual_entries.get_opt_mut(handle), 875 } 876 } 877 878 // Returns true if the image needs to be uploaded to the 879 // texture cache (either never uploaded, or has been 880 // evicted on a previous frame). needs_upload(&self, handle: &TextureCacheHandle) -> bool881 pub fn needs_upload(&self, handle: &TextureCacheHandle) -> bool { 882 !self.is_allocated(handle) 883 } 884 max_texture_size(&self) -> i32885 pub fn max_texture_size(&self) -> i32 { 886 self.max_texture_size 887 } 888 tiling_threshold(&self) -> i32889 pub fn tiling_threshold(&self) -> i32 { 890 self.tiling_threshold 891 } 892 893 #[cfg(feature = "replay")] color_formats(&self) -> TextureFormatPair<ImageFormat>894 pub fn color_formats(&self) -> TextureFormatPair<ImageFormat> { 895 self.shared_textures.color8_linear.texture_parameters().formats.clone() 896 } 897 898 #[cfg(feature = "replay")] swizzle_settings(&self) -> Option<SwizzleSettings>899 pub fn swizzle_settings(&self) -> Option<SwizzleSettings> { 900 self.swizzle 901 } 902 pending_updates(&mut self) -> TextureUpdateList903 pub fn pending_updates(&mut self) -> TextureUpdateList { 904 mem::replace(&mut self.pending_updates, TextureUpdateList::new()) 905 } 906 907 // Update the data stored by a given texture cache handle. update( &mut self, handle: &mut TextureCacheHandle, descriptor: ImageDescriptor, filter: TextureFilter, data: Option<CachedImageData>, user_data: [f32; 4], mut dirty_rect: ImageDirtyRect, gpu_cache: &mut GpuCache, eviction_notice: Option<&EvictionNotice>, uv_rect_kind: UvRectKind, eviction: Eviction, shader: TargetShader, )908 pub fn update( 909 &mut self, 910 handle: &mut TextureCacheHandle, 911 descriptor: ImageDescriptor, 912 filter: TextureFilter, 913 data: Option<CachedImageData>, 914 user_data: [f32; 4], 915 mut dirty_rect: ImageDirtyRect, 916 gpu_cache: &mut GpuCache, 917 eviction_notice: Option<&EvictionNotice>, 918 uv_rect_kind: UvRectKind, 919 eviction: Eviction, 920 shader: TargetShader, 921 ) { 922 debug_assert!(self.now.is_valid()); 923 // Determine if we need to allocate texture cache memory 924 // for this item. We need to reallocate if any of the following 925 // is true: 926 // - Never been in the cache 927 // - Has been in the cache but was evicted. 928 // - Exists in the cache but dimensions / format have changed. 929 let realloc = match self.get_entry_opt(handle) { 930 Some(entry) => { 931 entry.size != descriptor.size || (entry.input_format != descriptor.format && 932 entry.alternative_input_format() != descriptor.format) 933 } 934 None => { 935 // Not allocated, or was previously allocated but has been evicted. 936 true 937 } 938 }; 939 940 if realloc { 941 let params = CacheAllocParams { descriptor, filter, user_data, uv_rect_kind, shader }; 942 self.allocate(¶ms, handle, eviction); 943 944 // If we reallocated, we need to upload the whole item again. 945 dirty_rect = DirtyRect::All; 946 } 947 948 let entry = self.get_entry_opt_mut(handle) 949 .expect("BUG: There must be an entry at this handle now"); 950 951 // Install the new eviction notice for this update, if applicable. 952 entry.eviction_notice = eviction_notice.cloned(); 953 entry.uv_rect_kind = uv_rect_kind; 954 955 // Invalidate the contents of the resource rect in the GPU cache. 956 // This ensures that the update_gpu_cache below will add 957 // the new information to the GPU cache. 958 //TODO: only invalidate if the parameters change? 959 gpu_cache.invalidate(&entry.uv_rect_handle); 960 961 // Upload the resource rect and texture array layer. 962 entry.update_gpu_cache(gpu_cache); 963 964 // Create an update command, which the render thread processes 965 // to upload the new image data into the correct location 966 // in GPU memory. 967 if let Some(data) = data { 968 // If the swizzling is supported, we always upload in the internal 969 // texture format (thus avoiding the conversion by the driver). 970 // Otherwise, pass the external format to the driver. 971 let origin = entry.details.describe(); 972 let texture_id = entry.texture_id; 973 let size = entry.size; 974 let use_upload_format = self.swizzle.is_none(); 975 let op = TextureCacheUpdate::new_update( 976 data, 977 &descriptor, 978 origin, 979 size, 980 use_upload_format, 981 &dirty_rect, 982 ); 983 self.pending_updates.push_update(texture_id, op); 984 } 985 } 986 987 // Check if a given texture handle has a valid allocation 988 // in the texture cache. is_allocated(&self, handle: &TextureCacheHandle) -> bool989 pub fn is_allocated(&self, handle: &TextureCacheHandle) -> bool { 990 self.get_entry_opt(handle).is_some() 991 } 992 993 // Return the allocated size of the texture handle's associated data, 994 // or otherwise indicate the handle is invalid. get_allocated_size(&self, handle: &TextureCacheHandle) -> Option<usize>995 pub fn get_allocated_size(&self, handle: &TextureCacheHandle) -> Option<usize> { 996 self.get_entry_opt(handle).map(|entry| { 997 (entry.input_format.bytes_per_pixel() * entry.size.area()) as usize 998 }) 999 } 1000 1001 // Retrieve the details of an item in the cache. This is used 1002 // during batch creation to provide the resource rect address 1003 // to the shaders and texture ID to the batching logic. 1004 // This function will assert in debug modes if the caller 1005 // tries to get a handle that was not requested this frame. get(&self, handle: &TextureCacheHandle) -> CacheItem1006 pub fn get(&self, handle: &TextureCacheHandle) -> CacheItem { 1007 let (texture_id, uv_rect, swizzle, uv_rect_handle, user_data) = self.get_cache_location(handle); 1008 CacheItem { 1009 uv_rect_handle, 1010 texture_id: TextureSource::TextureCache( 1011 texture_id, 1012 swizzle, 1013 ), 1014 uv_rect, 1015 user_data, 1016 } 1017 } 1018 1019 /// A more detailed version of get(). This allows access to the actual 1020 /// device rect of the cache allocation. 1021 /// 1022 /// Returns a tuple identifying the texture, the layer, the region, 1023 /// and its GPU handle. get_cache_location( &self, handle: &TextureCacheHandle, ) -> (CacheTextureId, DeviceIntRect, Swizzle, GpuCacheHandle, [f32; 4])1024 pub fn get_cache_location( 1025 &self, 1026 handle: &TextureCacheHandle, 1027 ) -> (CacheTextureId, DeviceIntRect, Swizzle, GpuCacheHandle, [f32; 4]) { 1028 let entry = self 1029 .get_entry_opt(handle) 1030 .expect("BUG: was dropped from cache or not updated!"); 1031 debug_assert_eq!(entry.last_access, self.now); 1032 let origin = entry.details.describe(); 1033 ( 1034 entry.texture_id, 1035 DeviceIntRect::from_origin_and_size(origin, entry.size), 1036 entry.swizzle, 1037 entry.uv_rect_handle, 1038 entry.user_data, 1039 ) 1040 } 1041 1042 /// Internal helper function to evict a strong texture cache handle evict_impl( &mut self, entry: CacheEntry, )1043 fn evict_impl( 1044 &mut self, 1045 entry: CacheEntry, 1046 ) { 1047 entry.evict(); 1048 self.free(&entry); 1049 } 1050 1051 /// Evict a texture cache handle that was previously set to be in manual 1052 /// eviction mode. evict_handle(&mut self, handle: &TextureCacheHandle)1053 pub fn evict_handle(&mut self, handle: &TextureCacheHandle) { 1054 match handle { 1055 TextureCacheHandle::Manual(handle) => { 1056 // Find the strong handle that matches this weak handle. If this 1057 // ever shows up in profiles, we can make it a hash (but the number 1058 // of manual eviction handles is typically small). 1059 // Alternatively, we could make a more forgiving FreeList variant 1060 // which does not differentiate between strong and weak handles. 1061 let index = self.manual_handles.iter().position(|strong_handle| { 1062 strong_handle.matches(handle) 1063 }); 1064 if let Some(index) = index { 1065 let handle = self.manual_handles.swap_remove(index); 1066 let entry = self.manual_entries.free(handle); 1067 self.evict_impl(entry); 1068 } 1069 } 1070 TextureCacheHandle::Auto(handle) => { 1071 if let Some(entry) = self.lru_cache.remove(handle) { 1072 self.evict_impl(entry); 1073 } 1074 } 1075 _ => {} 1076 } 1077 } 1078 dump_color8_linear_as_svg(&self, output: &mut dyn std::io::Write) -> std::io::Result<()>1079 pub fn dump_color8_linear_as_svg(&self, output: &mut dyn std::io::Write) -> std::io::Result<()> { 1080 self.shared_textures.color8_linear.dump_as_svg(output) 1081 } 1082 dump_color8_glyphs_as_svg(&self, output: &mut dyn std::io::Write) -> std::io::Result<()>1083 pub fn dump_color8_glyphs_as_svg(&self, output: &mut dyn std::io::Write) -> std::io::Result<()> { 1084 self.shared_textures.color8_glyphs.dump_as_svg(output) 1085 } 1086 dump_alpha8_glyphs_as_svg(&self, output: &mut dyn std::io::Write) -> std::io::Result<()>1087 pub fn dump_alpha8_glyphs_as_svg(&self, output: &mut dyn std::io::Write) -> std::io::Result<()> { 1088 self.shared_textures.alpha8_glyphs.dump_as_svg(output) 1089 } 1090 dump_alpha8_linear_as_svg(&self, output: &mut dyn std::io::Write) -> std::io::Result<()>1091 pub fn dump_alpha8_linear_as_svg(&self, output: &mut dyn std::io::Write) -> std::io::Result<()> { 1092 self.shared_textures.alpha8_linear.dump_as_svg(output) 1093 } 1094 1095 /// Get the eviction threshold, in bytes, for the given budget type. get_eviction_threshold(&self, budget_type: BudgetType) -> usize1096 fn get_eviction_threshold(&self, budget_type: BudgetType) -> usize { 1097 if budget_type == BudgetType::Standalone { 1098 // For standalone textures, the only reason to evict textures is 1099 // to save GPU memory. Batching / draw call concerns do not apply 1100 // to standalone textures, because unused textures don't cause 1101 // extra draw calls. 1102 return 8 * 1024 * 1024; 1103 } 1104 1105 // For shared textures, evicting an entry only frees up GPU memory if it 1106 // causes one of the shared textures to become empty, so we want to avoid 1107 // getting slightly above the capacity of a texture. 1108 // The other concern for shared textures is batching: The entries that 1109 // are needed in the current frame should be distributed across as few 1110 // shared textures as possible, to minimize the number of draw calls. 1111 // Ideally we only want one texture per type under simple workloads. 1112 1113 let bytes_per_texture = self.shared_textures.bytes_per_shared_texture(budget_type); 1114 1115 // Number of allocated bytes under which we don't bother with evicting anything 1116 // from the cache. Above the threshold we consider evicting the coldest items 1117 // depending on how cold they are. 1118 // 1119 // Above all else we want to make sure that even after a heavy workload, the 1120 // shared cache settles back to a single texture atlas per type over some reasonable 1121 // period of time. 1122 // This is achieved by the compaction logic which will try to consolidate items that 1123 // are spread over multiple textures into few ones, and by evicting old items 1124 // so that the compaction logic has room to do its job. 1125 // 1126 // The other goal is to leave enough empty space in the texture atlases 1127 // so that we are not too likely to have to allocate a new texture atlas on 1128 // the next frame if we switch to a new tab or load a new page. That's why 1129 // the following thresholds are rather low. Note that even when above the threshold, 1130 // we only evict cold items and ramp up the eviction pressure depending on the amount 1131 // of allocated memory (See should_continue_evicting). 1132 let ideal_utilization = match budget_type { 1133 BudgetType::SharedAlpha8Glyphs | BudgetType::SharedColor8Glyphs => { 1134 // Glyphs are usually small and tightly packed so they waste very little 1135 // space in the cache. 1136 bytes_per_texture * 2 / 3 1137 } 1138 _ => { 1139 // Other types of images come with a variety of sizes making them more 1140 // prone to wasting pixels and causing fragmentation issues so we put 1141 // more pressure on them. 1142 bytes_per_texture / 3 1143 } 1144 }; 1145 1146 ideal_utilization 1147 } 1148 1149 /// Returns whether to continue eviction and how cold an item need to be to be evicted. 1150 /// 1151 /// If the None is returned, stop evicting. 1152 /// If the Some(n) is returned, continue evicting if the coldest item hasn't been used 1153 /// for more than n frames. should_continue_evicting( &self, budget_type: BudgetType, eviction_count: usize, ) -> Option<usize>1154 fn should_continue_evicting( 1155 &self, 1156 budget_type: BudgetType, 1157 eviction_count: usize, 1158 ) -> Option<usize> { 1159 1160 let threshold = self.get_eviction_threshold(budget_type); 1161 let bytes_allocated = self.bytes_allocated[budget_type as usize]; 1162 1163 let uses_multiple_atlases = self.shared_textures.has_multiple_textures(budget_type); 1164 1165 // If current memory usage is below selected threshold, we can stop evicting items 1166 // except when using shared texture atlases and more than one texture is in use. 1167 // This is not very common but can happen due to fragmentation and the only way 1168 // to get rid of that fragmentation is to continue evicting. 1169 if bytes_allocated < threshold && !uses_multiple_atlases { 1170 return None; 1171 } 1172 1173 // Number of frames since last use that is considered too recent for eviction, 1174 // depending on the cache pressure. 1175 let age_theshold = match bytes_allocated / threshold { 1176 0 => 400, 1177 1 => 200, 1178 2 => 100, 1179 3 => 50, 1180 4 => 25, 1181 5 => 10, 1182 6 => 5, 1183 _ => 1, 1184 }; 1185 1186 // If current memory usage is significantly more than the threshold, keep evicting this frame 1187 if bytes_allocated > 4 * threshold { 1188 return Some(age_theshold); 1189 } 1190 1191 // Otherwise, only allow evicting up to a certain number of items per frame. This allows evictions 1192 // to be spread over a number of frames, to avoid frame spikes. 1193 if eviction_count < Self::MAX_EVICTIONS_PER_FRAME { 1194 return Some(age_theshold) 1195 } 1196 1197 None 1198 } 1199 1200 1201 /// Evict old items from the shared and standalone caches, if we're over a 1202 /// threshold memory usage value evict_items_from_cache_if_required(&mut self, profile: &mut TransactionProfile)1203 fn evict_items_from_cache_if_required(&mut self, profile: &mut TransactionProfile) { 1204 let previous_frame_id = self.now.frame_id() - 1; 1205 let mut eviction_count = 0; 1206 let mut youngest_evicted = FrameId::first(); 1207 1208 for budget in BudgetType::iter() { 1209 while let Some(age_threshold) = self.should_continue_evicting( 1210 budget, 1211 eviction_count, 1212 ) { 1213 if let Some(entry) = self.lru_cache.peek_oldest(budget as u8) { 1214 // Only evict this item if it wasn't used in the previous frame. The reason being that if it 1215 // was used the previous frame then it will likely be used in this frame too, and we don't 1216 // want to be continually evicting and reuploading the item every frame. 1217 if entry.last_access.frame_id() + age_threshold > previous_frame_id { 1218 // Since the LRU cache is ordered by frame access, we can break out of the loop here because 1219 // we know that all remaining items were also used in the previous frame (or more recently). 1220 break; 1221 } 1222 if entry.last_access.frame_id() > youngest_evicted { 1223 youngest_evicted = entry.last_access.frame_id(); 1224 } 1225 let entry = self.lru_cache.pop_oldest(budget as u8).unwrap(); 1226 entry.evict(); 1227 self.free(&entry); 1228 eviction_count += 1; 1229 } else { 1230 // The LRU cache is empty, all remaining items use manual 1231 // eviction. In this case, there's nothing we can do until 1232 // the calling code manually evicts items to reduce the 1233 // allocated cache size. 1234 break; 1235 } 1236 } 1237 } 1238 1239 if eviction_count > 0 { 1240 profile.set(profiler::TEXTURE_CACHE_EVICTION_COUNT, eviction_count); 1241 profile.set( 1242 profiler::TEXTURE_CACHE_YOUNGEST_EVICTION, 1243 self.now.frame_id().as_usize() - youngest_evicted.as_usize() 1244 ); 1245 } 1246 } 1247 1248 // Free a cache entry from the standalone list or shared cache. free(&mut self, entry: &CacheEntry)1249 fn free(&mut self, entry: &CacheEntry) { 1250 match entry.details { 1251 EntryDetails::Standalone { size_in_bytes, .. } => { 1252 self.bytes_allocated[BudgetType::Standalone as usize] -= size_in_bytes; 1253 1254 // This is a standalone texture allocation. Free it directly. 1255 self.pending_updates.push_free(entry.texture_id); 1256 } 1257 EntryDetails::Cache { origin, alloc_id, allocated_size_in_bytes } => { 1258 let (allocator_list, budget_type) = self.shared_textures.select( 1259 entry.input_format, 1260 entry.filter, 1261 entry.shader, 1262 ); 1263 1264 allocator_list.deallocate(entry.texture_id, alloc_id); 1265 1266 self.bytes_allocated[budget_type as usize] -= allocated_size_in_bytes; 1267 1268 if self.debug_flags.contains( 1269 DebugFlags::TEXTURE_CACHE_DBG | 1270 DebugFlags::TEXTURE_CACHE_DBG_CLEAR_EVICTED) 1271 { 1272 self.pending_updates.push_debug_clear( 1273 entry.texture_id, 1274 origin, 1275 entry.size.width, 1276 entry.size.height, 1277 ); 1278 } 1279 } 1280 } 1281 } 1282 1283 /// Allocate a block from the shared cache. allocate_from_shared_cache( &mut self, params: &CacheAllocParams, ) -> (CacheEntry, BudgetType)1284 fn allocate_from_shared_cache( 1285 &mut self, 1286 params: &CacheAllocParams, 1287 ) -> (CacheEntry, BudgetType) { 1288 let (allocator_list, budget_type) = self.shared_textures.select( 1289 params.descriptor.format, 1290 params.filter, 1291 params.shader, 1292 ); 1293 1294 // To avoid referring to self in the closure. 1295 let next_id = &mut self.next_id; 1296 let pending_updates = &mut self.pending_updates; 1297 1298 let (texture_id, alloc_id, allocated_rect) = allocator_list.allocate( 1299 params.descriptor.size, 1300 &mut |size, parameters| { 1301 let texture_id = *next_id; 1302 next_id.0 += 1; 1303 pending_updates.push_alloc( 1304 texture_id, 1305 TextureCacheAllocInfo { 1306 target: ImageBufferKind::Texture2D, 1307 width: size.width, 1308 height: size.height, 1309 format: parameters.formats.internal, 1310 filter: parameters.filter, 1311 is_shared_cache: true, 1312 has_depth: false, 1313 category: TextureCacheCategory::Atlas, 1314 }, 1315 ); 1316 1317 texture_id 1318 }, 1319 ); 1320 1321 let formats = &allocator_list.texture_parameters().formats; 1322 1323 let swizzle = if formats.external == params.descriptor.format { 1324 Swizzle::default() 1325 } else { 1326 match self.swizzle { 1327 Some(_) => Swizzle::Bgra, 1328 None => Swizzle::default(), 1329 } 1330 }; 1331 1332 let bpp = formats.internal.bytes_per_pixel(); 1333 let allocated_size_in_bytes = (allocated_rect.area() * bpp) as usize; 1334 self.bytes_allocated[budget_type as usize] += allocated_size_in_bytes; 1335 1336 (CacheEntry { 1337 size: params.descriptor.size, 1338 user_data: params.user_data, 1339 last_access: self.now, 1340 details: EntryDetails::Cache { 1341 origin: allocated_rect.min, 1342 alloc_id, 1343 allocated_size_in_bytes, 1344 }, 1345 uv_rect_handle: GpuCacheHandle::new(), 1346 input_format: params.descriptor.format, 1347 filter: params.filter, 1348 swizzle, 1349 texture_id, 1350 eviction_notice: None, 1351 uv_rect_kind: params.uv_rect_kind, 1352 shader: params.shader 1353 }, budget_type) 1354 } 1355 1356 // Returns true if the given image descriptor *may* be 1357 // placed in the shared texture cache. is_allowed_in_shared_cache( &self, filter: TextureFilter, descriptor: &ImageDescriptor, ) -> bool1358 pub fn is_allowed_in_shared_cache( 1359 &self, 1360 filter: TextureFilter, 1361 descriptor: &ImageDescriptor, 1362 ) -> bool { 1363 let mut allowed_in_shared_cache = true; 1364 1365 if matches!(descriptor.format, ImageFormat::RGBA8 | ImageFormat::BGRA8) 1366 && filter == TextureFilter::Linear 1367 { 1368 // Allow the maximum that can fit in the linear color texture's two column layout. 1369 let max = self.shared_textures.color8_linear.size() / 2; 1370 allowed_in_shared_cache = descriptor.size.width.max(descriptor.size.height) <= max; 1371 } else if descriptor.size.width > TEXTURE_REGION_DIMENSIONS { 1372 allowed_in_shared_cache = false; 1373 } 1374 1375 if descriptor.size.height > TEXTURE_REGION_DIMENSIONS { 1376 allowed_in_shared_cache = false; 1377 } 1378 1379 // TODO(gw): For now, alpha formats of the texture cache can only be linearly sampled. 1380 // Nearest sampling gets a standalone texture. 1381 // This is probably rare enough that it can be fixed up later. 1382 if filter == TextureFilter::Nearest && 1383 descriptor.format.bytes_per_pixel() <= 2 1384 { 1385 allowed_in_shared_cache = false; 1386 } 1387 1388 allowed_in_shared_cache 1389 } 1390 1391 /// Allocate a render target via the pending updates sent to the renderer alloc_render_target( &mut self, size: DeviceIntSize, format: ImageFormat, ) -> CacheTextureId1392 pub fn alloc_render_target( 1393 &mut self, 1394 size: DeviceIntSize, 1395 format: ImageFormat, 1396 ) -> CacheTextureId { 1397 let texture_id = self.next_id; 1398 self.next_id.0 += 1; 1399 1400 // Push a command to allocate device storage of the right size / format. 1401 let info = TextureCacheAllocInfo { 1402 target: ImageBufferKind::Texture2D, 1403 width: size.width, 1404 height: size.height, 1405 format, 1406 filter: TextureFilter::Linear, 1407 is_shared_cache: false, 1408 has_depth: false, 1409 category: TextureCacheCategory::RenderTarget, 1410 }; 1411 1412 self.pending_updates.push_alloc(texture_id, info); 1413 1414 texture_id 1415 } 1416 1417 /// Free an existing render target free_render_target( &mut self, id: CacheTextureId, )1418 pub fn free_render_target( 1419 &mut self, 1420 id: CacheTextureId, 1421 ) { 1422 self.pending_updates.push_free(id); 1423 } 1424 1425 /// Allocates a new standalone cache entry. allocate_standalone_entry( &mut self, params: &CacheAllocParams, ) -> (CacheEntry, BudgetType)1426 fn allocate_standalone_entry( 1427 &mut self, 1428 params: &CacheAllocParams, 1429 ) -> (CacheEntry, BudgetType) { 1430 let texture_id = self.next_id; 1431 self.next_id.0 += 1; 1432 1433 // Push a command to allocate device storage of the right size / format. 1434 let info = TextureCacheAllocInfo { 1435 target: ImageBufferKind::Texture2D, 1436 width: params.descriptor.size.width, 1437 height: params.descriptor.size.height, 1438 format: params.descriptor.format, 1439 filter: params.filter, 1440 is_shared_cache: false, 1441 has_depth: false, 1442 category: TextureCacheCategory::Standalone, 1443 }; 1444 1445 let size_in_bytes = (info.width * info.height * info.format.bytes_per_pixel()) as usize; 1446 self.bytes_allocated[BudgetType::Standalone as usize] += size_in_bytes; 1447 1448 self.pending_updates.push_alloc(texture_id, info); 1449 1450 // Special handing for BGRA8 textures that may need to be swizzled. 1451 let swizzle = if params.descriptor.format == ImageFormat::BGRA8 { 1452 self.swizzle.map(|s| s.bgra8_sampling_swizzle) 1453 } else { 1454 None 1455 }; 1456 1457 (CacheEntry::new_standalone( 1458 texture_id, 1459 self.now, 1460 params, 1461 swizzle.unwrap_or_default(), 1462 size_in_bytes, 1463 ), BudgetType::Standalone) 1464 } 1465 1466 /// Allocates a cache entry for the given parameters, and updates the 1467 /// provided handle to point to the new entry. allocate( &mut self, params: &CacheAllocParams, handle: &mut TextureCacheHandle, eviction: Eviction, )1468 fn allocate( 1469 &mut self, 1470 params: &CacheAllocParams, 1471 handle: &mut TextureCacheHandle, 1472 eviction: Eviction, 1473 ) { 1474 debug_assert!(self.now.is_valid()); 1475 assert!(!params.descriptor.size.is_empty()); 1476 1477 // If this image doesn't qualify to go in the shared (batching) cache, 1478 // allocate a standalone entry. 1479 let use_shared_cache = self.is_allowed_in_shared_cache(params.filter, ¶ms.descriptor); 1480 let (new_cache_entry, budget_type) = if use_shared_cache { 1481 self.allocate_from_shared_cache(params) 1482 } else { 1483 self.allocate_standalone_entry(params) 1484 }; 1485 1486 let details = new_cache_entry.details.clone(); 1487 let texture_id = new_cache_entry.texture_id; 1488 1489 // If the handle points to a valid cache entry, we want to replace the 1490 // cache entry with our newly updated location. We also need to ensure 1491 // that the storage (region or standalone) associated with the previous 1492 // entry here gets freed. 1493 // 1494 // If the handle is invalid, we need to insert the data, and append the 1495 // result to the corresponding vector. 1496 let old_entry = match (&mut *handle, eviction) { 1497 (TextureCacheHandle::Auto(handle), Eviction::Auto) => { 1498 self.lru_cache.replace_or_insert(handle, budget_type as u8, new_cache_entry) 1499 }, 1500 (TextureCacheHandle::Manual(handle), Eviction::Manual) => { 1501 let entry = self.manual_entries.get_opt_mut(handle) 1502 .expect("Don't call this after evicting"); 1503 Some(mem::replace(entry, new_cache_entry)) 1504 }, 1505 (TextureCacheHandle::Manual(_), Eviction::Auto) | 1506 (TextureCacheHandle::Auto(_), Eviction::Manual) => { 1507 panic!("Can't change eviction policy after initial allocation"); 1508 }, 1509 (TextureCacheHandle::Empty, Eviction::Auto) => { 1510 let new_handle = self.lru_cache.push_new(budget_type as u8, new_cache_entry); 1511 *handle = TextureCacheHandle::Auto(new_handle); 1512 None 1513 }, 1514 (TextureCacheHandle::Empty, Eviction::Manual) => { 1515 let manual_handle = self.manual_entries.insert(new_cache_entry); 1516 let new_handle = manual_handle.weak(); 1517 self.manual_handles.push(manual_handle); 1518 *handle = TextureCacheHandle::Manual(new_handle); 1519 None 1520 }, 1521 }; 1522 if let Some(old_entry) = old_entry { 1523 old_entry.evict(); 1524 self.free(&old_entry); 1525 } 1526 1527 if let EntryDetails::Cache { alloc_id, .. } = details { 1528 let allocator_list = self.shared_textures.select( 1529 params.descriptor.format, 1530 params.filter, 1531 params.shader, 1532 ).0; 1533 1534 allocator_list.set_handle(texture_id, alloc_id, handle); 1535 } 1536 } 1537 shared_alpha_expected_format(&self) -> ImageFormat1538 pub fn shared_alpha_expected_format(&self) -> ImageFormat { 1539 self.shared_textures.alpha8_linear.texture_parameters().formats.external 1540 } 1541 shared_color_expected_format(&self) -> ImageFormat1542 pub fn shared_color_expected_format(&self) -> ImageFormat { 1543 self.shared_textures.color8_linear.texture_parameters().formats.external 1544 } 1545 1546 1547 #[cfg(test)] total_allocated_bytes_for_testing(&self) -> usize1548 pub fn total_allocated_bytes_for_testing(&self) -> usize { 1549 BudgetType::iter().map(|b| self.bytes_allocated[b as usize]).sum() 1550 } 1551 report_memory(&self, ops: &mut MallocSizeOfOps) -> usize1552 pub fn report_memory(&self, ops: &mut MallocSizeOfOps) -> usize { 1553 self.lru_cache.size_of(ops) 1554 } 1555 } 1556 1557 #[cfg_attr(feature = "capture", derive(Serialize))] 1558 #[cfg_attr(feature = "replay", derive(Deserialize))] 1559 pub struct TextureParameters { 1560 pub formats: TextureFormatPair<ImageFormat>, 1561 pub filter: TextureFilter, 1562 } 1563 1564 impl TextureCacheUpdate { 1565 // Constructs a TextureCacheUpdate operation to be passed to the 1566 // rendering thread in order to do an upload to the right 1567 // location in the texture cache. new_update( data: CachedImageData, descriptor: &ImageDescriptor, origin: DeviceIntPoint, size: DeviceIntSize, use_upload_format: bool, dirty_rect: &ImageDirtyRect, ) -> TextureCacheUpdate1568 fn new_update( 1569 data: CachedImageData, 1570 descriptor: &ImageDescriptor, 1571 origin: DeviceIntPoint, 1572 size: DeviceIntSize, 1573 use_upload_format: bool, 1574 dirty_rect: &ImageDirtyRect, 1575 ) -> TextureCacheUpdate { 1576 let source = match data { 1577 CachedImageData::Blob => { 1578 panic!("The vector image should have been rasterized."); 1579 } 1580 CachedImageData::External(ext_image) => match ext_image.image_type { 1581 ExternalImageType::TextureHandle(_) => { 1582 panic!("External texture handle should not go through texture_cache."); 1583 } 1584 ExternalImageType::Buffer => TextureUpdateSource::External { 1585 id: ext_image.id, 1586 channel_index: ext_image.channel_index, 1587 }, 1588 }, 1589 CachedImageData::Raw(bytes) => { 1590 let finish = descriptor.offset + 1591 descriptor.size.width * descriptor.format.bytes_per_pixel() + 1592 (descriptor.size.height - 1) * descriptor.compute_stride(); 1593 assert!(bytes.len() >= finish as usize); 1594 1595 TextureUpdateSource::Bytes { data: bytes } 1596 } 1597 }; 1598 let format_override = if use_upload_format { 1599 Some(descriptor.format) 1600 } else { 1601 None 1602 }; 1603 1604 match *dirty_rect { 1605 DirtyRect::Partial(dirty) => { 1606 // the dirty rectangle doesn't have to be within the area but has to intersect it, at least 1607 let stride = descriptor.compute_stride(); 1608 let offset = descriptor.offset + dirty.min.y * stride + dirty.min.x * descriptor.format.bytes_per_pixel(); 1609 1610 TextureCacheUpdate { 1611 rect: DeviceIntRect::from_origin_and_size( 1612 DeviceIntPoint::new(origin.x + dirty.min.x, origin.y + dirty.min.y), 1613 DeviceIntSize::new( 1614 dirty.width().min(size.width - dirty.min.x), 1615 dirty.height().min(size.height - dirty.min.y), 1616 ), 1617 ), 1618 source, 1619 stride: Some(stride), 1620 offset, 1621 format_override, 1622 } 1623 } 1624 DirtyRect::All => { 1625 TextureCacheUpdate { 1626 rect: DeviceIntRect::from_origin_and_size(origin, size), 1627 source, 1628 stride: descriptor.stride, 1629 offset: descriptor.offset, 1630 format_override, 1631 } 1632 } 1633 } 1634 } 1635 } 1636 1637 #[cfg(test)] 1638 mod test_texture_cache { 1639 #[test] check_allocation_size_balance()1640 fn check_allocation_size_balance() { 1641 // Allocate some glyphs, observe the total allocation size, and free 1642 // the glyphs again. Check that the total allocation size is back at the 1643 // original value. 1644 1645 use crate::texture_cache::{TextureCache, TextureCacheHandle, Eviction, TargetShader}; 1646 use crate::gpu_cache::GpuCache; 1647 use crate::device::TextureFilter; 1648 use crate::gpu_types::UvRectKind; 1649 use api::{ImageDescriptor, ImageDescriptorFlags, ImageFormat, DirtyRect}; 1650 use api::units::*; 1651 use euclid::size2; 1652 let mut texture_cache = TextureCache::new_for_testing(2048, ImageFormat::BGRA8); 1653 let mut gpu_cache = GpuCache::new_for_testing(); 1654 1655 let sizes: &[DeviceIntSize] = &[ 1656 size2(23, 27), 1657 size2(15, 22), 1658 size2(11, 5), 1659 size2(20, 25), 1660 size2(38, 41), 1661 size2(11, 19), 1662 size2(13, 21), 1663 size2(37, 40), 1664 size2(13, 15), 1665 size2(14, 16), 1666 size2(10, 9), 1667 size2(25, 28), 1668 ]; 1669 1670 let bytes_at_start = texture_cache.total_allocated_bytes_for_testing(); 1671 1672 let handles: Vec<TextureCacheHandle> = sizes.iter().map(|size| { 1673 let mut texture_cache_handle = TextureCacheHandle::invalid(); 1674 texture_cache.request(&texture_cache_handle, &mut gpu_cache); 1675 texture_cache.update( 1676 &mut texture_cache_handle, 1677 ImageDescriptor { 1678 size: *size, 1679 stride: None, 1680 format: ImageFormat::BGRA8, 1681 flags: ImageDescriptorFlags::empty(), 1682 offset: 0, 1683 }, 1684 TextureFilter::Linear, 1685 None, 1686 [0.0; 4], 1687 DirtyRect::All, 1688 &mut gpu_cache, 1689 None, 1690 UvRectKind::Rect, 1691 Eviction::Manual, 1692 TargetShader::Text, 1693 ); 1694 texture_cache_handle 1695 }).collect(); 1696 1697 let bytes_after_allocating = texture_cache.total_allocated_bytes_for_testing(); 1698 assert!(bytes_after_allocating > bytes_at_start); 1699 1700 for handle in handles { 1701 texture_cache.evict_handle(&handle); 1702 } 1703 1704 let bytes_at_end = texture_cache.total_allocated_bytes_for_testing(); 1705 assert_eq!(bytes_at_end, bytes_at_start); 1706 } 1707 } 1708