1 // Copyright 2018 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef COMPONENTS_VIZ_COMMON_GL_SCALER_H_ 6 #define COMPONENTS_VIZ_COMMON_GL_SCALER_H_ 7 8 #include <stdint.h> 9 10 #include <map> 11 #include <memory> 12 #include <ostream> 13 #include <string> 14 #include <tuple> 15 #include <utility> 16 #include <vector> 17 18 #include "base/compiler_specific.h" 19 #include "base/macros.h" 20 #include "base/memory/scoped_refptr.h" 21 #include "base/optional.h" 22 #include "components/viz/common/gpu/context_lost_observer.h" 23 #include "components/viz/common/viz_common_export.h" 24 #include "gpu/command_buffer/client/gles2_interface.h" 25 #include "ui/gfx/color_space.h" 26 #include "ui/gfx/geometry/rect.h" 27 #include "ui/gfx/geometry/rect_f.h" 28 #include "ui/gfx/geometry/size.h" 29 #include "ui/gfx/geometry/vector2d.h" 30 31 namespace gfx { 32 class ColorTransform; 33 } // namespace gfx 34 35 namespace viz { 36 37 class ContextProvider; 38 39 // A high-performance texture scaler for use with an OpenGL ES 2.0 context. It 40 // can be configured to operate at different quality levels, manages/converts 41 // color spaces, and optionally re-arranges/formats data in output textures for 42 // use with more-efficient texture readback pipelines. 43 class VIZ_COMMON_EXPORT GLScaler final : public ContextLostObserver { 44 public: 45 struct VIZ_COMMON_EXPORT Parameters { 46 // Relative scale from/to factors. Both of these must be non-zero. 47 gfx::Vector2d scale_from = gfx::Vector2d(1, 1); 48 gfx::Vector2d scale_to = gfx::Vector2d(1, 1); 49 50 // The color space of the source texture and the desired color space of the 51 // output texture. If |source_color_space| is not set (or invalid), sRGB is 52 // assumed. If |output_color_space| is not set (or invalid), the source 53 // color space is assumed. 54 gfx::ColorSpace source_color_space; 55 gfx::ColorSpace output_color_space; 56 57 // Enable color management heuristics, using higher precision texture and 58 // gamma-aware scaling? 59 // 60 // When disabled, the gamma of the source color space and other concerns are 61 // ignored and 8-bit precision is used. 62 // 63 // When enabled, scaling occurs in a linear color space with 16-bit floats. 64 // This produces excellent results for virtually all color spaces while 65 // typically requiring twice the memory and execution resources. The caller 66 // must ensure the GL context supports the use of GL_RGBA16F format 67 // textures. 68 // 69 // Relevant reading: http://www.ericbrasseur.org/gamma.html 70 bool enable_precise_color_management = false; 71 72 // Selects the trade-off between quality and speed. 73 enum class Quality : int8_t { 74 // Bilinear single pass. Fastest possible. Do not use this unless the GL 75 // implementation is so slow that the other quality options won't work. 76 FAST, 77 78 // Bilinear upscale + N * 50% bilinear downscales. This is still fast 79 // enough for general-purpose use, and image quality is nearly as good as 80 // BEST when downscaling. 81 GOOD, 82 83 // Bicubic upscale + N * 50% bicubic downscales. Produces very good 84 // quality scaled images, but it's 2-8x slower than the "GOOD" quality. 85 BEST, 86 } quality = Quality::GOOD; 87 88 // Is the source texture Y-flipped (i.e., the origin is the lower-left 89 // corner and not the upper-left corner)? Most GL textures are Y-flipped. 90 // This information is required so that the scaler can correctly compute the 91 // sampling region. 92 bool is_flipped_source = true; 93 94 // Should the output be vertically flipped? Usually, this is used when the 95 // source is not Y-flipped, but the destination texture needs to be. Or, it 96 // can be used to draw the final output upside-down to avoid having to copy 97 // the rows in reverse order after a glReadPixels(). 98 bool flip_output = false; 99 100 // Optionally rearrange the image data for export. Generally, this is used 101 // to make later readback steps more efficient (e.g., using glReadPixels() 102 // will produce the raw bytes in their correct locations). 103 // 104 // Output textures are assumed to be using one of the 4-channel RGBA 105 // formats. While it may be more "proper" to use a single-component texture 106 // format for the planar-oriented image data, not all GL implementations 107 // support the use of those formats. However, all must support at least 108 // GL_RGBA. Therefore, each RGBA pixel is treated as a generic "vec4" (a 109 // quad of values). 110 // 111 // When using this feature, it is usually necessary to adjust the 112 // |output_rect| passed to Scale() or ScaleToMultipleOutputs(). See notes 113 // below. 114 enum class ExportFormat : int8_t { 115 // Do not rearrange the image data: 116 // 117 // (interleaved quads) (interleaved quads) 118 // RGBA RGBA RGBA RGBA RGBA RGBA RGBA RGBA 119 // RGBA RGBA RGBA RGBA --> RGBA RGBA RGBA RGBA 120 // RGBA RGBA RGBA RGBA RGBA RGBA RGBA RGBA 121 INTERLEAVED_QUADS, 122 123 // Select one color channel, packing each of 4 pixels' values into the 4 124 // elements of one output quad. 125 // 126 // For example, for CHANNEL_0: 127 // 128 // (interleaved quads) (channel 0) 129 // RGBA RGBA RGBA RGBA RGBA RGBA RGBA RGBA RRRR RRRR 130 // RGBA RGBA RGBA RGBA RGBA RGBA RGBA RGBA --> RRRR RRRR 131 // RGBA RGBA RGBA RGBA RGBA RGBA RGBA RGBA RRRR RRRR 132 // 133 // Note: Because of this packing, the horizontal coordinates of the 134 // |output_rect| used with Scale() should be divided by 4. 135 CHANNEL_0, 136 CHANNEL_1, 137 CHANNEL_2, 138 CHANNEL_3, 139 140 // I422 sampling, delivered via two output textures (NV61 format): The 141 // first texture is produced the same as CHANNEL_0, while the second 142 // texture contains CHANNEL_1 and CHANNEL_2 at half-width interleaved and 143 // full-height. For example, if this is combined with RGB→YUV color space 144 // conversion: 145 // 146 // (interleaved quads) 147 // RGBA RGBA RGBA RGBA RGBA RGBA RGBA RGBA 148 // RGBA RGBA RGBA RGBA RGBA RGBA RGBA RGBA 149 // RGBA RGBA RGBA RGBA RGBA RGBA RGBA RGBA 150 // | 151 // | (luma plane) (chroma, interleaved) 152 // | YYYY YYYY UVUV UVUV 153 // +---> { YYYY YYYY + UVUV UVUV } 154 // YYYY YYYY UVUV UVUV 155 // 156 // Note: Because of this packing, the horizontal coordinates of the 157 // |output_rect| used with ScaleToMultipleOutputs() should be divided by 158 // 4. 159 // Note 2: This requires a GL context that supports multiple render 160 // targets with at least two draw buffers. 161 NV61, 162 163 // Deinterleave into two output textures. 164 // 165 // UVUV UVUV UUUU VVVV 166 // UVUV UVUV --> { UUUU + VVVV } 167 // UVUV UVUV UUUU VVVV 168 // 169 // Note: Because of this packing, the horizontal coordinates of the 170 // |output_rect| used with ScaleToMultipleOutputs() should be divided by 171 // 2. 172 // Note 2: This requires a GL context that supports multiple render 173 // targets with at least two draw buffers. 174 DEINTERLEAVE_PAIRWISE, 175 } export_format = ExportFormat::INTERLEAVED_QUADS; 176 177 // Optionally swizzle the ordering of the values in each output quad. If the 178 // output of the scaler is not going to be read back (e.g., used with 179 // glReadPixels()), simply leave these unchanged. Otherwise, changing this 180 // allows a read-back pipeline to use the native format of the platform to 181 // avoid having to perform extra "BGRA⇄RGBA swizzle" memcpy's. Usually, this 182 // should match the format to be used with glReadPixels(), and that should 183 // match the GL_IMPLEMENTATION_COLOR_READ_FORMAT. 184 GLenum swizzle[2] = { 185 GL_RGBA, // For |dest_texture_0|. 186 GL_RGBA, // For |dest_texture_1|. 187 }; 188 189 Parameters(); 190 Parameters(const Parameters& other); 191 ~Parameters(); 192 }; 193 194 explicit GLScaler(ContextProvider* context_provider); 195 196 ~GLScaler() final; 197 198 // Returns true if the GL context provides the necessary support for enabling 199 // precise color management (see Parameters::enable_precise_color_management). 200 bool SupportsPreciseColorManagement() const; 201 202 // Returns the maximum number of simultaneous drawing buffers supported by the 203 // GL context. Certain Parameters can only be used when this is more than 1. 204 int GetMaxDrawBuffersSupported() const; 205 206 // [Re]Configure the scaler with the given |new_params|. Returns true on 207 // success, or false on failure. 208 bool Configure(const Parameters& new_params) WARN_UNUSED_RESULT; 209 210 // Returns the currently-configured and resolved Parameters. Note that these 211 // Parameters might not be exactly the same as those that were passed to 212 // Configure() because some properties (e.g., color spaces) are auto-resolved; 213 // however, ParametersAreEquivalent() will still return true. Results are 214 // undefined if Configure() has never been called successfully. params()215 const Parameters& params() const { return params_; } 216 217 // Scales a portion of |src_texture| and draws the result into |dest_texture| 218 // at offset (0, 0). Returns true to indicate success, or false if this 219 // GLScaler is not valid. 220 // 221 // |src_texture_size| is the full, allocated size of the |src_texture|. This 222 // is required for computing texture coordinate transforms (and only because 223 // the OpenGL ES 2.0 API lacks the ability to query this info). 224 // 225 // |src_offset| is the offset in the source texture corresponding to point 226 // (0,0) in the source/output coordinate spaces. This prevents the need for 227 // extra texture copies just to re-position the source coordinate system. 228 // 229 // |output_rect| selects the region to draw (in the scaled, not the source, 230 // coordinate space). This is used to save work in cases where only a portion 231 // needs to be re-scaled. The implementation will back-compute, internally, to 232 // determine the region of the |src_texture| to sample. 233 // 234 // WARNING: The output will always be placed at (0, 0) in the |dest_texture|, 235 // and not at |output_rect.origin()|. 236 // 237 // Note that the |src_texture| will have the min/mag filter set to GL_LINEAR 238 // and wrap_s/t set to CLAMP_TO_EDGE in this call. Scale(GLuint src_texture,const gfx::Size & src_texture_size,const gfx::Vector2d & src_offset,GLuint dest_texture,const gfx::Rect & output_rect)239 bool Scale(GLuint src_texture, 240 const gfx::Size& src_texture_size, 241 const gfx::Vector2d& src_offset, 242 GLuint dest_texture, 243 const gfx::Rect& output_rect) WARN_UNUSED_RESULT { 244 return ScaleToMultipleOutputs(src_texture, src_texture_size, src_offset, 245 dest_texture, 0, output_rect); 246 } 247 248 // Same as above, but for use cases where there are two output textures drawn 249 // (see Parameters::ExportFormat). 250 bool ScaleToMultipleOutputs(GLuint src_texture, 251 const gfx::Size& src_texture_size, 252 const gfx::Vector2d& src_offset, 253 GLuint dest_texture_0, 254 GLuint dest_texture_1, 255 const gfx::Rect& output_rect) WARN_UNUSED_RESULT; 256 257 // Returns true if from:to represent the same scale ratio as that specified in 258 // |params|. 259 static bool ParametersHasSameScaleRatio(const Parameters& params, 260 const gfx::Vector2d& from, 261 const gfx::Vector2d& to); 262 263 // Returns true if configuring a GLScaler with either |a| or |b| will produce 264 // identical behaviors and results. 265 static bool ParametersAreEquivalent(const Parameters& a, const Parameters& b); 266 267 private: 268 friend class GLScalerOverscanPixelTest; 269 friend class GLScalerShaderPixelTest; 270 friend VIZ_COMMON_EXPORT std::ostream& operator<<(std::ostream&, 271 const GLScaler&); 272 273 using GLES2Interface = gpu::gles2::GLES2Interface; 274 275 enum Axis { HORIZONTAL = 0, VERTICAL = 1 }; 276 277 // The shaders used by each stage in the scaling pipeline. 278 enum class Shader : int8_t { 279 BILINEAR, 280 BILINEAR2, 281 BILINEAR3, 282 BILINEAR4, 283 BILINEAR2X2, 284 BICUBIC_UPSCALE, 285 BICUBIC_HALF_1D, 286 PLANAR_CHANNEL_0, 287 PLANAR_CHANNEL_1, 288 PLANAR_CHANNEL_2, 289 PLANAR_CHANNEL_3, 290 I422_NV61_MRT, 291 DEINTERLEAVE_PAIRWISE_MRT, 292 }; 293 294 // A cached, re-usable shader program that performs one step in the scaling 295 // pipeline. 296 class VIZ_COMMON_EXPORT ShaderProgram { 297 public: 298 ShaderProgram(GLES2Interface* gl, 299 Shader shader, 300 GLenum texture_type, 301 const gfx::ColorTransform* color_transform, 302 const GLenum swizzle[2]); 303 ~ShaderProgram(); 304 shader()305 Shader shader() const { return shader_; } texture_type()306 GLenum texture_type() const { return texture_type_; } 307 308 // UseProgram must be called with GL_ARRAY_BUFFER bound to a vertex 309 // attribute buffer. |src_texture_size| is the size of the entire source 310 // texture, regardless of which region is to be sampled. |src_rect| is the 311 // source region, not including overscan pixels past the edges. 312 // |primary_axis| determines whether multiple texture samplings occur in one 313 // direction or the other (for some shaders). Note that this cannot 314 // necessarily be determined by just comparing the src and dst sizes. 315 // |flip_y| causes the |src_rect| to be scanned upside-down, to produce a 316 // vertically-flipped result. 317 void UseProgram(const gfx::Size& src_texture_size, 318 const gfx::RectF& src_rect, 319 const gfx::Size& dst_size, 320 Axis primary_axis, 321 bool flip_y); 322 323 // GL_ARRAY_BUFFER data that must be bound when drawing with a 324 // ShaderProgram. These are the vertex attributes that will sweep the entire 325 // source area when executing the program. They represent triangle strip 326 // coordinates: The first two columns are (x,y) values interpolated to 327 // produce the vertex coordinates in object space, while the latter two 328 // columns are (s,t) values interpolated to produce the texture coordinates 329 // that correspond to the vertex coordinates. 330 static const GLfloat kVertexAttributes[16]; 331 332 private: 333 GLES2Interface* const gl_; 334 const Shader shader_; 335 const GLenum texture_type_; 336 337 // A program for copying a source texture into a destination texture. 338 const GLuint program_; 339 340 // The location of the position in the program. 341 GLint position_location_ = -1; 342 // The location of the texture coordinate in the program. 343 GLint texcoord_location_ = -1; 344 // The location of the source texture in the program. 345 GLint texture_location_ = -1; 346 // The location of the texture coordinate of the source rectangle in the 347 // program. 348 GLint src_rect_location_ = -1; 349 // Location of size of source image in pixels. 350 GLint src_pixelsize_location_ = -1; 351 // Location of vector for scaling ratio between source and dest textures. 352 GLint scaling_vector_location_ = -1; 353 354 DISALLOW_COPY_AND_ASSIGN(ShaderProgram); 355 }; 356 357 // One scaling stage in a chain of scaler pipeline stages. Each ScalerStage 358 // owns the previous ScalerStage in the chain: At execution time, a "working 359 // backwards" approach is used: The previous "input" stage renders an 360 // intermediate result that will be used as input for the current stage. 361 // 362 // Each ScalerStage caches textures and framebuffers to avoid reallocating 363 // them for each separate image scaling, which can be expensive on some 364 // platforms/drivers. 365 class VIZ_COMMON_EXPORT ScalerStage { 366 public: 367 ScalerStage(GLES2Interface* gl, 368 Shader shader, 369 Axis primary_axis, 370 const gfx::Vector2d& scale_from, 371 const gfx::Vector2d& scale_to); 372 ~ScalerStage(); 373 shader()374 Shader shader() const { return shader_; } scale_from()375 const gfx::Vector2d& scale_from() const { return scale_from_; } scale_to()376 const gfx::Vector2d& scale_to() const { return scale_to_; } 377 input_stage()378 ScalerStage* input_stage() const { return input_stage_.get(); } set_input_stage(std::unique_ptr<ScalerStage> stage)379 void set_input_stage(std::unique_ptr<ScalerStage> stage) { 380 input_stage_ = std::move(stage); 381 } take_input_stage()382 std::unique_ptr<ScalerStage> take_input_stage() { 383 return std::move(input_stage_); 384 } 385 shader_program()386 ShaderProgram* shader_program() const { return program_; } set_shader_program(ShaderProgram * program)387 void set_shader_program(ShaderProgram* program) { program_ = program; } 388 is_flipped_source()389 bool is_flipped_source() const { return is_flipped_source_; } set_is_flipped_source(bool flipped)390 void set_is_flipped_source(bool flipped) { is_flipped_source_ = flipped; } 391 flip_output()392 bool flip_output() const { return flip_output_; } set_flip_output(bool flip)393 void set_flip_output(bool flip) { flip_output_ = flip; } 394 395 void ScaleToMultipleOutputs(GLuint src_texture, 396 gfx::Size src_texture_size, 397 const gfx::Vector2d& src_offset, 398 GLuint dest_texture_0, 399 GLuint dest_texture_1, 400 const gfx::Rect& output_rect); 401 402 private: 403 friend class GLScalerOverscanPixelTest; 404 405 // Returns the given |output_rect| mapped to the input stage's coordinate 406 // system. 407 gfx::RectF ToSourceRect(const gfx::Rect& output_rect) const; 408 409 // Returns the given |source_rect| padded to include the overscan pixels the 410 // shader program will access. 411 gfx::Rect ToInputRect(gfx::RectF source_rect) const; 412 413 // Generates the intermediate texture and/or re-defines it if its size has 414 // changed. 415 void EnsureIntermediateTextureDefined(const gfx::Size& size); 416 417 GLES2Interface* const gl_; 418 const Shader shader_; 419 const Axis primary_axis_; 420 const gfx::Vector2d scale_from_; 421 const gfx::Vector2d scale_to_; 422 423 std::unique_ptr<ScalerStage> input_stage_; 424 ShaderProgram* program_ = nullptr; 425 bool is_flipped_source_ = false; 426 bool flip_output_ = false; 427 428 GLuint intermediate_texture_ = 0; 429 gfx::Size intermediate_texture_size_; 430 GLuint dest_framebuffer_ = 0; 431 432 DISALLOW_COPY_AND_ASSIGN(ScalerStage); 433 }; 434 435 // ContextLostObserver implementation. 436 void OnContextLost() final; 437 438 // Returns a cached ShaderProgram, creating one on-demand if necessary. 439 ShaderProgram* GetShaderProgram(Shader shader, 440 GLenum texture_type, 441 const gfx::ColorTransform* color_transform, 442 const GLenum swizzle[2]); 443 444 // Create a scaling chain using the bilinear shaders. 445 static std::unique_ptr<ScalerStage> CreateAGoodScalingChain( 446 gpu::gles2::GLES2Interface* gl, 447 const gfx::Vector2d& scale_from, 448 const gfx::Vector2d& scale_to); 449 450 // Create a scaling chain using the bicubic shaders. 451 static std::unique_ptr<ScalerStage> CreateTheBestScalingChain( 452 gpu::gles2::GLES2Interface* gl, 453 const gfx::Vector2d& scale_from, 454 const gfx::Vector2d& scale_to); 455 456 // Modifies |chain| by appending an export stage, to rearrange the image data 457 // according to the requested |export_format|. In some cases, this will delete 458 // the final stage in |chain| before appending the export stage. 459 static std::unique_ptr<ScalerStage> MaybeAppendExportStage( 460 gpu::gles2::GLES2Interface* gl, 461 std::unique_ptr<ScalerStage> chain, 462 Parameters::ExportFormat export_format); 463 464 // Returns the other of the two axes. 465 static Axis TheOtherAxis(Axis axis); 466 467 // Returns the name of the |shader| in string form, for logging purposes. 468 static const char* GetShaderName(Shader shader); 469 470 // Returns true if the given |gl| context mentions all of |names| in its 471 // extensions string. 472 static bool AreAllGLExtensionsPresent(gpu::gles2::GLES2Interface* gl, 473 const std::vector<std::string>& names); 474 475 // The provider of the GL context. This is non-null while the GL context is 476 // valid and GLScaler is observing for context loss. 477 ContextProvider* context_provider_; 478 479 // Set by Configure() to the resolved set of Parameters. 480 Parameters params_; 481 482 // If set to true, half-float textures are supported. This is lazy-initialized 483 // by SupportsPreciseColorManagement(). 484 mutable base::Optional<bool> supports_half_floats_; 485 486 // The maximum number of simultaneous draw buffers, lazy-initialized by 487 // GetMaxDrawBuffersSupported(). -1 means "not yet known." 488 mutable int max_draw_buffers_ = -1; 489 490 // Cache of ShaderPrograms. The cache key consists of fields that correspond 491 // to the arguments of GetShaderProgram(): the shader, the texture format, the 492 // source and output color spaces (color transform), and the two swizzles. 493 using ShaderCacheKey = std:: 494 tuple<Shader, GLenum, gfx::ColorSpace, gfx::ColorSpace, GLenum, GLenum>; 495 std::map<ShaderCacheKey, ShaderProgram> shader_programs_; 496 497 // The GL_ARRAY_BUFFER that holds the vertices and the texture coordinates 498 // data for sweeping the source area when a ScalerStage draws a quad (to 499 // execute its shader program). 500 GLuint vertex_attributes_buffer_ = 0; 501 502 // The chain of ScalerStages. 503 std::unique_ptr<ScalerStage> chain_; 504 505 // The color space in which the scaling stages operate. 506 gfx::ColorSpace scaling_color_space_; 507 508 DISALLOW_COPY_AND_ASSIGN(GLScaler); 509 }; 510 511 // For logging. 512 VIZ_COMMON_EXPORT std::ostream& operator<<(std::ostream& out, 513 const GLScaler& scaler); 514 515 } // namespace viz 516 517 #endif // COMPONENTS_VIZ_COMMON_GL_SCALER_H_ 518