1 #ifndef _MOVIT_EFFECT_H 2 #define _MOVIT_EFFECT_H 1 3 4 // Effect is the base class for every effect. It basically represents a single 5 // GLSL function, with an optional set of user-settable parameters. 6 // 7 // A note on naming: Since all effects run in the same GLSL namespace, 8 // you can't use any name you want for global variables (e.g. uniforms). 9 // The framework assigns a prefix to you which will be unique for each 10 // effect instance; use the macro PREFIX() around your identifiers to 11 // automatically prepend that prefix. 12 13 #include <epoxy/gl.h> 14 #include <assert.h> 15 #include <stddef.h> 16 #include <map> 17 #include <string> 18 #include <vector> 19 #include <Eigen/Core> 20 21 #include "defs.h" 22 23 namespace movit { 24 25 class EffectChain; 26 class Node; 27 28 // Can alias on a float[2]. 29 struct Point2D { Point2DPoint2D30 Point2D() {} Point2DPoint2D31 Point2D(float x, float y) 32 : x(x), y(y) {} 33 34 float x, y; 35 }; 36 37 // Can alias on a float[3]. 38 struct RGBTriplet { RGBTripletRGBTriplet39 RGBTriplet() {} RGBTripletRGBTriplet40 RGBTriplet(float r, float g, float b) 41 : r(r), g(g), b(b) {} 42 43 float r, g, b; 44 }; 45 46 // Can alias on a float[4]. 47 struct RGBATuple { RGBATupleRGBATuple48 RGBATuple() {} RGBATupleRGBATuple49 RGBATuple(float r, float g, float b, float a) 50 : r(r), g(g), b(b), a(a) {} 51 52 float r, g, b, a; 53 }; 54 55 // Represents a registered uniform. 56 template<class T> 57 struct Uniform { 58 std::string name; // Without prefix. 59 const T *value; // Owner by the effect. 60 size_t num_values; // Number of elements; for arrays only. _Not_ the vector length. 61 std::string prefix; // Filled in only after phases have been constructed. 62 GLint location; // Filled in only after phases have been constructed. -1 if no location. 63 }; 64 65 class Effect { 66 public: ~Effect()67 virtual ~Effect() {} 68 69 // An identifier for this type of effect, mostly used for debug output 70 // (but some special names, like "ColorspaceConversionEffect", holds special 71 // meaning). Same as the class name is fine. 72 virtual std::string effect_type_id() const = 0; 73 74 // Whether this effects expects its input (and output) to be in 75 // linear gamma, ie. without an applied gamma curve. Most effects 76 // will want this, although the ones that never actually look at 77 // the pixels, e.g. mirror, won't need to care, and can set this 78 // to false. If so, the input gamma will be undefined. 79 // 80 // Also see the note on needs_texture_bounce(), below. needs_linear_light()81 virtual bool needs_linear_light() const { return true; } 82 83 // Whether this effect expects its input to be in the sRGB 84 // color space, ie. use the sRGB/Rec. 709 RGB primaries. 85 // (If not, it would typically come in as some slightly different 86 // set of RGB primaries; you would currently not get YCbCr 87 // or something similar). 88 // 89 // Again, most effects will want this, but you can set it to false 90 // if you process each channel independently, equally _and_ 91 // in a linear fashion. needs_srgb_primaries()92 virtual bool needs_srgb_primaries() const { return true; } 93 94 // How this effect handles alpha, ie. what it outputs in its 95 // alpha channel. The choices are basically blank (alpha is always 1.0), 96 // premultiplied and postmultiplied. 97 // 98 // Premultiplied alpha is when the alpha value has been be multiplied 99 // into the three color components, so e.g. 100% red at 50% alpha 100 // would be (0.5, 0.0, 0.0, 0.5) instead of (1.0, 0.0, 0.0, 0.5) 101 // as it is stored in most image formats (postmultiplied alpha). 102 // The multiplication is taken to have happened in linear light. 103 // This is the most natural format for processing, and the default in 104 // most of Movit (just like linear light is). 105 // 106 // If you set INPUT_AND_OUTPUT_PREMULTIPLIED_ALPHA or 107 // INPUT_PREMULTIPLIED_ALPHA_KEEP_BLANK, all of your inputs 108 // (if any) are guaranteed to also be in premultiplied alpha. 109 // Otherwise, you can get postmultiplied or premultiplied alpha; 110 // you won't know. If you have multiple inputs, you will get the same 111 // (pre- or postmultiplied) for all inputs, although most likely, 112 // you will want to combine them in a premultiplied fashion anyway 113 // in that case. 114 enum AlphaHandling { 115 // Always outputs blank alpha (ie. alpha=1.0). Only appropriate 116 // for inputs that do not output an alpha channel. 117 // Blank alpha is special in that it can be treated as both 118 // pre- and postmultiplied. 119 OUTPUT_BLANK_ALPHA, 120 121 // Always outputs postmultiplied alpha. Only appropriate for inputs. 122 OUTPUT_POSTMULTIPLIED_ALPHA, 123 124 // Always outputs premultiplied alpha. As noted above, 125 // you will then also get all inputs in premultiplied alpha. 126 // If you set this, you should also set needs_linear_light(). 127 INPUT_AND_OUTPUT_PREMULTIPLIED_ALPHA, 128 129 // Like INPUT_AND_OUTPUT_PREMULTIPLIED_ALPHA, but also guarantees 130 // that if you get blank alpha in, you also keep blank alpha out. 131 // This is a somewhat weaker guarantee than DONT_CARE_ALPHA_TYPE, 132 // but is still useful in many situations, and appropriate when 133 // e.g. you don't touch alpha at all. 134 // 135 // Does not make sense for inputs. 136 INPUT_PREMULTIPLIED_ALPHA_KEEP_BLANK, 137 138 // Keeps the type of alpha (premultiplied, postmultiplied, blank) 139 // unchanged from input to output. Usually appropriate if you 140 // process all color channels in a linear fashion, do not change 141 // alpha, and do not produce any new pixels that have alpha != 1.0. 142 // 143 // Does not make sense for inputs. 144 DONT_CARE_ALPHA_TYPE, 145 }; alpha_handling()146 virtual AlphaHandling alpha_handling() const { return INPUT_AND_OUTPUT_PREMULTIPLIED_ALPHA; } 147 148 // Whether this effect expects its input to come directly from 149 // a texture. If this is true, the framework will not chain the 150 // input from other effects, but will store the results of the 151 // chain to a temporary (RGBA fp16) texture and let this effect 152 // sample directly from that. 153 // 154 // There are two good reasons why you might want to set this: 155 // 156 // 1. You are sampling more than once from the input, 157 // in which case computing all the previous steps might 158 // be more expensive than going to a memory intermediate. 159 // 2. You rely on previous effects, possibly including gamma 160 // expansion, to happen pre-filtering instead of post-filtering. 161 // (This is only relevant if you actually need the filtering; if 162 // you sample 1:1 between pixels and texels, it makes no difference.) 163 // 164 // Note that in some cases, you might get post-filtered gamma expansion 165 // even when setting this option. More specifically, if you are the 166 // first effect in the chain, and the GPU is doing sRGB gamma 167 // expansion, it is undefined (from OpenGL's side) whether expansion 168 // happens pre- or post-filtering. For most uses, however, 169 // either will be fine. needs_texture_bounce()170 virtual bool needs_texture_bounce() const { return false; } 171 172 // Whether this effect expects mipmaps or not. 173 enum MipmapRequirements { 174 // If chosen, you will be sampling with bilinear filtering, 175 // ie. the closest mipmap will be chosen, and then there will be 176 // bilinear interpolation inside it (GL_LINEAR_MIPMAP_NEAREST). 177 NEEDS_MIPMAPS, 178 179 // Whether the effect doesn't really care whether input textures 180 // are with or without mipmaps. You could get the same effect 181 // as NEEDS_MIPMAPS or CANNOT_ACCEPT_MIPMAPS; normally, you won't 182 // get them, but if a different effect in the same phase needs mipmaps, 183 // you will also get them. 184 DOES_NOT_NEED_MIPMAPS, 185 186 // The opposite of NEEDS_MIPMAPS; you will always be sampling from 187 // the most detailed mip level (GL_LINEAR). Effects with NEEDS_MIPMAPS 188 // and CANNOT_ACCEPT_MIPMAPS can not coexist within the same phase; 189 // such phases will be split. 190 // 191 // This is the only choice that makes sense for a compute shader, 192 // given that it doesn't have screen-space derivatives and thus 193 // always will sample the most detailed mip level. 194 CANNOT_ACCEPT_MIPMAPS, 195 }; needs_mipmaps()196 virtual MipmapRequirements needs_mipmaps() const { 197 if (is_compute_shader()) { 198 return CANNOT_ACCEPT_MIPMAPS; 199 } else { 200 return DOES_NOT_NEED_MIPMAPS; 201 } 202 } 203 204 // Whether there is a direct correspondence between input and output 205 // texels. Specifically, the effect must not: 206 // 207 // 1. Try to sample in the border (ie., outside the 0.0 to 1.0 area). 208 // 2. Try to sample between texels. 209 // 3. Sample with an x- or y-derivative different from -1 or 1. 210 // (This also means needs_mipmaps() and one_to_one_sampling() 211 // together would make no sense.) 212 // 213 // The most common case for this would be an effect that has an exact 214 // 1:1-correspondence between input and output texels, e.g. SaturationEffect. 215 // However, more creative things, like mirroring/flipping or padding, 216 // would also be allowed. 217 // 218 // The primary gain from setting this is that you can sample directly 219 // from an effect that changes output size (see changes_output_size() below), 220 // without going through a bounce texture. It won't work for effects that 221 // set sets_virtual_output_size(), though. 222 // 223 // Does not make a lot of sense together with needs_texture_bounce(). 224 // Cannot be set for compute shaders. one_to_one_sampling()225 virtual bool one_to_one_sampling() const { return strong_one_to_one_sampling(); } 226 227 // Similar in use to one_to_one_sampling(), but even stricter: 228 // The effect must not modify texture coordinate in any way when 229 // calling its input(s). This allows it to also be used after 230 // a compute shader, in the same phase. 231 // 232 // An effect that it strong one-to-one must also be one-to-one. strong_one_to_one_sampling()233 virtual bool strong_one_to_one_sampling() const { return false; } 234 235 // Whether this effect wants to output to a different size than 236 // its input(s) (see inform_input_size(), below). See also 237 // sets_virtual_output_size() below. changes_output_size()238 virtual bool changes_output_size() const { return false; } 239 240 // Whether your get_output_size() function (see below) intends to ever set 241 // virtual_width different from width, or similar for height. 242 // It does not make sense to set this to true if changes_output_size() is false. sets_virtual_output_size()243 virtual bool sets_virtual_output_size() const { return changes_output_size(); } 244 245 // Whether this effect is effectively sampling from a a single texture. 246 // If so, it will override needs_texture_bounce(); however, there are also 247 // two demands it needs to fulfill: 248 // 249 // 1. It needs to be an Input, ie. num_inputs() == 0. 250 // 2. It needs to allocate exactly one sampler in set_gl_state(), 251 // and allow dependent effects to change that sampler state. is_single_texture()252 virtual bool is_single_texture() const { return false; } 253 254 // If set, this effect should never be bounced to an output, even if a 255 // dependent effect demands texture bounce. 256 // 257 // Note that setting this can invoke undefined behavior, up to and including crashing, 258 // so you should only use it if you have deep understanding of your entire chain 259 // and Movit's processing of it. The most likely use case is if you have an input 260 // that's cheap to compute but not a single texture (e.g. YCbCrInput), and want 261 // to run a ResampleEffect directly from it. Normally, this would require a bounce, 262 // but it's faster not to. (However, also note that in this case, effective texel 263 // subpixel precision will be too optimistic, since chroma is already subsampled.) 264 // 265 // Has no effect if is_single_texture() is set. override_disable_bounce()266 virtual bool override_disable_bounce() const { return false; } 267 268 // If changes_output_size() is true, you must implement this to tell 269 // the framework what output size you want. Also, you can set a 270 // virtual width/height, which is the size the next effect (if any) 271 // will _think_ your data is in. This is primarily useful if you are 272 // relying on getting OpenGL's bilinear resizing for free; otherwise, 273 // your virtual_width/virtual_height should be the same as width/height. 274 // 275 // Note that it is explicitly allowed to change width and height 276 // from frame to frame; EffectChain will reallocate textures as needed. get_output_size(unsigned * width,unsigned * height,unsigned * virtual_width,unsigned * virtual_height)277 virtual void get_output_size(unsigned *width, unsigned *height, 278 unsigned *virtual_width, unsigned *virtual_height) const { 279 assert(false); 280 } 281 282 // Whether this effect uses a compute shader instead of a regular fragment shader. 283 // Compute shaders are more flexible in that they can have multiple outputs 284 // for each invocation and also communicate between instances (by using shared 285 // memory within each group), but are not universally supported. The typical 286 // pattern would be to check movit_compute_shaders_supported and rewrite the 287 // graph to use a compute shader effect instead of a regular effect if it is 288 // available, in order to get better performance. Since compute shaders can reuse 289 // loads (again typically through shared memory), using needs_texture_bounce() 290 // is usually not needed, although it is allowed; the best candidates for compute 291 // shaders are typically those that sample many times from their input 292 // but can reuse those loads across neighboring instances. 293 // 294 // Compute shaders commonly work with unnormalized texture coordinates 295 // (where coordinates are integers [0..W) and [0..H)), whereas the rest 296 // of Movit, including any inputs you may want to sample from, works 297 // with normalized coordinates ([0..1)). Movit gives you uniforms 298 // PREFIX(inv_output_size) and PREFIX(output_texcoord_adjust) that you 299 // can use to transform unnormalized to normalized, as well as a macro 300 // NORMALIZE_TEXTURE_COORDS(vec2) that does it for you. 301 // 302 // Since compute shaders have flexible output, it is difficult to chain other 303 // effects after them in the same phase, and thus, they will always be last. 304 // (This limitation may be lifted for the special case of one-to-one effects 305 // in the future.) Furthermore, they cannot write to the framebuffer, just to 306 // textures, so Movit may have to insert an extra phase just to do the output 307 // from a texture to the screen in some cases. However, this is transparent 308 // to both the effect and the user. is_compute_shader()309 virtual bool is_compute_shader() const { return false; } 310 311 // For a compute shader (see the previous member function), what dimensions 312 // it should be invoked over. Called every frame, before uniforms are set 313 // (so you are allowed to update uniforms based from this call). get_compute_dimensions(unsigned output_width,unsigned output_height,unsigned * x,unsigned * y,unsigned * z)314 virtual void get_compute_dimensions(unsigned output_width, unsigned output_height, 315 unsigned *x, unsigned *y, unsigned *z) const { 316 *x = output_width; 317 *y = output_height; 318 *z = 1; 319 } 320 321 // Tells the effect the resolution of each of its input. 322 // This will be called every frame, and always before get_output_size(), 323 // so you can change your output size based on the input if so desired. 324 // 325 // Note that in some cases, an input might not have a single well-defined 326 // resolution (for instance if you fade between two inputs with 327 // different resolutions). In this case, you will get width=0 and height=0 328 // for that input. If you cannot handle that, you will need to set 329 // needs_texture_bounce() to true, which will force a render to a single 330 // given resolution before you get the input. inform_input_size(unsigned input_num,unsigned width,unsigned height)331 virtual void inform_input_size(unsigned input_num, unsigned width, unsigned height) {} 332 333 // How many inputs this effect will take (a fixed number). 334 // If you have only one input, it will be called INPUT() in GLSL; 335 // if you have several, they will be INPUT1(), INPUT2(), and so on. num_inputs()336 virtual unsigned num_inputs() const { return 1; } 337 338 // Inform the effect that it has been just added to the EffectChain. 339 // The primary use for this is to store the ResourcePool uesd by 340 // the chain; for modifications to it, rewrite_graph() below 341 // is probably a better fit. inform_added(EffectChain * chain)342 virtual void inform_added(EffectChain *chain) {} 343 344 // Let the effect rewrite the effect chain as it sees fit. 345 // Most effects won't need to do this, but this is very useful 346 // if you have an effect that consists of multiple sub-effects 347 // (for instance, two passes). The effect is given to its own 348 // pointer, and it can add new ones (by using add_node() 349 // and connect_node()) as it sees fit. This is called at 350 // EffectChain::finalize() time, when the entire graph is known, 351 // in the order that the effects were originally added. 352 // 353 // Note that if the effect wants to take itself entirely out 354 // of the chain, it must set “disabled” to true and then disconnect 355 // itself from all other effects. rewrite_graph(EffectChain * graph,Node * self)356 virtual void rewrite_graph(EffectChain *graph, Node *self) {} 357 358 // Returns the GLSL fragment shader string for this effect. 359 virtual std::string output_fragment_shader() = 0; 360 361 // Set all OpenGL state that this effect needs before rendering. 362 // The default implementation sets one uniform per registered parameter, 363 // but no other state. 364 // 365 // <sampler_num> is the first free texture sampler. If you want to use 366 // textures, you can bind a texture to GL_TEXTURE0 + <sampler_num>, 367 // and then increment the number (so that the next effect in the chain 368 // will use a different sampler). 369 virtual void set_gl_state(GLuint glsl_program_num, const std::string& prefix, unsigned *sampler_num); 370 371 // If you set any special OpenGL state in set_gl_state(), you can clear it 372 // after rendering here. The default implementation does nothing. 373 virtual void clear_gl_state(); 374 375 // Set a parameter; intended to be called from user code. 376 // Neither of these take ownership of the pointer. 377 virtual bool set_int(const std::string &key, int value) MUST_CHECK_RESULT; 378 virtual bool set_ivec2(const std::string &key, const int *values) MUST_CHECK_RESULT; 379 virtual bool set_float(const std::string &key, float value) MUST_CHECK_RESULT; 380 virtual bool set_vec2(const std::string &key, const float *values) MUST_CHECK_RESULT; 381 virtual bool set_vec3(const std::string &key, const float *values) MUST_CHECK_RESULT; 382 virtual bool set_vec4(const std::string &key, const float *values) MUST_CHECK_RESULT; 383 384 protected: 385 // Register a parameter. Whenever set_*() is called with the same key, 386 // it will update the value in the given pointer (typically a pointer 387 // to some private member variable in your effect). It will also 388 // register a uniform of the same name (plus an arbitrary prefix 389 // which you can access using the PREFIX macro) that you can access. 390 // 391 // Neither of these take ownership of the pointer. 392 393 // These correspond directly to int/float/vec2/vec3/vec4 in GLSL. 394 void register_int(const std::string &key, int *value); 395 void register_ivec2(const std::string &key, int *values); 396 void register_float(const std::string &key, float *value); 397 void register_vec2(const std::string &key, float *values); 398 void register_vec3(const std::string &key, float *values); 399 void register_vec4(const std::string &key, float *values); 400 401 // Register uniforms, such that they will automatically be set 402 // before the shader runs. This is more efficient than set_uniform_* 403 // in effect_util.h, because it doesn't need to do name lookups 404 // every time. Also, in the future, it will use uniform buffer objects 405 // (UBOs) if available to reduce the number of calls into the driver. 406 // 407 // May not be called after output_fragment_shader() has returned. 408 // The pointer must be valid for the entire lifetime of the Effect, 409 // since the value is pulled from it each execution. The value is 410 // guaranteed to be read after set_gl_state() for the effect has 411 // returned, so you can safely update its value from there. 412 // 413 // Note that this will also declare the uniform in the shader for you, 414 // so you should not do that yourself. (This is so it can be part of 415 // the right uniform block.) However, it is probably a good idea to 416 // have a commented-out declaration so that it is easier to see the 417 // type and thus understand the shader on its own. 418 // 419 // Calling register_* will automatically imply register_uniform_*, 420 // except for register_int as noted above. 421 void register_uniform_sampler2d(const std::string &key, const int *value); 422 void register_uniform_bool(const std::string &key, const bool *value); 423 void register_uniform_int(const std::string &key, const int *value); 424 void register_uniform_ivec2(const std::string &key, const int *values); 425 void register_uniform_float(const std::string &key, const float *value); 426 void register_uniform_vec2(const std::string &key, const float *values); 427 void register_uniform_vec3(const std::string &key, const float *values); 428 void register_uniform_vec4(const std::string &key, const float *values); 429 void register_uniform_float_array(const std::string &key, const float *values, size_t num_values); 430 void register_uniform_vec2_array(const std::string &key, const float *values, size_t num_values); 431 void register_uniform_vec3_array(const std::string &key, const float *values, size_t num_values); 432 void register_uniform_vec4_array(const std::string &key, const float *values, size_t num_values); 433 void register_uniform_mat3(const std::string &key, const Eigen::Matrix3d *matrix); 434 435 private: 436 std::map<std::string, int *> params_int; 437 std::map<std::string, int *> params_ivec2; 438 std::map<std::string, float *> params_float; 439 std::map<std::string, float *> params_vec2; 440 std::map<std::string, float *> params_vec3; 441 std::map<std::string, float *> params_vec4; 442 443 // Picked out by EffectChain during finalization. 444 std::vector<Uniform<int>> uniforms_image2d; 445 std::vector<Uniform<int>> uniforms_sampler2d; 446 std::vector<Uniform<bool>> uniforms_bool; 447 std::vector<Uniform<int>> uniforms_int; 448 std::vector<Uniform<int>> uniforms_ivec2; 449 std::vector<Uniform<float>> uniforms_float; 450 std::vector<Uniform<float>> uniforms_vec2; 451 std::vector<Uniform<float>> uniforms_vec3; 452 std::vector<Uniform<float>> uniforms_vec4; 453 std::vector<Uniform<float>> uniforms_float_array; 454 std::vector<Uniform<float>> uniforms_vec2_array; 455 std::vector<Uniform<float>> uniforms_vec3_array; 456 std::vector<Uniform<float>> uniforms_vec4_array; 457 std::vector<Uniform<Eigen::Matrix3d>> uniforms_mat3; 458 friend class EffectChain; 459 }; 460 461 } // namespace movit 462 463 #endif // !defined(_MOVIT_EFFECT_H) 464