1/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2/* This Source Code Form is subject to the terms of the Mozilla Public 3 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 * You can obtain one at http://mozilla.org/MPL/2.0/. 5 * 6 * The origin of this IDL file is 7 * https://html.spec.whatwg.org/multipage/webappapis.html#images 8 * 9 * The origin of the extended IDL file is 10 * http://w3c.github.io/mediacapture-worker/#imagebitmap-extensions 11 */ 12 13// Extensions 14// Bug 1141979 - [FoxEye] Extend ImageBitmap with interfaces to access its 15// underlying image data 16// 17// Note: 18// Our overload resolution implementation doesn't deal with a union as the 19// distinguishing argument which means we cannot overload functions via union 20// types, a.k.a. we cannot overload createImageBitmap() via ImageBitmapSource 21// and BufferSource. Here, we work around this issue by adding the BufferSource 22// into ImageBitmapSource. 23 24typedef (HTMLImageElement or 25 HTMLVideoElement or 26 HTMLCanvasElement or 27 Blob or 28 ImageData or 29 CanvasRenderingContext2D or 30 ImageBitmap or 31 BufferSource) ImageBitmapSource; 32 33[Exposed=(Window,Worker)] 34interface ImageBitmap { 35 [Constant] 36 readonly attribute unsigned long width; 37 [Constant] 38 readonly attribute unsigned long height; 39}; 40 41// It's crucial that there be a way to explicitly dispose of ImageBitmaps 42// since they refer to potentially large graphics resources. Some uses 43// of this API proposal will result in repeated allocations of ImageBitmaps, 44// and garbage collection will not reliably reclaim them quickly enough. 45// Here we reuse close(), which also exists on another Transferable type, 46// MessagePort. Potentially, all Transferable types should inherit from a 47// new interface type "Closeable". 48partial interface ImageBitmap { 49 // Dispose of all graphical resources associated with this ImageBitmap. 50 undefined close(); 51}; 52 53// ImageBitmap-extensions 54// Bug 1141979 - [FoxEye] Extend ImageBitmap with interfaces to access its 55// underlying image data 56 57/* 58 * An image or a video frame is conceptually a two-dimensional array of data and 59 * each element in the array is called a pixel. The pixels are usually stored in 60 * a one-dimensional array and could be arranged in a variety of image formats. 61 * Developers need to know how the pixels are formatted so that they are able to 62 * process them. 63 * 64 * The image format describes how pixels in an image are arranged. A single 65 * pixel has at least one, but usually multiple pixel values. The range of a 66 * pixel value varies, which means different image formats use different data 67 * types to store a single pixel value. 68 * 69 * The most frequently used data type is 8-bit unsigned integer whose range is 70 * from 0 to 255, others could be 16-bit integer or 32-bit floating points and 71 * so forth. The number of pixel values of a single pixel is called the number 72 * of channels of the image format. Multiple pixel values of a pixel are used 73 * together to describe the captured property which could be color or depth 74 * information. For example, if the data is a color image in RGB color space, 75 * then it is a three-channel image format and a pixel is described by R, G and 76 * B three pixel values with range from 0 to 255. As another example, if the 77 * data is a gray image, then it is a single-channel image format with 8-bit 78 * unsigned integer data type and the pixel value describes the gray scale. For 79 * depth data, it is a single channel image format too, but the data type is 80 * 16-bit unsigned integer and the pixel value is the depth level. 81 * 82 * For those image formats whose pixels contain multiple pixel values, the pixel 83 * values might be arranged in one of the following ways: 84 * 1) Planar pixel layout: 85 * each channel has its pixel values stored consecutively in separated 86 * buffers (a.k.a. planes) and then all channel buffers are stored 87 * consecutively in memory. 88 * (Ex: RRRRRR......GGGGGG......BBBBBB......) 89 * 2) Interleaving pixel layout: 90 * each pixel has its pixel values from all channels stored together and 91 * interleaves all channels. 92 * (Ex: RGBRGBRGBRGBRGB......) 93 */ 94 95 96/* 97 * The ImageBitmap extensions use this enumeration to negotiate the image format 98 * while 1) accessing the underlying data of an ImageBitmap and 99 * 2) creating a new ImageBitmap. 100 * 101 * For each format in this enumeration, we use a 2x2 small image (4 pixels) as 102 * example to illustrate the pixel layout. 103 * 104 * 2x2 image: +--------+--------+ 105 * | pixel1 | pixel2 | 106 * +--------+--------+ 107 * | pixel3 | pixel4 | 108 * +--------+--------+ 109 * 110 */ 111enum ImageBitmapFormat { 112 /* 113 * Channel order: R, G, B, A 114 * Channel size: full rgba-chennels 115 * Pixel layout: interleaving rgba-channels 116 * Pixel layout illustration: 117 * [Plane1]: R1 G1 B1 A1 R2 G2 B2 A2 R3 G3 B3 A3 R4 G4 B4 A4 118 * Data type: 8-bit unsigned integer 119 */ 120 "RGBA32", 121 122 /* 123 * Channel order: B, G, R, A 124 * Channel size: full bgra-channels 125 * Pixel layout: interleaving bgra-channels 126 * Pixel layout illustration: 127 * [Plane1]: B1 G1 R1 A1 B2 G2 R2 A2 B3 G3 R3 A3 B4 G4 R4 A4 128 * Data type: 8-bit unsigned integer 129 */ 130 "BGRA32", 131 132 /* 133 * Channel order: R, G, B 134 * Channel size: full rgb-channels 135 * Pixel layout: interleaving rgb-channels 136 * Pixel layout illustration: 137 * [Plane1]: R1 G1 B1 R2 G2 B2 R3 G3 B3 R4 G4 B4 138 * Data type: 8-bit unsigned integer 139 */ 140 "RGB24", 141 142 /* 143 * Channel order: B, G, R 144 * Channel size: full bgr-channels 145 * Pixel layout: interleaving bgr-channels 146 * Pixel layout illustration: 147 * [Plane1]: B1 G1 R1 B2 G2 R2 B3 G3 R3 B4 G4 R4 148 * Data type: 8-bit unsigned integer 149 */ 150 "BGR24", 151 152 /* 153 * Channel order: GRAY 154 * Channel size: full gray-channel 155 * Pixel layout: planar gray-channel 156 * Pixel layout illustration: 157 * [Plane1]: GRAY1 GRAY2 GRAY3 GRAY4 158 * Data type: 8-bit unsigned integer 159 */ 160 "GRAY8", 161 162 /* 163 * Channel order: Y, U, V 164 * Channel size: full yuv-channels 165 * Pixel layout: planar yuv-channels 166 * Pixel layout illustration: 167 * [Plane1]: Y1 Y2 Y3 Y4 168 * [Plane2]: U1 U2 U3 U4 169 * [Plane3]: V1 V2 V3 V4 170 * Data type: 8-bit unsigned integer 171 */ 172 "YUV444P", 173 174 /* 175 * Channel order: Y, U, V 176 * Channel size: full y-channel, half uv-channels 177 * Pixel layout: planar yuv-channels 178 * Pixel layout illustration: 179 * [Plane1]: Y1 Y2 Y3 Y4 180 * [Plane2]: U1 U3 181 * [Plane3]: V1 V3 182 * Data type: 8-bit unsigned integer 183 */ 184 "YUV422P", 185 186 /* 187 * Channel order: Y, U, V 188 * Channel size: full y-channel, quarter uv-channels 189 * Pixel layout: planar yuv-channels 190 * Pixel layout illustration: 191 * [Plane1]: Y1 Y2 Y3 Y4 192 * [Plane2]: U1 193 * [Plane3]: V1 194 * Data type: 8-bit unsigned integer 195 */ 196 "YUV420P", 197 198 /* 199 * Channel order: Y, U, V 200 * Channel size: full y-channel, quarter uv-channels 201 * Pixel layout: planar y-channel, interleaving uv-channels 202 * Pixel layout illustration: 203 * [Plane1]: Y1 Y2 Y3 Y4 204 * [Plane2]: U1 V1 205 * Data type: 8-bit unsigned integer 206 */ 207 "YUV420SP_NV12", 208 209 /* 210 * Channel order: Y, V, U 211 * Channel size: full y-channel, quarter vu-channels 212 * Pixel layout: planar y-channel, interleaving vu-channels 213 * Pixel layout illustration: 214 * [Plane1]: Y1 Y2 Y3 Y4 215 * [Plane2]: V1 U1 216 * Data type: 8-bit unsigned integer 217 */ 218 "YUV420SP_NV21", 219 220 /* 221 * Channel order: H, S, V 222 * Channel size: full hsv-channels 223 * Pixel layout: interleaving hsv-channels 224 * Pixel layout illustration: 225 * [Plane1]: H1 S1 V1 H2 S2 V2 H3 S3 V3 226 * Data type: 32-bit floating point value 227 */ 228 "HSV", 229 230 /* 231 * Channel order: l, a, b 232 * Channel size: full lab-channels 233 * Pixel layout: interleaving lab-channels 234 * Pixel layout illustration: 235 * [Plane1]: l1 a1 b1 l2 a2 b2 l3 a3 b3 236 * Data type: 32-bit floating point value 237 */ 238 "Lab", 239 240 /* 241 * Channel order: DEPTH 242 * Channel size: full depth-channel 243 * Pixel layout: planar depth-channel 244 * Pixel layout illustration: 245 * [Plane1]: DEPTH1 DEPTH2 DEPTH3 DEPTH4 246 * Data type: 16-bit unsigned integer 247 */ 248 "DEPTH", 249}; 250 251enum ChannelPixelLayoutDataType { 252 "uint8", 253 "int8", 254 "uint16", 255 "int16", 256 "uint32", 257 "int32", 258 "float32", 259 "float64" 260}; 261 262/* 263 * Two concepts, ImagePixelLayout and ChannelPixelLayout, together generalize 264 * the variety of pixel layouts among image formats. 265 * 266 * The ChannelPixelLayout represents the pixel layout of a single channel in a 267 * certain image format and the ImagePixelLayout is just the collection of 268 * ChannelPixelLayouts. So, the ChannelPixelLayout is defined as a dictionary 269 * type with properties to describe the layout and the ImagePixelLayout is just 270 * an alias name to a sequence of ChannelPixelLayout objects. 271 * 272 * Since an image format is composed of at least one channel, an 273 * ImagePixelLayout object contains at least one ChannelPixelLayout object. 274 * 275 * Although an image or a video frame is a two-dimensional structure, its data 276 * is usually stored in a one-dimensional array in the row-major way and the 277 * ChannelPixelLayout objects use the following properties to describe the 278 * layout of pixel values in the buffer. 279 * 280 * 1) offset: 281 * denotes the beginning position of the channel's data relative to the 282 * beginning position of the one-dimensional array. 283 * 2) width & height: 284 * denote the width and height of the channel respectively. Each channel in 285 * an image format may have different height and width. 286 * 3) data type: 287 * denotes the format used to store one single pixel value. 288 * 4) stride: 289 * the number of bytes between the beginning two consecutive rows in memory. 290 * (The total bytes of each row plus the padding bytes of each raw.) 291 * 5) skip value: 292 * the value is zero for the planar pixel layout, and a positive integer for 293 * the interleaving pixel layout. (Describes how many bytes there are between 294 * two adjacent pixel values in this channel.) 295 */ 296 297/* 298 * Example1: RGBA image, width = 620, height = 480, stride = 2560 299 * 300 * chanel_r: offset = 0, width = 620, height = 480, data type = uint8, stride = 2560, skip = 3 301 * chanel_g: offset = 1, width = 620, height = 480, data type = uint8, stride = 2560, skip = 3 302 * chanel_b: offset = 2, width = 620, height = 480, data type = uint8, stride = 2560, skip = 3 303 * chanel_a: offset = 3, width = 620, height = 480, data type = uint8, stride = 2560, skip = 3 304 * 305 * <---------------------------- stride ----------------------------> 306 * <---------------------- width x 4 ----------------------> 307 * [index] 01234 8 12 16 20 24 28 2479 2559 308 * |||||---|---|---|---|---|---|----------------------------|-------| 309 * [data] RGBARGBARGBARGBARGBAR___R___R... A%%%%%%%% 310 * [data] RGBARGBARGBARGBARGBAR___R___R... A%%%%%%%% 311 * [data] RGBARGBARGBARGBARGBAR___R___R... A%%%%%%%% 312 * ^^^ 313 * r-skip 314 */ 315 316/* 317 * Example2: YUV420P image, width = 620, height = 480, stride = 640 318 * 319 * chanel_y: offset = 0, width = 620, height = 480, stride = 640, skip = 0 320 * chanel_u: offset = 307200, width = 310, height = 240, data type = uint8, stride = 320, skip = 0 321 * chanel_v: offset = 384000, width = 310, height = 240, data type = uint8, stride = 320, skip = 0 322 * 323 * <--------------------------- y-stride ---------------------------> 324 * <----------------------- y-width -----------------------> 325 * [index] 012345 619 639 326 * ||||||--------------------------------------------------|--------| 327 * [data] YYYYYYYYYYYYYYYYYYYYYYYYYYYYY... Y%%%%%%%%% 328 * [data] YYYYYYYYYYYYYYYYYYYYYYYYYYYYY... Y%%%%%%%%% 329 * [data] YYYYYYYYYYYYYYYYYYYYYYYYYYYYY... Y%%%%%%%%% 330 * [data] ...... 331 * <-------- u-stride ----------> 332 * <----- u-width -----> 333 * [index] 307200 307509 307519 334 * |-------------------|--------| 335 * [data] UUUUUUUUUU... U%%%%%%%%% 336 * [data] UUUUUUUUUU... U%%%%%%%%% 337 * [data] UUUUUUUUUU... U%%%%%%%%% 338 * [data] ...... 339 * <-------- v-stride ----------> 340 * <- --- v-width -----> 341 * [index] 384000 384309 384319 342 * |-------------------|--------| 343 * [data] VVVVVVVVVV... V%%%%%%%%% 344 * [data] VVVVVVVVVV... V%%%%%%%%% 345 * [data] VVVVVVVVVV... V%%%%%%%%% 346 * [data] ...... 347 */ 348 349/* 350 * Example3: YUV420SP_NV12 image, width = 620, height = 480, stride = 640 351 * 352 * chanel_y: offset = 0, width = 620, height = 480, stride = 640, skip = 0 353 * chanel_u: offset = 307200, width = 310, height = 240, data type = uint8, stride = 640, skip = 1 354 * chanel_v: offset = 307201, width = 310, height = 240, data type = uint8, stride = 640, skip = 1 355 * 356 * <--------------------------- y-stride --------------------------> 357 * <----------------------- y-width ----------------------> 358 * [index] 012345 619 639 359 * ||||||-------------------------------------------------|--------| 360 * [data] YYYYYYYYYYYYYYYYYYYYYYYYYYYYY... Y%%%%%%%%% 361 * [data] YYYYYYYYYYYYYYYYYYYYYYYYYYYYY... Y%%%%%%%%% 362 * [data] YYYYYYYYYYYYYYYYYYYYYYYYYYYYY... Y%%%%%%%%% 363 * [data] ...... 364 * <--------------------- u-stride / v-stride --------------------> 365 * <------------------ u-width + v-width -----------------> 366 * [index] 307200(u-offset) 307819 307839 367 * |------------------------------------------------------|-------| 368 * [index] |307201(v-offset) |307820 | 369 * ||-----------------------------------------------------||------| 370 * [data] UVUVUVUVUVUVUVUVUVUVUVUVUVUVUV... UV%%%%%%% 371 * [data] UVUVUVUVUVUVUVUVUVUVUVUVUVUVUV... UV%%%%%%% 372 * [data] UVUVUVUVUVUVUVUVUVUVUVUVUVUVUV... UV%%%%%%% 373 * ^ ^ 374 * u-skip v-skip 375 */ 376 377/* 378 * Example4: DEPTH image, width = 640, height = 480, stride = 1280 379 * 380 * chanel_d: offset = 0, width = 640, height = 480, data type = uint16, stride = 1280, skip = 0 381 * 382 * note: each DEPTH value uses two bytes 383 * 384 * <----------------------- d-stride ----------------------> 385 * <----------------------- d-width -----------------------> 386 * [index] 02468 1278 387 * |||||---------------------------------------------------| 388 * [data] DDDDDDDDDDDDDDDDDDDDDDDDDDDDD... D 389 * [data] DDDDDDDDDDDDDDDDDDDDDDDDDDDDD... D 390 * [data] DDDDDDDDDDDDDDDDDDDDDDDDDDDDD... D 391 * [data] ...... 392 */ 393 394dictionary ChannelPixelLayout { 395 required unsigned long offset; 396 required unsigned long width; 397 required unsigned long height; 398 required ChannelPixelLayoutDataType dataType; 399 required unsigned long stride; 400 required unsigned long skip; 401}; 402 403typedef sequence<ChannelPixelLayout> ImagePixelLayout; 404 405partial interface ImageBitmap { 406 [Throws, Func="mozilla::dom::DOMPrefs::ImageBitmapExtensionsEnabled"] 407 ImageBitmapFormat findOptimalFormat (optional sequence<ImageBitmapFormat> aPossibleFormats); 408 [Throws, Func="mozilla::dom::DOMPrefs::ImageBitmapExtensionsEnabled"] 409 long mappedDataLength (ImageBitmapFormat aFormat); 410 [Throws, Func="mozilla::dom::DOMPrefs::ImageBitmapExtensionsEnabled"] 411 Promise<ImagePixelLayout> mapDataInto (ImageBitmapFormat aFormat, BufferSource aBuffer, long aOffset); 412}; 413