1/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2/* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/.
5 *
6 * The origin of this IDL file is
7 * https://html.spec.whatwg.org/multipage/webappapis.html#images
8 *
9 * The origin of the extended IDL file is
10 * http://w3c.github.io/mediacapture-worker/#imagebitmap-extensions
11 */
12
13// Extensions
14// Bug 1141979 - [FoxEye] Extend ImageBitmap with interfaces to access its
15// underlying image data
16//
17// Note:
18// Our overload resolution implementation doesn't deal with a union as the
19// distinguishing argument which means we cannot overload functions via union
20// types, a.k.a. we cannot overload createImageBitmap() via ImageBitmapSource
21// and BufferSource. Here, we work around this issue by adding the BufferSource
22// into ImageBitmapSource.
23
24typedef (HTMLImageElement or
25         HTMLVideoElement or
26         HTMLCanvasElement or
27         Blob or
28         ImageData or
29         CanvasRenderingContext2D or
30         ImageBitmap or
31         BufferSource) ImageBitmapSource;
32
33[Exposed=(Window,Worker)]
34interface ImageBitmap {
35  [Constant]
36  readonly attribute unsigned long width;
37  [Constant]
38  readonly attribute unsigned long height;
39};
40
41// It's crucial that there be a way to explicitly dispose of ImageBitmaps
42// since they refer to potentially large graphics resources. Some uses
43// of this API proposal will result in repeated allocations of ImageBitmaps,
44// and garbage collection will not reliably reclaim them quickly enough.
45// Here we reuse close(), which also exists on another Transferable type,
46// MessagePort. Potentially, all Transferable types should inherit from a
47// new interface type "Closeable".
48partial interface ImageBitmap {
49  // Dispose of all graphical resources associated with this ImageBitmap.
50  undefined close();
51};
52
53// ImageBitmap-extensions
54// Bug 1141979 - [FoxEye] Extend ImageBitmap with interfaces to access its
55// underlying image data
56
57/*
58 * An image or a video frame is conceptually a two-dimensional array of data and
59 * each element in the array is called a pixel. The pixels are usually stored in
60 * a one-dimensional array and could be arranged in a variety of image formats.
61 * Developers need to know how the pixels are formatted so that they are able to
62 * process them.
63 *
64 * The image format describes how pixels in an image are arranged. A single
65 * pixel has at least one, but usually multiple pixel values. The range of a
66 * pixel value varies, which means different image formats use different data
67 * types to store a single pixel value.
68 *
69 * The most frequently used data type is 8-bit unsigned integer whose range is
70 * from 0 to 255, others could be 16-bit integer or 32-bit floating points and
71 * so forth. The number of pixel values of a single pixel is called the number
72 * of channels of the image format. Multiple pixel values of a pixel are used
73 * together to describe the captured property which could be color or depth
74 * information. For example, if the data is a color image in RGB color space,
75 * then it is a three-channel image format and a pixel is described by R, G and
76 * B three pixel values with range from 0 to 255. As another example, if the
77 * data is a gray image, then it is a single-channel image format with 8-bit
78 * unsigned integer data type and the pixel value describes the gray scale. For
79 * depth data, it is a single channel image format too, but the data type is
80 * 16-bit unsigned integer and the pixel value is the depth level.
81 *
82 * For those image formats whose pixels contain multiple pixel values, the pixel
83 * values might be arranged in one of the following ways:
84 * 1) Planar pixel layout:
85 *    each channel has its pixel values stored consecutively in separated
86 *    buffers (a.k.a. planes) and then all channel buffers are stored
87 *    consecutively in memory.
88 *    (Ex: RRRRRR......GGGGGG......BBBBBB......)
89 * 2) Interleaving pixel layout:
90 *    each pixel has its pixel values from all channels stored together and
91 *    interleaves all channels.
92 *    (Ex: RGBRGBRGBRGBRGB......)
93 */
94
95
96/*
97 * The ImageBitmap extensions use this enumeration to negotiate the image format
98 * while 1) accessing the underlying data of an ImageBitmap and
99 *       2) creating a new ImageBitmap.
100 *
101 * For each format in this enumeration, we use a 2x2 small image (4 pixels) as
102 * example to illustrate the pixel layout.
103 *
104 * 2x2 image:   +--------+--------+
105 *              | pixel1 | pixel2 |
106 *              +--------+--------+
107 *              | pixel3 | pixel4 |
108 *              +--------+--------+
109 *
110 */
111enum ImageBitmapFormat {
112  /*
113   * Channel order: R, G, B, A
114   * Channel size: full rgba-chennels
115   * Pixel layout: interleaving rgba-channels
116   * Pixel layout illustration:
117   *   [Plane1]: R1 G1 B1 A1 R2 G2 B2 A2 R3 G3 B3 A3 R4 G4 B4 A4
118   * Data type: 8-bit unsigned integer
119   */
120  "RGBA32",
121
122  /*
123   * Channel order: B, G, R, A
124   * Channel size: full bgra-channels
125   * Pixel layout: interleaving bgra-channels
126   * Pixel layout illustration:
127   *   [Plane1]: B1 G1 R1 A1 B2 G2 R2 A2 B3 G3 R3 A3 B4 G4 R4 A4
128   * Data type: 8-bit unsigned integer
129   */
130  "BGRA32",
131
132  /*
133   * Channel order: R, G, B
134   * Channel size: full rgb-channels
135   * Pixel layout: interleaving rgb-channels
136   * Pixel layout illustration:
137   *   [Plane1]: R1 G1 B1 R2 G2 B2 R3 G3 B3 R4 G4 B4
138   * Data type: 8-bit unsigned integer
139   */
140  "RGB24",
141
142  /*
143   * Channel order: B, G, R
144   * Channel size: full bgr-channels
145   * Pixel layout: interleaving bgr-channels
146   * Pixel layout illustration:
147   *   [Plane1]: B1 G1 R1 B2 G2 R2 B3 G3 R3 B4 G4 R4
148   * Data type: 8-bit unsigned integer
149   */
150  "BGR24",
151
152  /*
153   * Channel order: GRAY
154   * Channel size: full gray-channel
155   * Pixel layout: planar gray-channel
156   * Pixel layout illustration:
157   *   [Plane1]: GRAY1 GRAY2 GRAY3 GRAY4
158   * Data type: 8-bit unsigned integer
159   */
160  "GRAY8",
161
162  /*
163   * Channel order: Y, U, V
164   * Channel size: full yuv-channels
165   * Pixel layout: planar yuv-channels
166   * Pixel layout illustration:
167   *   [Plane1]: Y1 Y2 Y3 Y4
168   *   [Plane2]: U1 U2 U3 U4
169   *   [Plane3]: V1 V2 V3 V4
170   * Data type: 8-bit unsigned integer
171   */
172  "YUV444P",
173
174  /*
175   * Channel order: Y, U, V
176   * Channel size: full y-channel, half uv-channels
177   * Pixel layout: planar yuv-channels
178   * Pixel layout illustration:
179   *   [Plane1]: Y1 Y2 Y3 Y4
180   *   [Plane2]: U1 U3
181   *   [Plane3]: V1 V3
182   * Data type: 8-bit unsigned integer
183   */
184  "YUV422P",
185
186  /*
187   * Channel order: Y, U, V
188   * Channel size: full y-channel, quarter uv-channels
189   * Pixel layout: planar yuv-channels
190   * Pixel layout illustration:
191   *   [Plane1]: Y1 Y2 Y3 Y4
192   *   [Plane2]: U1
193   *   [Plane3]: V1
194   * Data type: 8-bit unsigned integer
195   */
196  "YUV420P",
197
198  /*
199   * Channel order: Y, U, V
200   * Channel size: full y-channel, quarter uv-channels
201   * Pixel layout: planar y-channel, interleaving uv-channels
202   * Pixel layout illustration:
203   *   [Plane1]: Y1 Y2 Y3 Y4
204   *   [Plane2]: U1 V1
205   * Data type: 8-bit unsigned integer
206   */
207  "YUV420SP_NV12",
208
209  /*
210   * Channel order: Y, V, U
211   * Channel size: full y-channel, quarter vu-channels
212   * Pixel layout: planar y-channel, interleaving vu-channels
213   * Pixel layout illustration:
214   *   [Plane1]: Y1 Y2 Y3 Y4
215   *   [Plane2]: V1 U1
216   * Data type: 8-bit unsigned integer
217   */
218  "YUV420SP_NV21",
219
220  /*
221   * Channel order: H, S, V
222   * Channel size: full hsv-channels
223   * Pixel layout: interleaving hsv-channels
224   * Pixel layout illustration:
225   *   [Plane1]: H1 S1 V1 H2 S2 V2 H3 S3 V3
226   * Data type: 32-bit floating point value
227   */
228  "HSV",
229
230  /*
231   * Channel order: l, a, b
232   * Channel size: full lab-channels
233   * Pixel layout: interleaving lab-channels
234   * Pixel layout illustration:
235   *   [Plane1]: l1 a1 b1 l2 a2 b2 l3 a3 b3
236   * Data type: 32-bit floating point value
237   */
238  "Lab",
239
240  /*
241   * Channel order: DEPTH
242   * Channel size: full depth-channel
243   * Pixel layout: planar depth-channel
244   * Pixel layout illustration:
245   *   [Plane1]: DEPTH1 DEPTH2 DEPTH3 DEPTH4
246   * Data type: 16-bit unsigned integer
247   */
248  "DEPTH",
249};
250
251enum ChannelPixelLayoutDataType {
252  "uint8",
253  "int8",
254  "uint16",
255  "int16",
256  "uint32",
257  "int32",
258  "float32",
259  "float64"
260};
261
262/*
263 * Two concepts, ImagePixelLayout and ChannelPixelLayout, together generalize
264 * the variety of pixel layouts among image formats.
265 *
266 * The ChannelPixelLayout represents the pixel layout of a single channel in a
267 * certain image format and the ImagePixelLayout is just the collection of
268 * ChannelPixelLayouts. So, the ChannelPixelLayout is defined as a dictionary
269 * type with properties to describe the layout and the ImagePixelLayout is just
270 * an alias name to a sequence of ChannelPixelLayout objects.
271 *
272 * Since an image format is composed of at least one channel, an
273 * ImagePixelLayout object contains at least one ChannelPixelLayout object.
274 *
275 * Although an image or a video frame is a two-dimensional structure, its data
276 * is usually stored in a one-dimensional array in the row-major way and the
277 * ChannelPixelLayout objects use the following properties to describe the
278 * layout of pixel values in the buffer.
279 *
280 * 1) offset:
281 *    denotes the beginning position of the channel's data relative to the
282 *    beginning position of the one-dimensional array.
283 * 2) width & height:
284 *    denote the width and height of the channel respectively. Each channel in
285 *    an image format may have different height and width.
286 * 3) data type:
287 *    denotes the format used to store one single pixel value.
288 * 4) stride:
289 *    the number of bytes between the beginning two consecutive rows in memory.
290 *    (The total bytes of each row plus the padding bytes of each raw.)
291 * 5) skip value:
292 *    the value is zero for the planar pixel layout, and a positive integer for
293 *    the interleaving pixel layout. (Describes how many bytes there are between
294 *    two adjacent pixel values in this channel.)
295 */
296
297/*
298 * Example1: RGBA image, width = 620, height = 480, stride = 2560
299 *
300 * chanel_r: offset = 0, width = 620, height = 480, data type = uint8, stride = 2560, skip = 3
301 * chanel_g: offset = 1, width = 620, height = 480, data type = uint8, stride = 2560, skip = 3
302 * chanel_b: offset = 2, width = 620, height = 480, data type = uint8, stride = 2560, skip = 3
303 * chanel_a: offset = 3, width = 620, height = 480, data type = uint8, stride = 2560, skip = 3
304 *
305 *         <---------------------------- stride ---------------------------->
306 *         <---------------------- width x 4 ---------------------->
307 * [index] 01234   8   12  16  20  24  28                           2479    2559
308 *         |||||---|---|---|---|---|---|----------------------------|-------|
309 * [data]  RGBARGBARGBARGBARGBAR___R___R...                         A%%%%%%%%
310 * [data]  RGBARGBARGBARGBARGBAR___R___R...                         A%%%%%%%%
311 * [data]  RGBARGBARGBARGBARGBAR___R___R...                         A%%%%%%%%
312 *              ^^^
313 *              r-skip
314 */
315
316/*
317 * Example2: YUV420P image, width = 620, height = 480, stride = 640
318 *
319 * chanel_y: offset = 0, width = 620, height = 480, stride = 640, skip = 0
320 * chanel_u: offset = 307200, width = 310, height = 240, data type = uint8, stride = 320, skip = 0
321 * chanel_v: offset = 384000, width = 310, height = 240, data type = uint8, stride = 320, skip = 0
322 *
323 *         <--------------------------- y-stride --------------------------->
324 *         <----------------------- y-width ----------------------->
325 * [index] 012345                                                  619      639
326 *         ||||||--------------------------------------------------|--------|
327 * [data]  YYYYYYYYYYYYYYYYYYYYYYYYYYYYY...                        Y%%%%%%%%%
328 * [data]  YYYYYYYYYYYYYYYYYYYYYYYYYYYYY...                        Y%%%%%%%%%
329 * [data]  YYYYYYYYYYYYYYYYYYYYYYYYYYYYY...                        Y%%%%%%%%%
330 * [data]  ......
331 *         <-------- u-stride ---------->
332 *         <----- u-width ----->
333 * [index] 307200              307509   307519
334 *         |-------------------|--------|
335 * [data]  UUUUUUUUUU...       U%%%%%%%%%
336 * [data]  UUUUUUUUUU...       U%%%%%%%%%
337 * [data]  UUUUUUUUUU...       U%%%%%%%%%
338 * [data]  ......
339 *         <-------- v-stride ---------->
340 *         <- --- v-width ----->
341 * [index] 384000              384309   384319
342 *         |-------------------|--------|
343 * [data]  VVVVVVVVVV...       V%%%%%%%%%
344 * [data]  VVVVVVVVVV...       V%%%%%%%%%
345 * [data]  VVVVVVVVVV...       V%%%%%%%%%
346 * [data]  ......
347 */
348
349/*
350 * Example3: YUV420SP_NV12 image, width = 620, height = 480, stride = 640
351 *
352 * chanel_y: offset = 0, width = 620, height = 480, stride = 640, skip = 0
353 * chanel_u: offset = 307200, width = 310, height = 240, data type = uint8, stride = 640, skip = 1
354 * chanel_v: offset = 307201, width = 310, height = 240, data type = uint8, stride = 640, skip = 1
355 *
356 *         <--------------------------- y-stride -------------------------->
357 *         <----------------------- y-width ---------------------->
358 * [index] 012345                                                 619      639
359 *         ||||||-------------------------------------------------|--------|
360 * [data]  YYYYYYYYYYYYYYYYYYYYYYYYYYYYY...                       Y%%%%%%%%%
361 * [data]  YYYYYYYYYYYYYYYYYYYYYYYYYYYYY...                       Y%%%%%%%%%
362 * [data]  YYYYYYYYYYYYYYYYYYYYYYYYYYYYY...                       Y%%%%%%%%%
363 * [data]  ......
364 *         <--------------------- u-stride / v-stride -------------------->
365 *         <------------------ u-width + v-width ----------------->
366 * [index] 307200(u-offset)                                       307819  307839
367 *         |------------------------------------------------------|-------|
368 * [index] |307201(v-offset)                                      |307820 |
369 *         ||-----------------------------------------------------||------|
370 * [data]  UVUVUVUVUVUVUVUVUVUVUVUVUVUVUV...                      UV%%%%%%%
371 * [data]  UVUVUVUVUVUVUVUVUVUVUVUVUVUVUV...                      UV%%%%%%%
372 * [data]  UVUVUVUVUVUVUVUVUVUVUVUVUVUVUV...                      UV%%%%%%%
373 *          ^            ^
374 *         u-skip        v-skip
375 */
376
377/*
378 * Example4: DEPTH image, width = 640, height = 480, stride = 1280
379 *
380 * chanel_d: offset = 0, width = 640, height = 480, data type = uint16, stride = 1280, skip = 0
381 *
382 * note: each DEPTH value uses two bytes
383 *
384 *         <----------------------- d-stride ---------------------->
385 *         <----------------------- d-width ----------------------->
386 * [index] 02468                                                   1278
387 *         |||||---------------------------------------------------|
388 * [data]  DDDDDDDDDDDDDDDDDDDDDDDDDDDDD...                        D
389 * [data]  DDDDDDDDDDDDDDDDDDDDDDDDDDDDD...                        D
390 * [data]  DDDDDDDDDDDDDDDDDDDDDDDDDDDDD...                        D
391 * [data]  ......
392 */
393
394dictionary ChannelPixelLayout {
395    required unsigned long              offset;
396    required unsigned long              width;
397    required unsigned long              height;
398    required ChannelPixelLayoutDataType dataType;
399    required unsigned long              stride;
400    required unsigned long              skip;
401};
402
403typedef sequence<ChannelPixelLayout> ImagePixelLayout;
404
405partial interface ImageBitmap {
406    [Throws, Func="mozilla::dom::DOMPrefs::ImageBitmapExtensionsEnabled"]
407    ImageBitmapFormat               findOptimalFormat (optional sequence<ImageBitmapFormat> aPossibleFormats);
408    [Throws, Func="mozilla::dom::DOMPrefs::ImageBitmapExtensionsEnabled"]
409    long                            mappedDataLength (ImageBitmapFormat aFormat);
410    [Throws, Func="mozilla::dom::DOMPrefs::ImageBitmapExtensionsEnabled"]
411    Promise<ImagePixelLayout> mapDataInto (ImageBitmapFormat aFormat, BufferSource aBuffer, long aOffset);
412};
413