1 ///////////////////////////////////////////////////////////////////////////
2 //
3 // Copyright (c) 2011, Industrial Light & Magic, a division of Lucas
4 // Digital Ltd. LLC
5 //
6 // All rights reserved.
7 //
8 // Redistribution and use in source and binary forms, with or without
9 // modification, are permitted provided that the following conditions are
10 // met:
11 // * Redistributions of source code must retain the above copyright
12 // notice, this list of conditions and the following disclaimer.
13 // * Redistributions in binary form must reproduce the above
14 // copyright notice, this list of conditions and the following disclaimer
15 // in the documentation and/or other materials provided with the
16 // distribution.
17 // * Neither the name of Industrial Light & Magic nor the names of
18 // its contributors may be used to endorse or promote products derived
19 // from this software without specific prior written permission.
20 //
21 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 //
33 ///////////////////////////////////////////////////////////////////////////
34
35
36 //-----------------------------------------------------------------------------
37 //
38 // class DeepScanLineInputFile
39 //
40 //-----------------------------------------------------------------------------
41
42 #include <ImfDeepScanLineInputFile.h>
43 #include <ImfChannelList.h>
44 #include <ImfMisc.h>
45 #include <ImfStdIO.h>
46 #include <ImfCompressor.h>
47 #include <ImfXdr.h>
48 #include <ImfConvert.h>
49 #include <ImfThreading.h>
50 #include <ImfPartType.h>
51 #include <ImfVersion.h>
52 #include "ImfMultiPartInputFile.h"
53 #include "ImfDeepFrameBuffer.h"
54 #include "ImfInputStreamMutex.h"
55 #include "ImfInputPartData.h"
56
57
58 #include "ImathBox.h"
59 #include "ImathFun.h"
60
61
62 #include "IlmThreadPool.h"
63 #include "IlmThreadSemaphore.h"
64 #include "IlmThreadMutex.h"
65
66 #include "Iex.h"
67
68 #include <string>
69 #include <vector>
70 #include <assert.h>
71 #include <limits>
72 #include <algorithm>
73
74
75 #include "ImfNamespace.h"
76 OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER
77
78 using IMATH_NAMESPACE::Box2i;
79 using IMATH_NAMESPACE::divp;
80 using IMATH_NAMESPACE::modp;
81 using std::string;
82 using std::vector;
83 using std::ifstream;
84 using std::min;
85 using std::max;
86 using ILMTHREAD_NAMESPACE::Mutex;
87 using ILMTHREAD_NAMESPACE::Lock;
88 using ILMTHREAD_NAMESPACE::Semaphore;
89 using ILMTHREAD_NAMESPACE::Task;
90 using ILMTHREAD_NAMESPACE::TaskGroup;
91 using ILMTHREAD_NAMESPACE::ThreadPool;
92
93 namespace {
94
95 struct InSliceInfo
96 {
97 PixelType typeInFrameBuffer;
98 PixelType typeInFile;
99 char * base;
100 char* pointerArrayBase;
101 size_t xPointerStride;
102 size_t yPointerStride;
103 size_t sampleStride;
104 int xSampling;
105 int ySampling;
106 bool fill;
107 bool skip;
108 double fillValue;
109
110 InSliceInfo (PixelType typeInFrameBuffer = HALF,
111 char * base = NULL,
112 PixelType typeInFile = HALF,
113 size_t xPointerStride = 0,
114 size_t yPointerStride = 0,
115 size_t sampleStride = 0,
116 int xSampling = 1,
117 int ySampling = 1,
118 bool fill = false,
119 bool skip = false,
120 double fillValue = 0.0);
121 };
122
123
InSliceInfo(PixelType tifb,char * b,PixelType tifl,size_t xpst,size_t ypst,size_t spst,int xsm,int ysm,bool f,bool s,double fv)124 InSliceInfo::InSliceInfo (PixelType tifb,
125 char * b,
126 PixelType tifl,
127 size_t xpst,
128 size_t ypst,
129 size_t spst,
130 int xsm, int ysm,
131 bool f, bool s,
132 double fv)
133 :
134 typeInFrameBuffer (tifb),
135 typeInFile (tifl),
136 base(b),
137 xPointerStride (xpst),
138 yPointerStride (ypst),
139 sampleStride (spst),
140 xSampling (xsm),
141 ySampling (ysm),
142 fill (f),
143 skip (s),
144 fillValue (fv)
145 {
146 // empty
147 }
148
149
150 struct LineBuffer
151 {
152 const char * uncompressedData;
153 char * buffer;
154 Int64 packedDataSize;
155 Int64 unpackedDataSize;
156
157 int minY;
158 int maxY;
159 Compressor * compressor;
160 Compressor::Format format;
161 int number;
162 bool hasException;
163 string exception;
164
165 LineBuffer ();
166 ~LineBuffer ();
167
wait__anonf40d4db70111::LineBuffer168 inline void wait () {_sem.wait();}
post__anonf40d4db70111::LineBuffer169 inline void post () {_sem.post();}
170
171 private:
172
173 Semaphore _sem;
174 };
175
176
LineBuffer()177 LineBuffer::LineBuffer ():
178 uncompressedData (0),
179 buffer (0),
180 packedDataSize (0),
181 compressor (0),
182 format (defaultFormat(compressor)),
183 number (-1),
184 hasException (false),
185 exception (),
186 _sem (1)
187 {
188 // empty
189 }
190
191
~LineBuffer()192 LineBuffer::~LineBuffer ()
193 {
194 if (compressor != 0)
195 delete compressor;
196 }
197
198 } // namespace
199
200
201 struct DeepScanLineInputFile::Data: public Mutex
202 {
203 Header header; // the image header
204 int version; // file's version
205 DeepFrameBuffer frameBuffer; // framebuffer to write into
206 LineOrder lineOrder; // order of the scanlines in file
207 int minX; // data window's min x coord
208 int maxX; // data window's max x coord
209 int minY; // data window's min y coord
210 int maxY; // data window's max x coord
211 vector<Int64> lineOffsets; // stores offsets in file for
212 // each line
213 bool fileIsComplete; // True if no scanlines are missing
214 // in the file
215 int nextLineBufferMinY; // minimum y of the next linebuffer
216 vector<size_t> bytesPerLine; // combined size of a line over all
217 // channels
218 vector<size_t> offsetInLineBuffer; // offset for each scanline in its
219 // linebuffer
220 vector<InSliceInfo*> slices; // info about channels in file
221
222 vector<LineBuffer*> lineBuffers; // each holds one line buffer
223 int linesInBuffer; // number of scanlines each buffer
224 // holds
225 int partNumber; // part number
226 int numThreads; // number of threads
227
228 bool multiPartBackwardSupport; // if we are reading a multipart file using single file API
229 MultiPartInputFile* multiPartFile; // for multipart files opened as single part
230 bool memoryMapped; // if the stream is memory mapped
231
232 Array2D<unsigned int> sampleCount; // the number of samples
233 // in each pixel
234
235 Array<unsigned int> lineSampleCount; // the number of samples
236 // in each line
237
238 Array<bool> gotSampleCount; // for each scanline, indicating if
239 // we have got its sample count table
240
241 char* sampleCountSliceBase; // pointer to the start of
242 // the sample count array
243 int sampleCountXStride; // x stride of the sample count array
244 int sampleCountYStride; // y stride of the sample count array
245 bool frameBufferValid; // set by setFrameBuffer: excepts if readPixelSampleCounts if false
246
247 Array<char> sampleCountTableBuffer;
248 // the buffer for sample count table
249
250 Compressor* sampleCountTableComp;
251 // the decompressor for sample count table
252
253 int combinedSampleSize; // total size of all channels combined: used to sanity check sample table size
254
255 int maxSampleCountTableSize;
256 // the max size in bytes for a pixel
257 // sample count table
258 InputStreamMutex* _streamData;
259 bool _deleteStream;
260
261
262 Data (int numThreads);
263 ~Data ();
264
265 inline LineBuffer * getLineBuffer (int number); // hash function from line
266 // buffer indices into our
267 // vector of line buffers
268 };
269
270
Data(int numThreads)271 DeepScanLineInputFile::Data::Data (int numThreads):
272 partNumber(-1),
273 numThreads(numThreads),
274 multiPartBackwardSupport(false),
275 multiPartFile(NULL),
276 memoryMapped(false),
277 frameBufferValid(false),
278 _streamData(NULL),
279 _deleteStream(false)
280 {
281 //
282 // We need at least one lineBuffer, but if threading is used,
283 // to keep n threads busy we need 2*n lineBuffers
284 //
285
286 lineBuffers.resize (max (1, 2 * numThreads));
287
288 for (size_t i = 0; i < lineBuffers.size(); i++)
289 lineBuffers[i] = 0;
290
291 sampleCountTableComp = 0;
292 }
293
294
~Data()295 DeepScanLineInputFile::Data::~Data ()
296 {
297 for (size_t i = 0; i < lineBuffers.size(); i++)
298 if (lineBuffers[i] != 0)
299 delete lineBuffers[i];
300
301 for (size_t i = 0; i < slices.size(); i++)
302 delete slices[i];
303
304 if (sampleCountTableComp != 0)
305 delete sampleCountTableComp;
306
307 if (multiPartBackwardSupport)
308 delete multiPartFile;
309 }
310
311
312 inline LineBuffer *
getLineBuffer(int lineBufferNumber)313 DeepScanLineInputFile::Data::getLineBuffer (int lineBufferNumber)
314 {
315 return lineBuffers[lineBufferNumber % lineBuffers.size()];
316 }
317
318
319 namespace {
320
321
322 void
reconstructLineOffsets(OPENEXR_IMF_INTERNAL_NAMESPACE::IStream & is,LineOrder lineOrder,vector<Int64> & lineOffsets)323 reconstructLineOffsets (OPENEXR_IMF_INTERNAL_NAMESPACE::IStream &is,
324 LineOrder lineOrder,
325 vector<Int64> &lineOffsets)
326 {
327 Int64 position = is.tellg();
328
329 try
330 {
331 for (unsigned int i = 0; i < lineOffsets.size(); i++)
332 {
333 Int64 lineOffset = is.tellg();
334
335 int y;
336 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (is, y);
337
338 Int64 packed_offset;
339 Int64 packed_sample;
340 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (is, packed_offset);
341 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (is, packed_sample);
342 //next is unpacked sample table size - skip this too
343 Xdr::skip <StreamIO> (is, packed_offset+packed_sample+8);
344
345 if (lineOrder == INCREASING_Y)
346 lineOffsets[i] = lineOffset;
347 else
348 lineOffsets[lineOffsets.size() - i - 1] = lineOffset;
349 }
350 }
351 catch (...)
352 {
353 //
354 // Suppress all exceptions. This functions is
355 // called only to reconstruct the line offset
356 // table for incomplete files, and exceptions
357 // are likely.
358 //
359 }
360
361 is.clear();
362 is.seekg (position);
363 }
364
365
366 void
readLineOffsets(OPENEXR_IMF_INTERNAL_NAMESPACE::IStream & is,LineOrder lineOrder,vector<Int64> & lineOffsets,bool & complete)367 readLineOffsets (OPENEXR_IMF_INTERNAL_NAMESPACE::IStream &is,
368 LineOrder lineOrder,
369 vector<Int64> &lineOffsets,
370 bool &complete)
371 {
372 for (unsigned int i = 0; i < lineOffsets.size(); i++)
373 {
374 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (is, lineOffsets[i]);
375 }
376
377 complete = true;
378
379 for (unsigned int i = 0; i < lineOffsets.size(); i++)
380 {
381 if (lineOffsets[i] <= 0)
382 {
383 //
384 // Invalid data in the line offset table mean that
385 // the file is probably incomplete (the table is
386 // the last thing written to the file). Either
387 // some process is still busy writing the file,
388 // or writing the file was aborted.
389 //
390 // We should still be able to read the existing
391 // parts of the file. In order to do this, we
392 // have to make a sequential scan over the scan
393 // line data to reconstruct the line offset table.
394 //
395
396 complete = false;
397 reconstructLineOffsets (is, lineOrder, lineOffsets);
398 break;
399 }
400 }
401 }
402
403
404 void
readPixelData(InputStreamMutex * streamData,DeepScanLineInputFile::Data * ifd,int minY,char * & buffer,Int64 & packedDataSize,Int64 & unpackedDataSize)405 readPixelData (InputStreamMutex *streamData,
406 DeepScanLineInputFile::Data *ifd,
407 int minY,
408 char *&buffer,
409 Int64 &packedDataSize,
410 Int64 &unpackedDataSize)
411 {
412 //
413 // Read a single line buffer from the input file.
414 //
415 // If the input file is not memory-mapped, we copy the pixel data into
416 // into the array pointed to by buffer. If the file is memory-mapped,
417 // then we change where buffer points to instead of writing into the
418 // array (hence buffer needs to be a reference to a char *).
419 //
420
421 int lineBufferNumber = (minY - ifd->minY) / ifd->linesInBuffer;
422
423 Int64 lineOffset = ifd->lineOffsets[lineBufferNumber];
424
425 if (lineOffset == 0)
426 THROW (IEX_NAMESPACE::InputExc, "Scan line " << minY << " is missing.");
427
428 //
429 // Seek to the start of the scan line in the file,
430 // if necessary.
431 //
432
433 if (!isMultiPart(ifd->version))
434 {
435 if (ifd->nextLineBufferMinY != minY)
436 streamData->is->seekg (lineOffset);
437 }
438 else
439 {
440 //
441 // In a multi-part file, the file pointer may have been moved by
442 // other parts, so we have to ask tellg() where we are.
443 //
444 if (streamData->is->tellg() != ifd->lineOffsets[lineBufferNumber])
445 streamData->is->seekg (lineOffset);
446 }
447
448 //
449 // Read the data block's header.
450 //
451
452 int yInFile;
453
454 //
455 // Read the part number when we are dealing with a multi-part file.
456 //
457
458 if (isMultiPart(ifd->version))
459 {
460 int partNumber;
461 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*streamData->is, partNumber);
462 if (partNumber != ifd->partNumber)
463 {
464 THROW (IEX_NAMESPACE::ArgExc, "Unexpected part number " << partNumber
465 << ", should be " << ifd->partNumber << ".");
466 }
467 }
468
469 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*streamData->is, yInFile);
470
471 if (yInFile != minY)
472 throw IEX_NAMESPACE::InputExc ("Unexpected data block y coordinate.");
473
474 Int64 sampleCountTableSize;
475 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*streamData->is, sampleCountTableSize);
476 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*streamData->is, packedDataSize);
477 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*streamData->is, unpackedDataSize);
478
479
480 //
481 // We make a check on the data size requirements here.
482 // Whilst we wish to store 64bit sizes on disk, not all the compressors
483 // have been made to work with such data sizes and are still limited to
484 // using signed 32 bit (int) for the data size. As such, this version
485 // insists that we validate that the data size does not exceed the data
486 // type max limit.
487 // @TODO refactor the compressor code to ensure full 64-bit support.
488 //
489
490 int compressorMaxDataSize = std::numeric_limits<int>::max();
491 if (packedDataSize > Int64(compressorMaxDataSize) ||
492 unpackedDataSize > Int64(compressorMaxDataSize))
493 {
494 THROW (IEX_NAMESPACE::ArgExc, "This version of the library does not support "
495 << "the allocation of data with size > " << compressorMaxDataSize
496 << " file unpacked size :" << unpackedDataSize
497 << " file packed size :" << packedDataSize << ".\n");
498 }
499
500 //
501 // Skip the pixel sample count table because we have read this data.
502 //
503
504 Xdr::skip <StreamIO> (*streamData->is, sampleCountTableSize);
505
506 //
507 // Read the pixel data.
508 //
509
510 if (streamData->is->isMemoryMapped ())
511 buffer = streamData->is->readMemoryMapped (packedDataSize);
512 else
513 {
514 // (TODO) check if the packed data size is too big?
515 // (TODO) better memory management. Don't delete buffer all the time.
516 if (buffer != 0) delete[] buffer;
517 buffer = new char[packedDataSize];
518 streamData->is->read (buffer, packedDataSize);
519 }
520
521 //
522 // Keep track of which scan line is the next one in
523 // the file, so that we can avoid redundant seekg()
524 // operations (seekg() can be fairly expensive).
525 //
526
527 if (ifd->lineOrder == INCREASING_Y)
528 ifd->nextLineBufferMinY = minY + ifd->linesInBuffer;
529 else
530 ifd->nextLineBufferMinY = minY - ifd->linesInBuffer;
531 }
532
533 //
534 // A LineBufferTask encapsulates the task uncompressing a set of
535 // scanlines (line buffer) and copying them into the frame buffer.
536 //
537
538 class LineBufferTask : public Task
539 {
540 public:
541
542 LineBufferTask (TaskGroup *group,
543 DeepScanLineInputFile::Data *ifd,
544 LineBuffer *lineBuffer,
545 int scanLineMin,
546 int scanLineMax);
547
548 virtual ~LineBufferTask ();
549
550 virtual void execute ();
551
552 private:
553
554 DeepScanLineInputFile::Data * _ifd;
555 LineBuffer * _lineBuffer;
556 int _scanLineMin;
557 int _scanLineMax;
558 };
559
560
LineBufferTask(TaskGroup * group,DeepScanLineInputFile::Data * ifd,LineBuffer * lineBuffer,int scanLineMin,int scanLineMax)561 LineBufferTask::LineBufferTask
562 (TaskGroup *group,
563 DeepScanLineInputFile::Data *ifd,
564 LineBuffer *lineBuffer,
565 int scanLineMin,
566 int scanLineMax)
567 :
568 Task (group),
569 _ifd (ifd),
570 _lineBuffer (lineBuffer),
571 _scanLineMin (scanLineMin),
572 _scanLineMax (scanLineMax)
573 {
574 // empty
575 }
576
577
~LineBufferTask()578 LineBufferTask::~LineBufferTask ()
579 {
580 //
581 // Signal that the line buffer is now free
582 //
583
584 _lineBuffer->post ();
585 }
586
587
588 void
execute()589 LineBufferTask::execute ()
590 {
591 try
592 {
593 //
594 // Uncompress the data, if necessary
595 //
596
597 if (_lineBuffer->uncompressedData == 0)
598 {
599 Int64 uncompressedSize = 0;
600 int maxY = min (_lineBuffer->maxY, _ifd->maxY);
601
602 for (int i = _lineBuffer->minY - _ifd->minY;
603 i <= maxY - _ifd->minY;
604 ++i)
605 {
606 uncompressedSize += (int) _ifd->bytesPerLine[i];
607 }
608
609 //
610 // Create the compressor everytime when we want to use it,
611 // because we don't know maxBytesPerLine beforehand.
612 // (TODO) optimize this. don't do this every time.
613 //
614
615 if (_lineBuffer->compressor != 0)
616 delete _lineBuffer->compressor;
617 Int64 maxBytesPerLine = 0;
618 for (int i = _lineBuffer->minY - _ifd->minY;
619 i <= maxY - _ifd->minY;
620 ++i)
621 {
622 if (_ifd->bytesPerLine[i] > maxBytesPerLine)
623 maxBytesPerLine = _ifd->bytesPerLine[i];
624 }
625 _lineBuffer->compressor = newCompressor(_ifd->header.compression(),
626 maxBytesPerLine,
627 _ifd->header);
628
629 if (_lineBuffer->compressor &&
630 _lineBuffer->packedDataSize < uncompressedSize)
631 {
632 _lineBuffer->format = _lineBuffer->compressor->format();
633
634 _lineBuffer->packedDataSize = _lineBuffer->compressor->uncompress
635 (_lineBuffer->buffer, _lineBuffer->packedDataSize,
636 _lineBuffer->minY, _lineBuffer->uncompressedData);
637 }
638 else
639 {
640 //
641 // If the line is uncompressed, it's in XDR format,
642 // regardless of the compressor's output format.
643 //
644
645 _lineBuffer->format = Compressor::XDR;
646 _lineBuffer->uncompressedData = _lineBuffer->buffer;
647 }
648 }
649
650 int yStart, yStop, dy;
651
652 if (_ifd->lineOrder == INCREASING_Y)
653 {
654 yStart = _scanLineMin;
655 yStop = _scanLineMax + 1;
656 dy = 1;
657 }
658 else
659 {
660 yStart = _scanLineMax;
661 yStop = _scanLineMin - 1;
662 dy = -1;
663 }
664
665 for (int y = yStart; y != yStop; y += dy)
666 {
667 //
668 // Convert one scan line's worth of pixel data back
669 // from the machine-independent representation, and
670 // store the result in the frame buffer.
671 //
672
673 const char *readPtr = _lineBuffer->uncompressedData +
674 _ifd->offsetInLineBuffer[y - _ifd->minY];
675
676 //
677 // Iterate over all image channels.
678 //
679
680 for (unsigned int i = 0; i < _ifd->slices.size(); ++i)
681 {
682 //
683 // Test if scan line y of this channel contains any data
684 // (the scan line contains data only if y % ySampling == 0).
685 //
686
687 InSliceInfo &slice = *_ifd->slices[i];
688
689 if (modp (y, slice.ySampling) != 0)
690 continue;
691
692 //
693 // Find the x coordinates of the leftmost and rightmost
694 // sampled pixels (i.e. pixels within the data window
695 // for which x % xSampling == 0).
696 //
697
698 //
699 // Fill the frame buffer with pixel data.
700 //
701
702 if (slice.skip)
703 {
704 //
705 // The file contains data for this channel, but
706 // the frame buffer contains no slice for this channel.
707 //
708
709 skipChannel (readPtr, slice.typeInFile,
710 _ifd->lineSampleCount[y - _ifd->minY]);
711 }
712 else
713 {
714 //
715 // The frame buffer contains a slice for this channel.
716 //
717
718 int width = (_ifd->maxX - _ifd->minX + 1);
719
720 copyIntoDeepFrameBuffer (readPtr, slice.base,
721 (char*) (&_ifd->sampleCount[0][0]
722 - _ifd->minX
723 - _ifd->minY * width),
724 sizeof(unsigned int) * 1,
725 sizeof(unsigned int) * width,
726 y, _ifd->minX, _ifd->maxX,
727 0, 0,
728 0, 0,
729 slice.sampleStride,
730 slice.xPointerStride,
731 slice.yPointerStride,
732 slice.fill,
733 slice.fillValue, _lineBuffer->format,
734 slice.typeInFrameBuffer,
735 slice.typeInFile);
736 }
737 }
738 }
739 }
740 catch (std::exception &e)
741 {
742 if (!_lineBuffer->hasException)
743 {
744 _lineBuffer->exception = e.what();
745 _lineBuffer->hasException = true;
746 }
747 }
748 catch (...)
749 {
750 if (!_lineBuffer->hasException)
751 {
752 _lineBuffer->exception = "unrecognized exception";
753 _lineBuffer->hasException = true;
754 }
755 }
756 }
757
758
759 LineBufferTask *
newLineBufferTask(TaskGroup * group,DeepScanLineInputFile::Data * ifd,int number,int scanLineMin,int scanLineMax)760 newLineBufferTask
761 (TaskGroup *group,
762 DeepScanLineInputFile::Data *ifd,
763 int number,
764 int scanLineMin,
765 int scanLineMax)
766 {
767 //
768 // Wait for a line buffer to become available, fill the line
769 // buffer with raw data from the file if necessary, and create
770 // a new LineBufferTask whose execute() method will uncompress
771 // the contents of the buffer and copy the pixels into the
772 // frame buffer.
773 //
774
775 LineBuffer *lineBuffer = ifd->getLineBuffer (number);
776
777 try
778 {
779 lineBuffer->wait ();
780
781 if (lineBuffer->number != number)
782 {
783 lineBuffer->minY = ifd->minY + number * ifd->linesInBuffer;
784 lineBuffer->maxY = lineBuffer->minY + ifd->linesInBuffer - 1;
785
786 lineBuffer->number = number;
787 lineBuffer->uncompressedData = 0;
788
789 readPixelData (ifd->_streamData, ifd, lineBuffer->minY,
790 lineBuffer->buffer,
791 lineBuffer->packedDataSize,
792 lineBuffer->unpackedDataSize);
793 }
794 }
795 catch (std::exception &e)
796 {
797 if (!lineBuffer->hasException)
798 {
799 lineBuffer->exception = e.what();
800 lineBuffer->hasException = true;
801 }
802 lineBuffer->number = -1;
803 lineBuffer->post();
804 throw;
805 }
806 catch (...)
807 {
808 //
809 // Reading from the file caused an exception.
810 // Signal that the line buffer is free, and
811 // re-throw the exception.
812 //
813
814 lineBuffer->exception = "unrecognized exception";
815 lineBuffer->hasException = true;
816 lineBuffer->number = -1;
817 lineBuffer->post();
818 throw;
819 }
820
821 scanLineMin = max (lineBuffer->minY, scanLineMin);
822 scanLineMax = min (lineBuffer->maxY, scanLineMax);
823
824 return new LineBufferTask (group, ifd, lineBuffer,
825 scanLineMin, scanLineMax);
826 }
827
828 } // namespace
829
830
initialize(const Header & header)831 void DeepScanLineInputFile::initialize(const Header& header)
832 {
833 try
834 {
835 if (header.type() != DEEPSCANLINE)
836 throw IEX_NAMESPACE::ArgExc("Can't build a DeepScanLineInputFile from "
837 "a type-mismatched part.");
838
839 if(header.version()!=1)
840 {
841 THROW(IEX_NAMESPACE::ArgExc, "Version " << header.version() << " not supported for deepscanline images in this version of the library");
842 }
843
844 _data->header = header;
845
846 _data->lineOrder = _data->header.lineOrder();
847
848 const Box2i &dataWindow = _data->header.dataWindow();
849
850 _data->minX = dataWindow.min.x;
851 _data->maxX = dataWindow.max.x;
852 _data->minY = dataWindow.min.y;
853 _data->maxY = dataWindow.max.y;
854
855 _data->sampleCount.resizeErase(_data->maxY - _data->minY + 1,
856 _data->maxX - _data->minX + 1);
857 _data->lineSampleCount.resizeErase(_data->maxY - _data->minY + 1);
858
859 Compressor* compressor = newCompressor(_data->header.compression(),
860 0,
861 _data->header);
862
863 _data->linesInBuffer = numLinesInBuffer (compressor);
864
865 delete compressor;
866
867 _data->nextLineBufferMinY = _data->minY - 1;
868
869 int lineOffsetSize = (dataWindow.max.y - dataWindow.min.y +
870 _data->linesInBuffer) / _data->linesInBuffer;
871
872 _data->lineOffsets.resize (lineOffsetSize);
873
874 for (size_t i = 0; i < _data->lineBuffers.size(); i++)
875 _data->lineBuffers[i] = new LineBuffer ();
876
877 _data->gotSampleCount.resizeErase(_data->maxY - _data->minY + 1);
878 for (int i = 0; i < _data->maxY - _data->minY + 1; i++)
879 _data->gotSampleCount[i] = false;
880
881 _data->maxSampleCountTableSize = min(_data->linesInBuffer, _data->maxY - _data->minY + 1) *
882 (_data->maxX - _data->minX + 1) *
883 sizeof(unsigned int);
884
885 _data->sampleCountTableBuffer.resizeErase(_data->maxSampleCountTableSize);
886
887 _data->sampleCountTableComp = newCompressor(_data->header.compression(),
888 _data->maxSampleCountTableSize,
889 _data->header);
890
891 _data->bytesPerLine.resize (_data->maxY - _data->minY + 1);
892
893 const ChannelList & c=header.channels();
894
895 _data->combinedSampleSize=0;
896 for(ChannelList::ConstIterator i=c.begin();i!=c.end();i++)
897 {
898 switch(i.channel().type)
899 {
900 case OPENEXR_IMF_INTERNAL_NAMESPACE::HALF :
901 _data->combinedSampleSize+=Xdr::size<half>();
902 break;
903 case OPENEXR_IMF_INTERNAL_NAMESPACE::FLOAT :
904 _data->combinedSampleSize+=Xdr::size<float>();
905 break;
906 case OPENEXR_IMF_INTERNAL_NAMESPACE::UINT :
907 _data->combinedSampleSize+=Xdr::size<unsigned int>();
908 break;
909 default :
910 THROW(IEX_NAMESPACE::ArgExc, "Bad type for channel " << i.name() << " initializing deepscanline reader");
911
912 }
913 }
914
915 }
916 catch (...)
917 {
918 delete _data;
919 _data=NULL;
920 throw;
921 }
922 }
923
924
DeepScanLineInputFile(InputPartData * part)925 DeepScanLineInputFile::DeepScanLineInputFile(InputPartData* part)
926
927 {
928
929 _data = new Data(part->numThreads);
930 _data->_deleteStream=false;
931 _data->_streamData = part->mutex;
932 _data->memoryMapped = _data->_streamData->is->isMemoryMapped();
933 _data->version = part->version;
934
935 initialize(part->header);
936
937 _data->lineOffsets = part->chunkOffsets;
938
939 _data->partNumber = part->partNumber;
940 }
941
942
DeepScanLineInputFile(const char fileName[],int numThreads)943 DeepScanLineInputFile::DeepScanLineInputFile
944 (const char fileName[], int numThreads)
945 :
946 _data (new Data (numThreads))
947 {
948 _data->_streamData = new InputStreamMutex();
949 _data->_deleteStream = true;
950 OPENEXR_IMF_INTERNAL_NAMESPACE::IStream* is = 0;
951
952 try
953 {
954 is = new StdIFStream (fileName);
955 readMagicNumberAndVersionField(*is, _data->version);
956 //
957 // Backward compatibility to read multpart file.
958 //
959 if (isMultiPart(_data->version))
960 {
961 compatibilityInitialize(*is);
962 return;
963 }
964 _data->_streamData->is = is;
965 _data->memoryMapped = is->isMemoryMapped();
966 _data->header.readFrom (*_data->_streamData->is, _data->version);
967 _data->header.sanityCheck (isTiled (_data->version));
968
969 initialize(_data->header);
970
971 readLineOffsets (*_data->_streamData->is,
972 _data->lineOrder,
973 _data->lineOffsets,
974 _data->fileIsComplete);
975 }
976 catch (IEX_NAMESPACE::BaseExc &e)
977 {
978 if (is) delete is;
979 if (_data && _data->_streamData) delete _data->_streamData;
980 if (_data) delete _data;
981
982 REPLACE_EXC (e, "Cannot read image file "
983 "\"" << fileName << "\". " << e);
984 throw;
985 }
986 catch (...)
987 {
988 if (is) delete is;
989 if (_data && _data->_streamData) delete _data->_streamData;
990 if (_data) delete _data;
991
992 throw;
993 }
994 }
995
996
DeepScanLineInputFile(const Header & header,OPENEXR_IMF_INTERNAL_NAMESPACE::IStream * is,int version,int numThreads)997 DeepScanLineInputFile::DeepScanLineInputFile
998 (const Header &header,
999 OPENEXR_IMF_INTERNAL_NAMESPACE::IStream *is,
1000 int version,
1001 int numThreads)
1002 :
1003 _data (new Data (numThreads))
1004 {
1005 _data->_streamData=new InputStreamMutex();
1006 _data->_deleteStream=false;
1007 _data->_streamData->is = is;
1008
1009 _data->memoryMapped = is->isMemoryMapped();
1010
1011 _data->version =version;
1012
1013 initialize (header);
1014
1015 readLineOffsets (*_data->_streamData->is,
1016 _data->lineOrder,
1017 _data->lineOffsets,
1018 _data->fileIsComplete);
1019 }
1020
1021
~DeepScanLineInputFile()1022 DeepScanLineInputFile::~DeepScanLineInputFile ()
1023 {
1024 if (_data->_deleteStream)
1025 delete _data->_streamData->is;
1026
1027 if (_data)
1028 {
1029 if (!_data->memoryMapped)
1030 for (size_t i = 0; i < _data->lineBuffers.size(); i++)
1031 delete [] _data->lineBuffers[i]->buffer;
1032
1033 //
1034 // Unless this file was opened via the multipart API, delete the streamdata
1035 // object too.
1036 // (TODO) it should be "isMultiPart(data->version)", but when there is only
1037 // single part,
1038 // (see the above constructor) the version field is not set.
1039 //
1040 // (TODO) we should have a way to tell if the stream data is owned by this
1041 // file or by a parent multipart file.
1042 //
1043
1044 if (_data->partNumber == -1 && _data->_streamData)
1045 delete _data->_streamData;
1046
1047 delete _data;
1048 }
1049 }
1050
1051 void
compatibilityInitialize(OPENEXR_IMF_INTERNAL_NAMESPACE::IStream & is)1052 DeepScanLineInputFile::compatibilityInitialize(OPENEXR_IMF_INTERNAL_NAMESPACE::IStream& is)
1053 {
1054 is.seekg(0);
1055 //
1056 // Construct a MultiPartInputFile, initialize TiledInputFile
1057 // with the part 0 data.
1058 // (TODO) maybe change the third parameter of the constructor of MultiPartInputFile later.
1059 //
1060 _data->multiPartBackwardSupport = true;
1061 _data->multiPartFile = new MultiPartInputFile(is, _data->numThreads);
1062 InputPartData* part = _data->multiPartFile->getPart(0);
1063
1064 multiPartInitialize(part);
1065 }
1066
multiPartInitialize(InputPartData * part)1067 void DeepScanLineInputFile::multiPartInitialize(InputPartData* part)
1068 {
1069
1070 _data->_streamData = part->mutex;
1071 _data->memoryMapped = _data->_streamData->is->isMemoryMapped();
1072 _data->version = part->version;
1073
1074 initialize(part->header);
1075
1076 _data->lineOffsets = part->chunkOffsets;
1077
1078 _data->partNumber = part->partNumber;
1079
1080 }
1081
1082
1083 const char *
fileName() const1084 DeepScanLineInputFile::fileName () const
1085 {
1086 return _data->_streamData->is->fileName();
1087 }
1088
1089
1090 const Header &
header() const1091 DeepScanLineInputFile::header () const
1092 {
1093 return _data->header;
1094 }
1095
1096
1097 int
version() const1098 DeepScanLineInputFile::version () const
1099 {
1100 return _data->version;
1101 }
1102
1103
1104 void
setFrameBuffer(const DeepFrameBuffer & frameBuffer)1105 DeepScanLineInputFile::setFrameBuffer (const DeepFrameBuffer &frameBuffer)
1106 {
1107 Lock lock (*_data->_streamData);
1108
1109
1110 //
1111 // Check if the new frame buffer descriptor is
1112 // compatible with the image file header.
1113 //
1114
1115 const ChannelList &channels = _data->header.channels();
1116
1117 for (DeepFrameBuffer::ConstIterator j = frameBuffer.begin();
1118 j != frameBuffer.end();
1119 ++j)
1120 {
1121 ChannelList::ConstIterator i = channels.find (j.name());
1122
1123 if (i == channels.end())
1124 continue;
1125
1126 if (i.channel().xSampling != j.slice().xSampling ||
1127 i.channel().ySampling != j.slice().ySampling)
1128 THROW (IEX_NAMESPACE::ArgExc, "X and/or y subsampling factors "
1129 "of \"" << i.name() << "\" channel "
1130 "of input file \"" << fileName() << "\" are "
1131 "not compatible with the frame buffer's "
1132 "subsampling factors.");
1133 }
1134
1135 //
1136 // Store the pixel sample count table.
1137 // (TODO) Support for different sampling rates?
1138 //
1139
1140 const Slice& sampleCountSlice = frameBuffer.getSampleCountSlice();
1141 if (sampleCountSlice.base == 0)
1142 {
1143 throw IEX_NAMESPACE::ArgExc ("Invalid base pointer, please set a proper sample count slice.");
1144 }
1145 else
1146 {
1147 _data->sampleCountSliceBase = sampleCountSlice.base;
1148 _data->sampleCountXStride = sampleCountSlice.xStride;
1149 _data->sampleCountYStride = sampleCountSlice.yStride;
1150 }
1151
1152 //
1153 // Initialize the slice table for readPixels().
1154 //
1155
1156 vector<InSliceInfo*> slices;
1157 ChannelList::ConstIterator i = channels.begin();
1158
1159 for (DeepFrameBuffer::ConstIterator j = frameBuffer.begin();
1160 j != frameBuffer.end();
1161 ++j)
1162 {
1163 while (i != channels.end() && strcmp (i.name(), j.name()) < 0)
1164 {
1165 //
1166 // Channel i is present in the file but not
1167 // in the frame buffer; data for channel i
1168 // will be skipped during readPixels().
1169 //
1170
1171 slices.push_back (new InSliceInfo (i.channel().type,
1172 NULL,
1173 i.channel().type,
1174 0,
1175 0,
1176 0, // sampleStride
1177 i.channel().xSampling,
1178 i.channel().ySampling,
1179 false, // fill
1180 true, // skip
1181 0.0)); // fillValue
1182 ++i;
1183 }
1184
1185 bool fill = false;
1186
1187 if (i == channels.end() || strcmp (i.name(), j.name()) > 0)
1188 {
1189 //
1190 // Channel i is present in the frame buffer, but not in the file.
1191 // In the frame buffer, slice j will be filled with a default value.
1192 //
1193
1194 fill = true;
1195 }
1196
1197 slices.push_back (new InSliceInfo (j.slice().type,
1198 j.slice().base,
1199 fill? j.slice().type:
1200 i.channel().type,
1201 j.slice().xStride,
1202 j.slice().yStride,
1203 j.slice().sampleStride,
1204 j.slice().xSampling,
1205 j.slice().ySampling,
1206 fill,
1207 false, // skip
1208 j.slice().fillValue));
1209
1210
1211 if (i != channels.end() && !fill)
1212 ++i;
1213 }
1214
1215 //
1216 // Client may want data to be filled in multiple arrays,
1217 // so we reset gotSampleCount and bytesPerLine.
1218 //
1219
1220 for (long i = 0; i < _data->gotSampleCount.size(); i++)
1221 _data->gotSampleCount[i] = false;
1222 for (size_t i = 0; i < _data->bytesPerLine.size(); i++)
1223 _data->bytesPerLine[i] = 0;
1224
1225 //
1226 // Store the new frame buffer.
1227 //
1228
1229 _data->frameBuffer = frameBuffer;
1230
1231 for (size_t i = 0; i < _data->slices.size(); i++)
1232 delete _data->slices[i];
1233 _data->slices = slices;
1234 _data->frameBufferValid = true;
1235 }
1236
1237
1238 const DeepFrameBuffer &
frameBuffer() const1239 DeepScanLineInputFile::frameBuffer () const
1240 {
1241 Lock lock (*_data->_streamData);
1242 return _data->frameBuffer;
1243 }
1244
1245
1246 bool
isComplete() const1247 DeepScanLineInputFile::isComplete () const
1248 {
1249 return _data->fileIsComplete;
1250 }
1251
1252
1253 void
readPixels(int scanLine1,int scanLine2)1254 DeepScanLineInputFile::readPixels (int scanLine1, int scanLine2)
1255 {
1256 try
1257 {
1258 Lock lock (*_data->_streamData);
1259
1260 if (_data->slices.size() == 0)
1261 throw IEX_NAMESPACE::ArgExc ("No frame buffer specified "
1262 "as pixel data destination.");
1263
1264 int scanLineMin = min (scanLine1, scanLine2);
1265 int scanLineMax = max (scanLine1, scanLine2);
1266
1267 if (scanLineMin < _data->minY || scanLineMax > _data->maxY)
1268 throw IEX_NAMESPACE::ArgExc ("Tried to read scan line outside "
1269 "the image file's data window.");
1270
1271 for (int i = scanLineMin; i <= scanLineMax; i++)
1272 {
1273 if (_data->gotSampleCount[i - _data->minY] == false)
1274 throw IEX_NAMESPACE::ArgExc ("Tried to read scan line without "
1275 "knowing the sample counts, please"
1276 "read the sample counts first.");
1277 }
1278
1279
1280 //
1281 // We impose a numbering scheme on the lineBuffers where the first
1282 // scanline is contained in lineBuffer 1.
1283 //
1284 // Determine the first and last lineBuffer numbers in this scanline
1285 // range. We always attempt to read the scanlines in the order that
1286 // they are stored in the file.
1287 //
1288
1289 int start, stop, dl;
1290
1291 if (_data->lineOrder == INCREASING_Y)
1292 {
1293 start = (scanLineMin - _data->minY) / _data->linesInBuffer;
1294 stop = (scanLineMax - _data->minY) / _data->linesInBuffer + 1;
1295 dl = 1;
1296 }
1297 else
1298 {
1299 start = (scanLineMax - _data->minY) / _data->linesInBuffer;
1300 stop = (scanLineMin - _data->minY) / _data->linesInBuffer - 1;
1301 dl = -1;
1302 }
1303
1304 //
1305 // Create a task group for all line buffer tasks. When the
1306 // task group goes out of scope, the destructor waits until
1307 // all tasks are complete.
1308 //
1309
1310 {
1311 TaskGroup taskGroup;
1312
1313 //
1314 // Add the line buffer tasks.
1315 //
1316 // The tasks will execute in the order that they are created
1317 // because we lock the line buffers during construction and the
1318 // constructors are called by the main thread. Hence, in order
1319 // for a successive task to execute the previous task which
1320 // used that line buffer must have completed already.
1321 //
1322
1323 for (int l = start; l != stop; l += dl)
1324 {
1325 ThreadPool::addGlobalTask (newLineBufferTask (&taskGroup,
1326 _data, l,
1327 scanLineMin,
1328 scanLineMax));
1329 }
1330
1331 //
1332 // finish all tasks
1333 //
1334 }
1335
1336 //
1337 // Exeption handling:
1338 //
1339 // LineBufferTask::execute() may have encountered exceptions, but
1340 // those exceptions occurred in another thread, not in the thread
1341 // that is executing this call to ScanLineInputFile::readPixels().
1342 // LineBufferTask::execute() has caught all exceptions and stored
1343 // the exceptions' what() strings in the line buffers.
1344 // Now we check if any line buffer contains a stored exception; if
1345 // this is the case then we re-throw the exception in this thread.
1346 // (It is possible that multiple line buffers contain stored
1347 // exceptions. We re-throw the first exception we find and
1348 // ignore all others.)
1349 //
1350
1351 const string *exception = 0;
1352
1353 for (size_t i = 0; i < _data->lineBuffers.size(); ++i)
1354 {
1355 LineBuffer *lineBuffer = _data->lineBuffers[i];
1356
1357 if (lineBuffer->hasException && !exception)
1358 exception = &lineBuffer->exception;
1359
1360 lineBuffer->hasException = false;
1361 }
1362
1363 if (exception)
1364 throw IEX_NAMESPACE::IoExc (*exception);
1365 }
1366 catch (IEX_NAMESPACE::BaseExc &e)
1367 {
1368 REPLACE_EXC (e, "Error reading pixel data from image "
1369 "file \"" << fileName() << "\". " << e);
1370 throw;
1371 }
1372 }
1373
1374
1375 void
readPixels(int scanLine)1376 DeepScanLineInputFile::readPixels (int scanLine)
1377 {
1378 readPixels (scanLine, scanLine);
1379 }
1380
1381
1382 void
rawPixelData(int firstScanLine,char * pixelData,Int64 & pixelDataSize)1383 DeepScanLineInputFile::rawPixelData (int firstScanLine,
1384 char *pixelData,
1385 Int64 &pixelDataSize)
1386 {
1387
1388
1389 int minY = lineBufferMinY
1390 (firstScanLine, _data->minY, _data->linesInBuffer);
1391 int lineBufferNumber = (minY - _data->minY) / _data->linesInBuffer;
1392
1393 Int64 lineOffset = _data->lineOffsets[lineBufferNumber];
1394
1395 if (lineOffset == 0)
1396 THROW (IEX_NAMESPACE::InputExc, "Scan line " << minY << " is missing.");
1397
1398
1399 // enter the lock here - prevent another thread reseeking the file during read
1400 Lock lock (*_data->_streamData);
1401
1402 //
1403 // Seek to the start of the scan line in the file,
1404 //
1405
1406 if (_data->_streamData->is->tellg() != _data->lineOffsets[lineBufferNumber])
1407 _data->_streamData->is->seekg (lineOffset);
1408
1409 //
1410 // Read the data block's header.
1411 //
1412
1413 int yInFile;
1414
1415 //
1416 // Read the part number when we are dealing with a multi-part file.
1417 //
1418
1419 if (isMultiPart(_data->version))
1420 {
1421 int partNumber;
1422 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*_data->_streamData->is, partNumber);
1423 if (partNumber != _data->partNumber)
1424 {
1425 THROW (IEX_NAMESPACE::ArgExc, "Unexpected part number " << partNumber
1426 << ", should be " << _data->partNumber << ".");
1427 }
1428 }
1429
1430 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*_data->_streamData->is, yInFile);
1431
1432 if (yInFile != minY)
1433 throw IEX_NAMESPACE::InputExc ("Unexpected data block y coordinate.");
1434
1435 Int64 sampleCountTableSize;
1436 Int64 packedDataSize;
1437 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*_data->_streamData->is, sampleCountTableSize);
1438 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*_data->_streamData->is, packedDataSize);
1439
1440 // total requirement for reading all the data
1441
1442 Int64 totalSizeRequired=28+sampleCountTableSize+packedDataSize;
1443
1444 bool big_enough = totalSizeRequired<=pixelDataSize;
1445
1446 pixelDataSize = totalSizeRequired;
1447
1448 // was the block we were given big enough?
1449 if(!big_enough || pixelData==NULL)
1450 {
1451 // special case: seek stream back to start if we are at the beginning (regular reading pixels assumes it doesn't need to seek
1452 // in single part files)
1453 if(!isMultiPart(_data->version))
1454 {
1455 if (_data->nextLineBufferMinY == minY)
1456 _data->_streamData->is->seekg (lineOffset);
1457 }
1458 // leave lock here - bail before reading more data
1459 return;
1460 }
1461
1462 // copy the values we have read into the output block
1463 *(int *) pixelData = yInFile;
1464 *(Int64 *) (pixelData+4) =sampleCountTableSize;
1465 *(Int64 *) (pixelData+12) = packedDataSize;
1466
1467 // didn't read the unpackedsize - do that now
1468 Xdr::read<StreamIO> (*_data->_streamData->is, *(Int64 *) (pixelData+20));
1469
1470 // read the actual data
1471 _data->_streamData->is->read(pixelData+28, sampleCountTableSize+packedDataSize);
1472
1473 // special case: seek stream back to start if we are at the beginning (regular reading pixels assumes it doesn't need to seek
1474 // in single part files)
1475 if(!isMultiPart(_data->version))
1476 {
1477 if (_data->nextLineBufferMinY == minY)
1478 _data->_streamData->is->seekg (lineOffset);
1479 }
1480
1481 // leave lock here
1482
1483 }
1484
readPixels(const char * rawPixelData,const DeepFrameBuffer & frameBuffer,int scanLine1,int scanLine2) const1485 void DeepScanLineInputFile::readPixels (const char* rawPixelData,
1486 const DeepFrameBuffer& frameBuffer,
1487 int scanLine1,
1488 int scanLine2) const
1489 {
1490 //
1491 // read header from block - already converted from Xdr to native format
1492 //
1493 int data_scanline = *(int *) rawPixelData;
1494 Int64 sampleCountTableDataSize=*(Int64 *) (rawPixelData+4);
1495 Int64 packedDataSize = *(Int64 *) (rawPixelData+12);
1496 Int64 unpackedDataSize = *(Int64 *) (rawPixelData+20);
1497
1498
1499
1500 //
1501 // Uncompress the data, if necessary
1502 //
1503
1504
1505 Compressor * decomp = NULL;
1506 const char * uncompressed_data;
1507 Compressor::Format format = Compressor::XDR;
1508 if(packedDataSize <unpackedDataSize)
1509 {
1510 decomp = newCompressor(_data->header.compression(),
1511 unpackedDataSize,
1512 _data->header);
1513
1514 decomp->uncompress(rawPixelData+28+sampleCountTableDataSize,
1515 packedDataSize,
1516 data_scanline, uncompressed_data);
1517 format = decomp->format();
1518 }
1519 else
1520 {
1521 //
1522 // If the line is uncompressed, it's in XDR format,
1523 // regardless of the compressor's output format.
1524 //
1525
1526 format = Compressor::XDR;
1527 uncompressed_data = rawPixelData+28+sampleCountTableDataSize;
1528 }
1529
1530
1531 int yStart, yStop, dy;
1532
1533 if (_data->lineOrder == INCREASING_Y)
1534 {
1535 yStart = scanLine1;
1536 yStop = scanLine2 + 1;
1537 dy = 1;
1538 }
1539 else
1540 {
1541 yStart = scanLine2;
1542 yStop = scanLine1 - 1;
1543 dy = -1;
1544 }
1545
1546
1547
1548 const char* samplecount_base = frameBuffer.getSampleCountSlice().base;
1549 int samplecount_xstride = frameBuffer.getSampleCountSlice().xStride;
1550 int samplecount_ystride = frameBuffer.getSampleCountSlice().yStride;
1551
1552 //
1553 // For each line within the block, get the count of bytes.
1554 //
1555
1556 int minYInLineBuffer = data_scanline;
1557 int maxYInLineBuffer = min(minYInLineBuffer + _data->linesInBuffer - 1, _data->maxY);
1558
1559 vector<size_t> bytesPerLine(1+_data->maxY-_data->minY);
1560
1561
1562 bytesPerDeepLineTable (_data->header,
1563 minYInLineBuffer,
1564 maxYInLineBuffer,
1565 samplecount_base,
1566 samplecount_xstride,
1567 samplecount_ystride,
1568 bytesPerLine);
1569
1570 //
1571 // For each scanline within the block, get the offset.
1572 //
1573
1574 vector<size_t> offsetInLineBuffer;
1575 offsetInLineBufferTable (bytesPerLine,
1576 minYInLineBuffer - _data->minY,
1577 maxYInLineBuffer - _data->minY,
1578 _data->linesInBuffer,
1579 offsetInLineBuffer);
1580
1581
1582 const ChannelList & channels=header().channels();
1583
1584
1585 for (int y = yStart; y != yStop; y += dy)
1586 {
1587
1588 const char *readPtr =uncompressed_data +
1589 offsetInLineBuffer[y - _data->minY];
1590
1591 //
1592 // need to know the total number of samples on a scanline to skip channels
1593 // compute on demand: -1 means uncomputed
1594 //
1595 int lineSampleCount = -1;
1596
1597
1598 //
1599 // Iterate over all image channels in frame buffer
1600 //
1601
1602
1603 ChannelList::ConstIterator i = channels.begin();
1604
1605 for (DeepFrameBuffer::ConstIterator j = frameBuffer.begin();
1606 j != frameBuffer.end();
1607 ++j)
1608 {
1609 while (i != channels.end() && strcmp (i.name(), j.name()) < 0)
1610 {
1611 //
1612 // Channel i is present in the file but not
1613 // in the frame buffer; skip
1614
1615 if(lineSampleCount==-1)
1616 {
1617 lineSampleCount=0;
1618 const char * ptr = (samplecount_base+y*samplecount_ystride + samplecount_xstride*_data->minX);
1619 for(int x=_data->minX;x<=_data->maxX;x++)
1620 {
1621
1622 lineSampleCount+=*(const unsigned int *) ptr;
1623 ptr+=samplecount_xstride;
1624 }
1625 }
1626
1627 skipChannel (readPtr, i.channel().type, lineSampleCount );
1628
1629 ++i;
1630 }
1631
1632 bool fill = false;
1633
1634 if (i == channels.end() || strcmp (i.name(), j.name()) > 0)
1635 {
1636 //
1637 // Channel i is present in the frame buffer, but not in the file.
1638 // In the frame buffer, slice j will be filled with a default value.
1639 //
1640
1641 fill = true;
1642 }
1643 if (modp (y, i.channel().ySampling) == 0)
1644 {
1645
1646 copyIntoDeepFrameBuffer (readPtr, j.slice().base,
1647 samplecount_base,
1648 samplecount_xstride,
1649 samplecount_ystride,
1650 y, _data->minX, _data->maxX,
1651 0, 0,
1652 0, 0,
1653 j.slice().sampleStride,
1654 j.slice().xStride,
1655 j.slice().yStride,
1656 fill,
1657 j.slice().fillValue,
1658 format,
1659 j.slice().type,
1660 i.channel().type);
1661
1662 ++i;
1663
1664 }
1665 }//next slice in framebuffer
1666 }//next row in image
1667
1668 //
1669 // clean up
1670 //
1671
1672 delete decomp;
1673 }
1674
1675
1676
readPixelSampleCounts(const char * rawPixelData,const DeepFrameBuffer & frameBuffer,int scanLine1,int scanLine2) const1677 void DeepScanLineInputFile::readPixelSampleCounts (const char* rawPixelData,
1678 const DeepFrameBuffer& frameBuffer,
1679 int scanLine1,
1680 int scanLine2) const
1681 {
1682 //
1683 // read header from block - already converted from Xdr to native format
1684 //
1685 int data_scanline = *(int *) rawPixelData;
1686 Int64 sampleCountTableDataSize=*(Int64 *) (rawPixelData+4);
1687
1688
1689 int maxY;
1690 maxY = min(data_scanline + _data->linesInBuffer - 1, _data->maxY);
1691
1692 if(scanLine1 != data_scanline)
1693 {
1694 THROW(IEX_NAMESPACE::ArgExc,"readPixelSampleCounts(rawPixelData,frameBuffer,"<< scanLine1 << ',' << scanLine2 << ") called with incorrect start scanline - should be " << data_scanline );
1695 }
1696
1697 if(scanLine2 != maxY)
1698 {
1699 THROW(IEX_NAMESPACE::ArgExc,"readPixelSampleCounts(rawPixelData,frameBuffer,"<< scanLine1 << ',' << scanLine2 << ") called with incorrect end scanline - should be " << maxY );
1700 }
1701
1702
1703 //
1704 // If the sample count table is compressed, we'll uncompress it.
1705 //
1706
1707 Int64 rawSampleCountTableSize = (maxY - data_scanline + 1) * (_data->maxX - _data->minX + 1) *
1708 Xdr::size <unsigned int> ();
1709
1710
1711 Compressor * decomp=NULL;
1712 const char* readPtr;
1713 if (sampleCountTableDataSize < rawSampleCountTableSize)
1714 {
1715 decomp = newCompressor(_data->header.compression(),
1716 rawSampleCountTableSize,
1717 _data->header);
1718
1719 decomp->uncompress(rawPixelData+28,
1720 sampleCountTableDataSize,
1721 data_scanline,
1722 readPtr);
1723 }
1724 else readPtr = rawPixelData+28;
1725
1726 char* base = frameBuffer.getSampleCountSlice().base;
1727 int xStride = frameBuffer.getSampleCountSlice().xStride;
1728 int yStride = frameBuffer.getSampleCountSlice().yStride;
1729
1730
1731
1732 for (int y = scanLine1; y <= scanLine2; y++)
1733 {
1734 int lastAccumulatedCount = 0;
1735 for (int x = _data->minX; x <= _data->maxX; x++)
1736 {
1737 int accumulatedCount, count;
1738
1739 //
1740 // Read the sample count for pixel (x, y).
1741 //
1742
1743 Xdr::read <CharPtrIO> (readPtr, accumulatedCount);
1744 if (x == _data->minX)
1745 count = accumulatedCount;
1746 else
1747 count = accumulatedCount - lastAccumulatedCount;
1748 lastAccumulatedCount = accumulatedCount;
1749
1750 //
1751 // Store the data in both internal and external data structure.
1752 //
1753
1754 sampleCount(base, xStride, yStride, x, y) = count;
1755 }
1756 }
1757
1758 if(decomp)
1759 {
1760 delete decomp;
1761 }
1762 }
1763
1764
1765
1766 namespace
1767 {
1768
1769 void
readSampleCountForLineBlock(InputStreamMutex * streamData,DeepScanLineInputFile::Data * data,int lineBlockId)1770 readSampleCountForLineBlock(InputStreamMutex* streamData,
1771 DeepScanLineInputFile::Data* data,
1772 int lineBlockId)
1773 {
1774 streamData->is->seekg(data->lineOffsets[lineBlockId]);
1775
1776 if (isMultiPart(data->version))
1777 {
1778 int partNumber;
1779 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*streamData->is, partNumber);
1780
1781 if (partNumber != data->partNumber)
1782 throw IEX_NAMESPACE::ArgExc("Unexpected part number.");
1783 }
1784
1785 int minY;
1786 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*streamData->is, minY);
1787
1788 //
1789 // Check the correctness of minY.
1790 //
1791
1792 if (minY != data->minY + lineBlockId * data->linesInBuffer)
1793 throw IEX_NAMESPACE::ArgExc("Unexpected data block y coordinate.");
1794
1795 int maxY;
1796 maxY = min(minY + data->linesInBuffer - 1, data->maxY);
1797
1798 Int64 sampleCountTableDataSize;
1799 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*streamData->is, sampleCountTableDataSize);
1800
1801
1802
1803 if(sampleCountTableDataSize>data->maxSampleCountTableSize)
1804 {
1805 THROW (IEX_NAMESPACE::ArgExc, "Bad sampleCountTableDataSize read from chunk "<< lineBlockId << ": expected " << data->maxSampleCountTableSize << " or less, got "<< sampleCountTableDataSize);
1806 }
1807
1808 Int64 packedDataSize;
1809 Int64 unpackedDataSize;
1810 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*streamData->is, packedDataSize);
1811 OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read <OPENEXR_IMF_INTERNAL_NAMESPACE::StreamIO> (*streamData->is, unpackedDataSize);
1812
1813
1814
1815 //
1816 // We make a check on the data size requirements here.
1817 // Whilst we wish to store 64bit sizes on disk, not all the compressors
1818 // have been made to work with such data sizes and are still limited to
1819 // using signed 32 bit (int) for the data size. As such, this version
1820 // insists that we validate that the data size does not exceed the data
1821 // type max limit.
1822 // @TODO refactor the compressor code to ensure full 64-bit support.
1823 //
1824
1825 int compressorMaxDataSize = std::numeric_limits<int>::max();
1826 if (sampleCountTableDataSize > Int64(compressorMaxDataSize))
1827 {
1828 THROW (IEX_NAMESPACE::ArgExc, "This version of the library does not "
1829 << "support the allocation of data with size > "
1830 << compressorMaxDataSize
1831 << " file table size :" << sampleCountTableDataSize << ".\n");
1832 }
1833 streamData->is->read(data->sampleCountTableBuffer, sampleCountTableDataSize);
1834
1835 const char* readPtr;
1836
1837 //
1838 // If the sample count table is compressed, we'll uncompress it.
1839 //
1840
1841
1842 if (sampleCountTableDataSize < data->maxSampleCountTableSize)
1843 {
1844 if(!data->sampleCountTableComp)
1845 {
1846 THROW(IEX_NAMESPACE::ArgExc,"Deep scanline data corrupt at chunk " << lineBlockId << " (sampleCountTableDataSize error)");
1847 }
1848 data->sampleCountTableComp->uncompress(data->sampleCountTableBuffer,
1849 sampleCountTableDataSize,
1850 minY,
1851 readPtr);
1852 }
1853 else readPtr = data->sampleCountTableBuffer;
1854
1855 char* base = data->sampleCountSliceBase;
1856 int xStride = data->sampleCountXStride;
1857 int yStride = data->sampleCountYStride;
1858
1859 // total number of samples in block: used to check samplecount table doesn't
1860 // reference more data than exists
1861
1862 size_t cumulative_total_samples=0;
1863
1864 for (int y = minY; y <= maxY; y++)
1865 {
1866 int yInDataWindow = y - data->minY;
1867 data->lineSampleCount[yInDataWindow] = 0;
1868
1869 int lastAccumulatedCount = 0;
1870 for (int x = data->minX; x <= data->maxX; x++)
1871 {
1872 int accumulatedCount, count;
1873
1874 //
1875 // Read the sample count for pixel (x, y).
1876 //
1877
1878 Xdr::read <CharPtrIO> (readPtr, accumulatedCount);
1879
1880 // sample count table should always contain monotonically
1881 // increasing values.
1882 if (accumulatedCount < lastAccumulatedCount)
1883 {
1884 THROW(IEX_NAMESPACE::ArgExc,"Deep scanline sampleCount data corrupt at chunk " << lineBlockId << " (negative sample count detected)");
1885 }
1886
1887 count = accumulatedCount - lastAccumulatedCount;
1888 lastAccumulatedCount = accumulatedCount;
1889
1890 //
1891 // Store the data in both internal and external data structure.
1892 //
1893
1894 data->sampleCount[yInDataWindow][x - data->minX] = count;
1895 data->lineSampleCount[yInDataWindow] += count;
1896 sampleCount(base, xStride, yStride, x, y) = count;
1897 }
1898 cumulative_total_samples+=data->lineSampleCount[yInDataWindow];
1899 if(cumulative_total_samples*data->combinedSampleSize > unpackedDataSize)
1900 {
1901 THROW(IEX_NAMESPACE::ArgExc,"Deep scanline sampleCount data corrupt at chunk " << lineBlockId << ": pixel data only contains " << unpackedDataSize
1902 << " bytes of data but table references at least " << cumulative_total_samples*data->combinedSampleSize << " bytes of sample data" );
1903 }
1904 data->gotSampleCount[y - data->minY] = true;
1905 }
1906 }
1907
1908
1909 void
fillSampleCountFromCache(int y,DeepScanLineInputFile::Data * data)1910 fillSampleCountFromCache(int y, DeepScanLineInputFile::Data* data)
1911 {
1912 int yInDataWindow = y - data->minY;
1913 char* base = data->sampleCountSliceBase;
1914 int xStride = data->sampleCountXStride;
1915 int yStride = data->sampleCountYStride;
1916
1917 for (int x = data->minX; x <= data->maxX; x++)
1918 {
1919 unsigned int count = data->sampleCount[yInDataWindow][x - data->minX];
1920 sampleCount(base, xStride, yStride, x, y) = count;
1921 }
1922 }
1923
1924 } // namespace
1925
1926 void
readPixelSampleCounts(int scanline1,int scanline2)1927 DeepScanLineInputFile::readPixelSampleCounts (int scanline1, int scanline2)
1928 {
1929 Int64 savedFilePos = 0;
1930
1931 if(!_data->frameBufferValid)
1932 {
1933 throw IEX_NAMESPACE::ArgExc("readPixelSampleCounts called with no valid frame buffer");
1934 }
1935
1936 try
1937 {
1938 Lock lock (*_data->_streamData);
1939
1940 savedFilePos = _data->_streamData->is->tellg();
1941
1942 int scanLineMin = min (scanline1, scanline2);
1943 int scanLineMax = max (scanline1, scanline2);
1944
1945 if (scanLineMin < _data->minY || scanLineMax > _data->maxY)
1946 throw IEX_NAMESPACE::ArgExc ("Tried to read scan line sample counts outside "
1947 "the image file's data window.");
1948
1949 for (int i = scanLineMin; i <= scanLineMax; i++)
1950 {
1951 //
1952 // if scanline is already read, it'll be in the cache
1953 // otherwise, read from file, store in cache and in caller's framebuffer
1954 //
1955 if (_data->gotSampleCount[i - _data->minY])
1956 {
1957 fillSampleCountFromCache(i,_data);
1958
1959 }else{
1960
1961 int lineBlockId = ( i - _data->minY ) / _data->linesInBuffer;
1962
1963 readSampleCountForLineBlock ( _data->_streamData, _data, lineBlockId );
1964
1965 int minYInLineBuffer = lineBlockId * _data->linesInBuffer + _data->minY;
1966 int maxYInLineBuffer = min ( minYInLineBuffer + _data->linesInBuffer - 1, _data->maxY );
1967
1968 //
1969 // For each line within the block, get the count of bytes.
1970 //
1971
1972 bytesPerDeepLineTable ( _data->header,
1973 minYInLineBuffer,
1974 maxYInLineBuffer,
1975 _data->sampleCountSliceBase,
1976 _data->sampleCountXStride,
1977 _data->sampleCountYStride,
1978 _data->bytesPerLine );
1979
1980 //
1981 // For each scanline within the block, get the offset.
1982 //
1983
1984 offsetInLineBufferTable ( _data->bytesPerLine,
1985 minYInLineBuffer - _data->minY,
1986 maxYInLineBuffer - _data->minY,
1987 _data->linesInBuffer,
1988 _data->offsetInLineBuffer );
1989 }
1990 }
1991
1992 _data->_streamData->is->seekg(savedFilePos);
1993 }
1994 catch (IEX_NAMESPACE::BaseExc &e)
1995 {
1996 REPLACE_EXC (e, "Error reading sample count data from image "
1997 "file \"" << fileName() << "\". " << e);
1998
1999 _data->_streamData->is->seekg(savedFilePos);
2000
2001 throw;
2002 }
2003 }
2004
2005 void
readPixelSampleCounts(int scanline)2006 DeepScanLineInputFile::readPixelSampleCounts(int scanline)
2007 {
2008 readPixelSampleCounts(scanline, scanline);
2009 }
2010
2011 int
firstScanLineInChunk(int y) const2012 DeepScanLineInputFile::firstScanLineInChunk(int y) const
2013 {
2014 return int((y-_data->minY)/_data->linesInBuffer)*_data->linesInBuffer + _data->minY;
2015 }
2016
2017 int
lastScanLineInChunk(int y) const2018 DeepScanLineInputFile::lastScanLineInChunk(int y) const
2019 {
2020 int minY = firstScanLineInChunk(y);
2021 return min(minY+_data->linesInBuffer-1,_data->maxY);
2022 }
2023
2024
2025 OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT
2026