1 /******************************************************************************
2 *
3 * Project: Generic Raw Binary Driver
4 * Purpose: Implementation of RawDataset and RawRasterBand classes.
5 * Author: Frank Warmerdam, warmerda@pobox.com
6 *
7 ******************************************************************************
8 * Copyright (c) 1999, Frank Warmerdam
9 * Copyright (c) 2007-2014, Even Rouault <even dot rouault at spatialys.com>
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a
12 * copy of this software and associated documentation files (the "Software"),
13 * to deal in the Software without restriction, including without limitation
14 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 * and/or sell copies of the Software, and to permit persons to whom the
16 * Software is furnished to do so, subject to the following conditions:
17 *
18 * The above copyright notice and this permission notice shall be included
19 * in all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 * DEALINGS IN THE SOFTWARE.
28 ****************************************************************************/
29
30 #include "cpl_port.h"
31 #include "cpl_vax.h"
32 #include "rawdataset.h"
33
34 #include <climits>
35 #include <cmath>
36 #include <cstddef>
37 #include <cstdint>
38 #include <cstdlib>
39 #include <cstring>
40 #if HAVE_FCNTL_H
41 # include <fcntl.h>
42 #endif
43 #include <algorithm>
44 #include <limits>
45 #include <vector>
46
47 #include "cpl_conv.h"
48 #include "cpl_error.h"
49 #include "cpl_progress.h"
50 #include "cpl_string.h"
51 #include "cpl_virtualmem.h"
52 #include "cpl_vsi.h"
53 #include "cpl_safemaths.hpp"
54 #include "gdal.h"
55 #include "gdal_priv.h"
56
57 CPL_CVSID("$Id: rawdataset.cpp 11d5fd1088af3c1c129433e30914108ebbc82750 2021-04-06 21:39:37 +0200 Even Rouault $")
58
59 /************************************************************************/
60 /* RawRasterBand() */
61 /************************************************************************/
62
RawRasterBand(GDALDataset * poDSIn,int nBandIn,VSILFILE * fpRawLIn,vsi_l_offset nImgOffsetIn,int nPixelOffsetIn,int nLineOffsetIn,GDALDataType eDataTypeIn,int bNativeOrderIn,OwnFP bOwnsFPIn)63 RawRasterBand::RawRasterBand( GDALDataset *poDSIn, int nBandIn,
64 VSILFILE *fpRawLIn, vsi_l_offset nImgOffsetIn,
65 int nPixelOffsetIn, int nLineOffsetIn,
66 GDALDataType eDataTypeIn,
67 int bNativeOrderIn,
68 OwnFP bOwnsFPIn ) :
69 RawRasterBand(poDSIn, nBandIn, fpRawLIn,
70 nImgOffsetIn, nPixelOffsetIn, nLineOffsetIn,
71 eDataTypeIn,
72 #ifdef CPL_LSB
73 bNativeOrderIn ? ByteOrder::ORDER_LITTLE_ENDIAN : ByteOrder::ORDER_BIG_ENDIAN,
74 #else
75 bNativeOrderIn ? ByteOrder::ORDER_BIG_ENDIAN : ByteOrder::ORDER_LITTLE_ENDIAN,
76 #endif
77 bOwnsFPIn)
78 {
79 }
80
81 /************************************************************************/
82 /* RawRasterBand() */
83 /************************************************************************/
84
RawRasterBand(GDALDataset * poDSIn,int nBandIn,VSILFILE * fpRawLIn,vsi_l_offset nImgOffsetIn,int nPixelOffsetIn,int nLineOffsetIn,GDALDataType eDataTypeIn,ByteOrder eByteOrderIn,OwnFP bOwnsFPIn)85 RawRasterBand::RawRasterBand( GDALDataset *poDSIn, int nBandIn,
86 VSILFILE *fpRawLIn, vsi_l_offset nImgOffsetIn,
87 int nPixelOffsetIn, int nLineOffsetIn,
88 GDALDataType eDataTypeIn,
89 ByteOrder eByteOrderIn,
90 OwnFP bOwnsFPIn ) :
91 fpRawL(fpRawLIn),
92 nImgOffset(nImgOffsetIn),
93 nPixelOffset(nPixelOffsetIn),
94 nLineOffset(nLineOffsetIn),
95 eByteOrder(eByteOrderIn),
96 bOwnsFP(bOwnsFPIn == OwnFP::YES)
97 {
98 poDS = poDSIn;
99 nBand = nBandIn;
100 eDataType = eDataTypeIn;
101 nRasterXSize = poDSIn->GetRasterXSize();
102 nRasterYSize = poDSIn->GetRasterYSize();
103
104 CPLDebug("GDALRaw",
105 "RawRasterBand(%p,%d,%p,\n"
106 " Off=%d,PixOff=%d,LineOff=%d,%s,%d)",
107 poDS, nBand, fpRawL,
108 static_cast<unsigned int>(nImgOffset), nPixelOffset, nLineOffset,
109 GDALGetDataTypeName(eDataType), static_cast<int>(eByteOrder));
110
111 // Treat one scanline as the block size.
112 nBlockXSize = poDS->GetRasterXSize();
113 nBlockYSize = 1;
114
115 // Initialize other fields, and setup the line buffer.
116 Initialize();
117 }
118
119 /************************************************************************/
120 /* RawRasterBand() */
121 /************************************************************************/
122
RawRasterBand(VSILFILE * fpRawLIn,vsi_l_offset nImgOffsetIn,int nPixelOffsetIn,int nLineOffsetIn,GDALDataType eDataTypeIn,int bNativeOrderIn,int nXSize,int nYSize,OwnFP bOwnsFPIn)123 RawRasterBand::RawRasterBand( VSILFILE *fpRawLIn, vsi_l_offset nImgOffsetIn,
124 int nPixelOffsetIn, int nLineOffsetIn,
125 GDALDataType eDataTypeIn, int bNativeOrderIn,
126 int nXSize, int nYSize,
127 OwnFP bOwnsFPIn ) :
128 RawRasterBand(fpRawLIn, nImgOffsetIn, nPixelOffsetIn, nLineOffsetIn,
129 eDataTypeIn,
130 #ifdef CPL_LSB
131 bNativeOrderIn ? ByteOrder::ORDER_LITTLE_ENDIAN : ByteOrder::ORDER_BIG_ENDIAN,
132 #else
133 bNativeOrderIn ? ByteOrder::ORDER_BIG_ENDIAN : ByteOrder::ORDER_LITTLE_ENDIAN,
134 #endif
135 nXSize, nYSize,
136 bOwnsFPIn)
137 {
138 }
139
140 /************************************************************************/
141 /* RawRasterBand() */
142 /************************************************************************/
143
RawRasterBand(VSILFILE * fpRawLIn,vsi_l_offset nImgOffsetIn,int nPixelOffsetIn,int nLineOffsetIn,GDALDataType eDataTypeIn,ByteOrder eByteOrderIn,int nXSize,int nYSize,OwnFP bOwnsFPIn)144 RawRasterBand::RawRasterBand( VSILFILE *fpRawLIn, vsi_l_offset nImgOffsetIn,
145 int nPixelOffsetIn, int nLineOffsetIn,
146 GDALDataType eDataTypeIn,
147 ByteOrder eByteOrderIn,
148 int nXSize, int nYSize,
149 OwnFP bOwnsFPIn ) :
150 fpRawL(fpRawLIn),
151 nImgOffset(nImgOffsetIn),
152 nPixelOffset(nPixelOffsetIn),
153 nLineOffset(nLineOffsetIn),
154 eByteOrder(eByteOrderIn),
155 bOwnsFP(bOwnsFPIn == OwnFP::YES)
156 {
157 poDS = nullptr;
158 nBand = 1;
159 eDataType = eDataTypeIn;
160
161 CPLDebug("GDALRaw",
162 "RawRasterBand(floating,Off=%d,PixOff=%d,LineOff=%d,%s,%d)",
163 static_cast<unsigned int>(nImgOffset),
164 nPixelOffset, nLineOffset,
165 GDALGetDataTypeName(eDataType), static_cast<int>(eByteOrder));
166
167 // Treat one scanline as the block size.
168 nBlockXSize = nXSize;
169 nBlockYSize = 1;
170 nRasterXSize = nXSize;
171 nRasterYSize = nYSize;
172 if (!GDALCheckDatasetDimensions(nXSize, nYSize))
173 {
174 return;
175 }
176
177 // Initialize other fields, and setup the line buffer.
178 Initialize();
179 }
180
181 /************************************************************************/
182 /* Initialize() */
183 /************************************************************************/
184
Initialize()185 void RawRasterBand::Initialize()
186
187 {
188 vsi_l_offset nSmallestOffset = nImgOffset;
189 vsi_l_offset nLargestOffset = nImgOffset;
190 if( nLineOffset < 0 )
191 {
192 const auto nDelta = static_cast<vsi_l_offset>(
193 -static_cast<GIntBig>(nLineOffset)) * (nRasterYSize - 1);
194 if( nDelta > nImgOffset )
195 {
196 CPLError(CE_Failure, CPLE_AppDefined,
197 "Inconsistent nLineOffset, nRasterYSize and nImgOffset");
198 return;
199 }
200 nSmallestOffset -= nDelta;
201 }
202 else
203 {
204 if( nImgOffset > std::numeric_limits<vsi_l_offset>::max() -
205 static_cast<vsi_l_offset>(nLineOffset) * (nRasterYSize - 1) )
206 {
207 CPLError(CE_Failure, CPLE_AppDefined,
208 "Inconsistent nLineOffset, nRasterYSize and nImgOffset");
209 return;
210 }
211 nLargestOffset += static_cast<vsi_l_offset>(nLineOffset) * (nRasterYSize - 1);
212 }
213 if( nPixelOffset < 0 )
214 {
215 if( static_cast<vsi_l_offset>(-static_cast<GIntBig>(nPixelOffset)) *
216 (nRasterXSize - 1) > nSmallestOffset )
217 {
218 CPLError(CE_Failure, CPLE_AppDefined,
219 "Inconsistent nPixelOffset, nRasterXSize and nImgOffset");
220 return;
221 }
222 }
223 else
224 {
225 if( nLargestOffset > std::numeric_limits<vsi_l_offset>::max() -
226 static_cast<vsi_l_offset>(nPixelOffset) * (nRasterXSize - 1) )
227 {
228 CPLError(CE_Failure, CPLE_AppDefined,
229 "Inconsistent nPixelOffset, nRasterXSize and nImgOffset");
230 return;
231 }
232 nLargestOffset += static_cast<vsi_l_offset>(nPixelOffset) * (nRasterXSize - 1);
233 }
234 if( nLargestOffset > static_cast<vsi_l_offset>(GINTBIG_MAX) )
235 {
236 CPLError(CE_Failure, CPLE_AppDefined, "Too big largest offset");
237 return;
238 }
239
240
241 const int nDTSize = GDALGetDataTypeSizeBytes(GetRasterDataType());
242
243 // Allocate working scanline.
244 const bool bIsBIP = IsBIP();
245 if( bIsBIP )
246 {
247 if( nBand == 1 )
248 {
249 nLineSize = nPixelOffset * nBlockXSize;
250 pLineBuffer = VSIMalloc(nLineSize);
251 }
252 else
253 {
254 // Band > 1 : share the same buffer as band 1
255 pLineBuffer = nullptr;
256 const auto poFirstBand = cpl::down_cast<RawRasterBand*>(poDS->GetRasterBand(1));
257 if( poFirstBand->pLineBuffer != nullptr )
258 pLineStart = static_cast<char *>(poFirstBand->pLineBuffer) + (nBand - 1) * nDTSize;
259 return;
260 }
261 }
262 else if (nBlockXSize <= 0 ||
263 (nBlockXSize > 1 && std::abs(nPixelOffset) >
264 std::numeric_limits<int>::max() / (nBlockXSize - 1)) ||
265 std::abs(nPixelOffset) * (nBlockXSize - 1) >
266 std::numeric_limits<int>::max() - nDTSize)
267 {
268 nLineSize = 0;
269 pLineBuffer = nullptr;
270 }
271 else
272 {
273 nLineSize = std::abs(nPixelOffset) * (nBlockXSize - 1) + nDTSize;
274 pLineBuffer = VSIMalloc(nLineSize);
275 }
276
277 if (pLineBuffer == nullptr)
278 {
279 nLineSize = 0;
280 CPLError(CE_Failure, CPLE_AppDefined,
281 "Could not allocate line buffer: "
282 "nPixelOffset=%d, nBlockXSize=%d",
283 nPixelOffset, nBlockXSize);
284 return;
285 }
286
287 if( nPixelOffset >= 0 )
288 pLineStart = pLineBuffer;
289 else
290 pLineStart = static_cast<char *>(pLineBuffer) +
291 static_cast<std::ptrdiff_t>(std::abs(nPixelOffset)) *
292 (nBlockXSize - 1);
293 }
294
295 /************************************************************************/
296 /* ~RawRasterBand() */
297 /************************************************************************/
298
~RawRasterBand()299 RawRasterBand::~RawRasterBand()
300
301 {
302 if( poCT )
303 delete poCT;
304
305 CSLDestroy(papszCategoryNames);
306
307 RawRasterBand::FlushCache();
308
309 if (bOwnsFP)
310 {
311 if( VSIFCloseL(fpRawL) != 0 )
312 {
313 CPLError(CE_Failure, CPLE_FileIO, "I/O error");
314 }
315 }
316
317 CPLFree(pLineBuffer);
318 }
319
320 /************************************************************************/
321 /* IsBIP() */
322 /************************************************************************/
323
IsBIP() const324 bool RawRasterBand::IsBIP() const
325 {
326 const int nDTSize = GDALGetDataTypeSizeBytes(eDataType);
327 const bool bIsRawDataset = dynamic_cast<RawDataset*>(poDS) != nullptr;
328 if( bIsRawDataset && nPixelOffset > nDTSize &&
329 nLineOffset == static_cast<int64_t>(nPixelOffset) * nRasterXSize )
330 {
331 if( nBand == 1 )
332 {
333 return true;
334 }
335 const auto poFirstBand = dynamic_cast<RawRasterBand*>(poDS->GetRasterBand(1));
336 if( poFirstBand &&
337 eDataType == poFirstBand->eDataType &&
338 eByteOrder == poFirstBand->eByteOrder &&
339 nPixelOffset == poFirstBand->nPixelOffset &&
340 nLineOffset == poFirstBand->nLineOffset &&
341 nImgOffset == poFirstBand->nImgOffset + (nBand - 1) * nDTSize )
342 {
343 return true;
344 }
345 }
346 return false;
347 }
348
349 /************************************************************************/
350 /* SetAccess() */
351 /************************************************************************/
352
SetAccess(GDALAccess eAccessIn)353 void RawRasterBand::SetAccess(GDALAccess eAccessIn) { eAccess = eAccessIn; }
354
355 /************************************************************************/
356 /* FlushCache() */
357 /* */
358 /* We override this so we have the opportunity to call */
359 /* fflush(). We don't want to do this all the time in the */
360 /* write block function as it is kind of expensive. */
361 /************************************************************************/
362
FlushCache()363 CPLErr RawRasterBand::FlushCache()
364
365 {
366 CPLErr eErr = GDALRasterBand::FlushCache();
367 if( eErr != CE_None )
368 {
369 bNeedFileFlush = false;
370 return eErr;
371 }
372
373 RawRasterBand* masterBand = this;
374 if( nBand > 1 && poDS != nullptr && poDS->GetRasterCount() > 1 && IsBIP() )
375 {
376 // can't be null as IsBIP() checks that the first band is not null,
377 // which could happen during dataset destruction.
378 masterBand = cpl::down_cast<RawRasterBand *>(poDS->GetRasterBand(1));
379 }
380
381 if( !masterBand->FlushCurrentLine(false) )
382 {
383 masterBand->bNeedFileFlush = false;
384 return CE_Failure;
385 }
386
387 // If we have unflushed raw, flush it to disk now.
388 if ( masterBand->bNeedFileFlush )
389 {
390 int nRet = VSIFFlushL(fpRawL);
391
392 masterBand->bNeedFileFlush = false;
393 if( nRet < 0 )
394 return CE_Failure;
395 }
396
397 return CE_None;
398 }
399
400 /************************************************************************/
401 /* NeedsByteOrderChange() */
402 /************************************************************************/
403
NeedsByteOrderChange() const404 bool RawRasterBand::NeedsByteOrderChange() const
405 {
406 #ifdef CPL_LSB
407 return eDataType != GDT_Byte &&
408 eByteOrder != RawRasterBand::ByteOrder::ORDER_LITTLE_ENDIAN;
409 #else
410 return eDataType != GDT_Byte &&
411 eByteOrder != RawRasterBand::ByteOrder::ORDER_BIG_ENDIAN;
412 #endif
413 }
414
415 /************************************************************************/
416 /* DoByteSwap() */
417 /************************************************************************/
418
DoByteSwap(void * pBuffer,size_t nValues,int nByteSkip,bool bDiskToCPU) const419 void RawRasterBand::DoByteSwap(void* pBuffer, size_t nValues, int nByteSkip, bool bDiskToCPU) const
420 {
421 if( eByteOrder != RawRasterBand::ByteOrder::ORDER_VAX )
422 {
423 if( GDALDataTypeIsComplex(eDataType) )
424 {
425 const int nWordSize = GDALGetDataTypeSize(eDataType) / 16;
426 GDALSwapWordsEx(pBuffer, nWordSize, nValues, nByteSkip);
427 GDALSwapWordsEx(
428 static_cast<GByte *>(pBuffer) + nWordSize,
429 nWordSize, nValues, nByteSkip);
430 }
431 else
432 {
433 GDALSwapWordsEx(pBuffer, GDALGetDataTypeSizeBytes(eDataType),
434 nValues, nByteSkip);
435 }
436 }
437 else if( eDataType == GDT_Float32 || eDataType == GDT_CFloat32 )
438 {
439 GByte* pPtr = static_cast<GByte *>(pBuffer);
440 for( int k = 0; k < 2; k++ )
441 {
442 if( bDiskToCPU )
443 {
444 for( size_t i = 0; i < nValues; i++, pPtr += nByteSkip )
445 {
446 CPLVaxToIEEEFloat(pPtr);
447 }
448 }
449 else
450 {
451 for( size_t i = 0; i < nValues; i++, pPtr += nByteSkip )
452 {
453 CPLIEEEToVaxFloat(pPtr);
454 }
455 }
456 if( k == 0 && eDataType == GDT_CFloat32 )
457 pPtr = static_cast<GByte *>(pBuffer) + sizeof(float);
458 else
459 break;
460 }
461 }
462 else if( eDataType == GDT_Float64 || eDataType == GDT_CFloat64 )
463 {
464 GByte* pPtr = static_cast<GByte *>(pBuffer);
465 for( int k = 0; k < 2; k++ )
466 {
467 if( bDiskToCPU )
468 {
469 for( size_t i = 0; i < nValues; i++, pPtr += nByteSkip )
470 {
471 CPLVaxToIEEEDouble(pPtr);
472 }
473 }
474 else
475 {
476 for( size_t i = 0; i < nValues; i++, pPtr += nByteSkip )
477 {
478 CPLIEEEToVaxDouble(pPtr);
479 }
480 }
481 if( k == 0 && eDataType == GDT_CFloat64 )
482 pPtr = static_cast<GByte *>(pBuffer) + sizeof(double);
483 else
484 break;
485 }
486 }
487 }
488
489 /************************************************************************/
490 /* ComputeFileOffset() */
491 /************************************************************************/
492
ComputeFileOffset(int iLine) const493 vsi_l_offset RawRasterBand::ComputeFileOffset(int iLine) const
494 {
495 // Write formulas such that unsigned int overflow doesn't occur
496 vsi_l_offset nOffset = nImgOffset;
497 if( nLineOffset >= 0 )
498 {
499 nOffset += static_cast<GUIntBig>(nLineOffset) * iLine;
500 }
501 else
502 {
503 nOffset -= static_cast<GUIntBig>(-static_cast<GIntBig>(nLineOffset)) * iLine;
504 }
505 if( nPixelOffset < 0 )
506 {
507 const GUIntBig nPixelOffsetToSubtract =
508 static_cast<GUIntBig>(-static_cast<GIntBig>(nPixelOffset)) * (nBlockXSize - 1);
509 nOffset -= nPixelOffsetToSubtract;
510 }
511 return nOffset;
512 }
513
514 /************************************************************************/
515 /* AccessLine() */
516 /************************************************************************/
517
AccessLine(int iLine)518 CPLErr RawRasterBand::AccessLine( int iLine )
519
520 {
521 if (pLineBuffer == nullptr)
522 {
523 if( nBand > 1 && pLineStart != nullptr )
524 {
525 // BIP interleaved
526 return cpl::down_cast<RawRasterBand*>(poDS->GetRasterBand(1))->AccessLine(iLine);
527 }
528 return CE_Failure;
529 }
530
531 if( nLoadedScanline == iLine )
532 {
533 return CE_None;
534 }
535
536 if( !FlushCurrentLine(false) )
537 {
538 return CE_Failure;
539 }
540
541 // Figure out where to start reading.
542 const vsi_l_offset nReadStart = ComputeFileOffset(iLine);
543
544 // Seek to the correct line.
545 if( Seek(nReadStart, SEEK_SET) == -1 )
546 {
547 if (poDS != nullptr && poDS->GetAccess() == GA_ReadOnly)
548 {
549 CPLError(CE_Failure, CPLE_FileIO,
550 "Failed to seek to scanline %d @ " CPL_FRMT_GUIB ".",
551 iLine, nReadStart);
552 return CE_Failure;
553 }
554 else
555 {
556 memset(pLineBuffer, 0, nLineSize);
557 nLoadedScanline = iLine;
558 return CE_None;
559 }
560 }
561
562 // Read the line. Take care not to request any more bytes than
563 // are needed, and not to lose a partially successful scanline read.
564 const size_t nBytesToRead = nLineSize;
565 const size_t nBytesActuallyRead = Read(pLineBuffer, 1, nBytesToRead);
566 if( nBytesActuallyRead < nBytesToRead )
567 {
568 if (poDS != nullptr && poDS->GetAccess() == GA_ReadOnly &&
569 // ENVI datasets might be sparse (see #915)
570 poDS->GetMetadata("ENVI") == nullptr)
571 {
572 CPLError(CE_Failure, CPLE_FileIO,
573 "Failed to read scanline %d.",
574 iLine);
575 return CE_Failure;
576 }
577 else
578 {
579 memset(
580 static_cast<GByte *>(pLineBuffer) + nBytesActuallyRead,
581 0, nBytesToRead - nBytesActuallyRead);
582 }
583 }
584
585 // Byte swap the interesting data, if required.
586 if( NeedsByteOrderChange() )
587 {
588 if( poDS != nullptr && poDS->GetRasterCount() > 1 && IsBIP() )
589 {
590 const int nDTSize = GDALGetDataTypeSizeBytes(eDataType);
591 DoByteSwap(pLineBuffer, nBlockXSize * poDS->GetRasterCount(), nDTSize, true);
592 }
593 else
594 DoByteSwap(pLineBuffer, nBlockXSize, std::abs(nPixelOffset), true);
595 }
596
597 nLoadedScanline = iLine;
598
599 return CE_None;
600 }
601
602 /************************************************************************/
603 /* IReadBlock() */
604 /************************************************************************/
605
IReadBlock(CPL_UNUSED int nBlockXOff,int nBlockYOff,void * pImage)606 CPLErr RawRasterBand::IReadBlock(CPL_UNUSED int nBlockXOff,
607 int nBlockYOff,
608 void *pImage)
609 {
610 CPLAssert(nBlockXOff == 0);
611
612 const CPLErr eErr = AccessLine(nBlockYOff);
613 if( eErr == CE_Failure )
614 return eErr;
615
616 // Copy data from disk buffer to user block buffer.
617 const int nDTSize = GDALGetDataTypeSizeBytes(eDataType);
618 GDALCopyWords(pLineStart, eDataType, nPixelOffset,
619 pImage, eDataType, nDTSize,
620 nBlockXSize);
621
622 // Pre-cache block cache of other bands
623 if( poDS != nullptr && poDS->GetRasterCount() > 1 && IsBIP() )
624 {
625 for( int iBand = 1; iBand <= poDS->GetRasterCount(); iBand++ )
626 {
627 if( iBand != nBand )
628 {
629 auto poOtherBand = cpl::down_cast<RawRasterBand*>(poDS->GetRasterBand(iBand));
630 GDALRasterBlock *poBlock = poOtherBand->TryGetLockedBlockRef(0, nBlockYOff);
631 if( poBlock != nullptr )
632 {
633 poBlock->DropLock();
634 continue;
635 }
636 poBlock = poOtherBand->GetLockedBlockRef(0, nBlockYOff, true);
637 if( poBlock != nullptr )
638 {
639 GDALCopyWords(poOtherBand->pLineStart, eDataType, nPixelOffset,
640 poBlock->GetDataRef(), eDataType, nDTSize,
641 nBlockXSize);
642 poBlock->DropLock();
643 }
644 }
645 }
646 }
647
648 return eErr;
649 }
650
651 /************************************************************************/
652 /* BIPWriteBlock() */
653 /************************************************************************/
654
BIPWriteBlock(int nBlockYOff,int nCallingBand,const void * pImage)655 CPLErr RawRasterBand::BIPWriteBlock( int nBlockYOff,
656 int nCallingBand,
657 const void* pImage )
658 {
659 if( nLoadedScanline != nBlockYOff )
660 {
661 if( !FlushCurrentLine(false) )
662 return CE_Failure;
663 }
664
665 const int nBands = poDS->GetRasterCount();
666 std::vector<GDALRasterBlock*> apoBlocks(nBands);
667 bool bAllBlocksDirty = true;
668 const int nDTSize = GDALGetDataTypeSizeBytes(eDataType);
669
670 /* -------------------------------------------------------------------- */
671 /* If all blocks are cached and dirty then we do not need to reload */
672 /* the scanline from disk */
673 /* -------------------------------------------------------------------- */
674 for( int iBand = 0; iBand < nBands; ++iBand )
675 {
676 if( iBand + 1 != nCallingBand )
677 {
678 apoBlocks[iBand] =
679 cpl::down_cast<RawRasterBand *>(
680 poDS->GetRasterBand( iBand + 1 ))
681 ->TryGetLockedBlockRef( 0, nBlockYOff );
682
683 if( apoBlocks[iBand] == nullptr )
684 {
685 bAllBlocksDirty = false;
686 }
687 else if( !apoBlocks[iBand]->GetDirty() )
688 {
689 apoBlocks[iBand]->DropLock();
690 apoBlocks[iBand] = nullptr;
691 bAllBlocksDirty = false;
692 }
693 }
694 else
695 apoBlocks[iBand] = nullptr;
696 }
697
698 if( !bAllBlocksDirty )
699 {
700 // We only to read the scanline if we don't have data for all bands.
701 if( AccessLine(nBlockYOff) != CE_None )
702 {
703 for( int iBand = 0; iBand < nBands; ++iBand )
704 {
705 if( apoBlocks[iBand] != nullptr )
706 apoBlocks[iBand]->DropLock();
707 }
708 return CE_Failure;
709 }
710 }
711
712 for( int iBand = 0; iBand < nBands; ++iBand )
713 {
714 const GByte *pabyThisImage = nullptr;
715 GDALRasterBlock *poBlock = nullptr;
716
717 if( iBand + 1 == nCallingBand )
718 {
719 pabyThisImage = static_cast<const GByte *>( pImage );
720 }
721 else
722 {
723 poBlock = apoBlocks[iBand];
724 if( poBlock == nullptr )
725 continue;
726
727 if( !poBlock->GetDirty() )
728 {
729 poBlock->DropLock();
730 continue;
731 }
732
733 pabyThisImage = static_cast<const GByte *>( poBlock->GetDataRef() );
734 }
735
736 GByte *pabyOut = static_cast<GByte *>(pLineStart) + iBand * nDTSize;
737
738 GDALCopyWords(pabyThisImage, eDataType, nDTSize,
739 pabyOut, eDataType, nPixelOffset, nBlockXSize);
740
741 if( poBlock != nullptr )
742 {
743 poBlock->MarkClean();
744 poBlock->DropLock();
745 }
746 }
747
748 nLoadedScanline = nBlockYOff;
749 bLoadedScanlineDirty = true;
750
751 if( bAllBlocksDirty )
752 {
753 return FlushCurrentLine(true) ? CE_None : CE_Failure;
754 }
755
756 bNeedFileFlush = true;
757 return CE_None;
758 }
759
760
761 /************************************************************************/
762 /* IWriteBlock() */
763 /************************************************************************/
764
IWriteBlock(CPL_UNUSED int nBlockXOff,int nBlockYOff,void * pImage)765 CPLErr RawRasterBand::IWriteBlock( CPL_UNUSED int nBlockXOff,
766 int nBlockYOff,
767 void *pImage )
768 {
769 CPLAssert(nBlockXOff == 0);
770
771 if (pLineBuffer == nullptr)
772 {
773 if( poDS != nullptr && poDS->GetRasterCount() > 1 && IsBIP() )
774 {
775 auto poFirstBand = (nBand == 1) ? this :
776 cpl::down_cast<RawRasterBand *>(poDS->GetRasterBand(1));
777 CPLAssert(poFirstBand);
778 return poFirstBand->BIPWriteBlock(nBlockYOff, nBand, pImage);
779 }
780
781 return CE_Failure;
782 }
783
784 if( nLoadedScanline != nBlockYOff )
785 {
786 if( !FlushCurrentLine(false) )
787 return CE_Failure;
788 }
789
790 // If the data for this band is completely contiguous, we don't
791 // have to worry about pre-reading from disk.
792 CPLErr eErr = CE_None;
793 const int nDTSize = GDALGetDataTypeSizeBytes(eDataType);
794 if( std::abs(nPixelOffset) > nDTSize )
795 eErr = AccessLine(nBlockYOff);
796
797 // Copy data from user buffer into disk buffer.
798 GDALCopyWords(pImage, eDataType, nDTSize,
799 pLineStart, eDataType, nPixelOffset, nBlockXSize);
800
801 nLoadedScanline = nBlockYOff;
802 bLoadedScanlineDirty = true;
803
804 return eErr == CE_None && FlushCurrentLine(true) ? CE_None : CE_Failure;
805 }
806
807 /************************************************************************/
808 /* FlushCurrentLine() */
809 /************************************************************************/
810
FlushCurrentLine(bool bNeedUsableBufferAfter)811 bool RawRasterBand::FlushCurrentLine(bool bNeedUsableBufferAfter)
812 {
813 if( !bLoadedScanlineDirty )
814 return true;
815
816 bLoadedScanlineDirty = false;
817
818 bool ok = true;
819
820 // Byte swap (if necessary) back into disk order before writing.
821 if( NeedsByteOrderChange() )
822 {
823 if( poDS != nullptr && poDS->GetRasterCount() > 1 && IsBIP() )
824 {
825 const int nDTSize = GDALGetDataTypeSizeBytes(eDataType);
826 DoByteSwap(pLineBuffer, nBlockXSize * poDS->GetRasterCount(), nDTSize, false);
827 }
828 else
829 DoByteSwap(pLineBuffer, nBlockXSize, std::abs(nPixelOffset), false);
830 }
831
832 // Figure out where to start reading.
833 const vsi_l_offset nWriteStart = ComputeFileOffset(nLoadedScanline);
834
835 // Seek to correct location.
836 if( Seek(nWriteStart, SEEK_SET) == -1 )
837 {
838 CPLError(CE_Failure, CPLE_FileIO,
839 "Failed to seek to scanline %d @ " CPL_FRMT_GUIB
840 " to write to file.",
841 nLoadedScanline, nWriteStart);
842
843 ok = false;
844 }
845
846 // Write data buffer.
847 const int nBytesToWrite = nLineSize;
848 if( ok && Write(pLineBuffer, 1, nBytesToWrite) <
849 static_cast<size_t>(nBytesToWrite) )
850 {
851 CPLError(CE_Failure, CPLE_FileIO,
852 "Failed to write scanline %d to file.",
853 nLoadedScanline);
854
855 ok = false;
856 }
857
858 // Byte swap (if necessary) back into machine order so the
859 // buffer is still usable for reading purposes, unless this is not needed.
860 if( bNeedUsableBufferAfter && NeedsByteOrderChange() )
861 {
862 if( poDS != nullptr && poDS->GetRasterCount() > 1 && IsBIP() )
863 {
864 const int nDTSize = GDALGetDataTypeSizeBytes(eDataType);
865 DoByteSwap(pLineBuffer, nBlockXSize * poDS->GetRasterCount(), nDTSize, true);
866 }
867 else
868 DoByteSwap(pLineBuffer, nBlockXSize, std::abs(nPixelOffset), true);
869 }
870
871 bNeedFileFlush = true;
872
873 return ok;
874 }
875
876 /************************************************************************/
877 /* AccessBlock() */
878 /************************************************************************/
879
AccessBlock(vsi_l_offset nBlockOff,size_t nBlockSize,void * pData)880 CPLErr RawRasterBand::AccessBlock(vsi_l_offset nBlockOff, size_t nBlockSize,
881 void *pData)
882 {
883 // Seek to the correct block.
884 if( Seek(nBlockOff, SEEK_SET) == -1 )
885 {
886 memset(pData, 0, nBlockSize);
887 return CE_None;
888 }
889
890 // Read the block.
891 const size_t nBytesActuallyRead = Read(pData, 1, nBlockSize);
892 if( nBytesActuallyRead < nBlockSize )
893 {
894
895 memset(static_cast<GByte *>(pData) + nBytesActuallyRead,
896 0, nBlockSize - nBytesActuallyRead);
897 return CE_None;
898 }
899
900 // Byte swap the interesting data, if required.
901 if( NeedsByteOrderChange() )
902 {
903 DoByteSwap(pData, nBlockSize / nPixelOffset, std::abs(nPixelOffset), true);
904 }
905
906 return CE_None;
907 }
908
909 /************************************************************************/
910 /* IsSignificantNumberOfLinesLoaded() */
911 /* */
912 /* Check if there is a significant number of scanlines (>20%) from the */
913 /* specified block of lines already cached. */
914 /************************************************************************/
915
IsSignificantNumberOfLinesLoaded(int nLineOff,int nLines)916 int RawRasterBand::IsSignificantNumberOfLinesLoaded( int nLineOff, int nLines )
917 {
918 int nCountLoaded = 0;
919
920 for ( int iLine = nLineOff; iLine < nLineOff + nLines; iLine++ )
921 {
922 GDALRasterBlock *poBlock = TryGetLockedBlockRef(0, iLine);
923 if( poBlock != nullptr )
924 {
925 poBlock->DropLock();
926 nCountLoaded++;
927 if( nCountLoaded > nLines / 20 )
928 {
929 return TRUE;
930 }
931 }
932 }
933
934 return FALSE;
935 }
936
937 /************************************************************************/
938 /* CanUseDirectIO() */
939 /************************************************************************/
940
CanUseDirectIO(int,int nYOff,int nXSize,int nYSize,GDALDataType,GDALRasterIOExtraArg * psExtraArg)941 int RawRasterBand::CanUseDirectIO(int /* nXOff */,
942 int nYOff,
943 int nXSize,
944 int nYSize,
945 GDALDataType /* eBufType*/,
946 GDALRasterIOExtraArg* psExtraArg)
947 {
948
949 // Use direct IO without caching if:
950 //
951 // GDAL_ONE_BIG_READ is enabled
952 //
953 // or
954 //
955 // the length of a scanline on disk is more than 50000 bytes, and the
956 // width of the requested chunk is less than 40% of the whole scanline and
957 // no significant number of requested scanlines are already in the cache.
958
959 if( nPixelOffset < 0 ||
960 psExtraArg->eResampleAlg != GRIORA_NearestNeighbour )
961 {
962 return FALSE;
963 }
964
965 const char *pszGDAL_ONE_BIG_READ =
966 CPLGetConfigOption("GDAL_ONE_BIG_READ", nullptr);
967 if ( pszGDAL_ONE_BIG_READ == nullptr )
968 {
969 if ( nLineSize < 50000
970 || nXSize > nLineSize / nPixelOffset / 5 * 2
971 || IsSignificantNumberOfLinesLoaded(nYOff, nYSize) )
972 {
973 return FALSE;
974 }
975 return TRUE;
976 }
977
978 return CPLTestBool(pszGDAL_ONE_BIG_READ);
979 }
980
981 /************************************************************************/
982 /* IRasterIO() */
983 /************************************************************************/
984
IRasterIO(GDALRWFlag eRWFlag,int nXOff,int nYOff,int nXSize,int nYSize,void * pData,int nBufXSize,int nBufYSize,GDALDataType eBufType,GSpacing nPixelSpace,GSpacing nLineSpace,GDALRasterIOExtraArg * psExtraArg)985 CPLErr RawRasterBand::IRasterIO( GDALRWFlag eRWFlag,
986 int nXOff, int nYOff, int nXSize, int nYSize,
987 void * pData, int nBufXSize, int nBufYSize,
988 GDALDataType eBufType,
989 GSpacing nPixelSpace, GSpacing nLineSpace,
990 GDALRasterIOExtraArg* psExtraArg )
991
992 {
993 const int nBandDataSize = GDALGetDataTypeSizeBytes(eDataType);
994 #ifdef DEBUG
995 // Otherwise Coverity thinks that a divide by zero is possible in
996 // AccessBlock() in the complex data type wapping case.
997 if( nBandDataSize == 0 )
998 return CE_Failure;
999 #endif
1000 const int nBufDataSize = GDALGetDataTypeSizeBytes(eBufType);
1001
1002 if( !CanUseDirectIO(nXOff, nYOff, nXSize, nYSize, eBufType, psExtraArg) )
1003 {
1004 return GDALRasterBand::IRasterIO(eRWFlag, nXOff, nYOff,
1005 nXSize, nYSize,
1006 pData, nBufXSize, nBufYSize,
1007 eBufType,
1008 nPixelSpace, nLineSpace, psExtraArg);
1009 }
1010
1011 CPLDebug("RAW", "Using direct IO implementation");
1012
1013 if (pLineBuffer == nullptr)
1014 {
1015 if( poDS != nullptr && poDS->GetRasterCount() > 1 && IsBIP() )
1016 {
1017 auto poFirstBand = (nBand == 1) ? this :
1018 cpl::down_cast<RawRasterBand *>(poDS->GetRasterBand(1));
1019 CPLAssert(poFirstBand);
1020 if( poFirstBand->bNeedFileFlush )
1021 RawRasterBand::FlushCache();
1022 }
1023 }
1024 if( bNeedFileFlush )
1025 RawRasterBand::FlushCache();
1026
1027 // Read data.
1028 if ( eRWFlag == GF_Read )
1029 {
1030 // Do we have overviews that are appropriate to satisfy this request?
1031 if( (nBufXSize < nXSize || nBufYSize < nYSize)
1032 && GetOverviewCount() > 0 )
1033 {
1034 if( OverviewRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
1035 pData, nBufXSize, nBufYSize,
1036 eBufType, nPixelSpace, nLineSpace,
1037 psExtraArg) == CE_None)
1038 return CE_None;
1039 }
1040
1041 // 1. Simplest case when we should get contiguous block
1042 // of uninterleaved pixels.
1043 if ( nXSize == GetXSize()
1044 && nXSize == nBufXSize
1045 && nYSize == nBufYSize
1046 && eBufType == eDataType
1047 && nPixelOffset == nBandDataSize
1048 && nPixelSpace == nBufDataSize
1049 && nLineSpace == nPixelSpace * nXSize )
1050 {
1051 vsi_l_offset nOffset = nImgOffset;
1052 if( nLineOffset >= 0 )
1053 nOffset += nYOff * nLineOffset;
1054 else
1055 nOffset -= nYOff * static_cast<vsi_l_offset>(-nLineOffset);
1056
1057 const size_t nBytesToRead =
1058 static_cast<size_t>(nXSize) * nYSize * nBandDataSize;
1059 if ( AccessBlock(nOffset, nBytesToRead, pData) != CE_None )
1060 {
1061 CPLError(CE_Failure, CPLE_FileIO,
1062 "Failed to read " CPL_FRMT_GUIB
1063 " bytes at " CPL_FRMT_GUIB ".",
1064 static_cast<GUIntBig>(nBytesToRead), nOffset);
1065 return CE_Failure;
1066 }
1067 }
1068 // 2. Case when we need deinterleave and/or subsample data.
1069 else
1070 {
1071 const double dfSrcXInc = static_cast<double>(nXSize) / nBufXSize;
1072 const double dfSrcYInc = static_cast<double>(nYSize) / nBufYSize;
1073
1074 const size_t nBytesToRW =
1075 static_cast<size_t>(nPixelOffset) * nXSize;
1076 GByte *pabyData =
1077 static_cast<GByte *>(VSI_MALLOC_VERBOSE(nBytesToRW));
1078 if( pabyData == nullptr )
1079 return CE_Failure;
1080
1081 for ( int iLine = 0; iLine < nBufYSize; iLine++ )
1082 {
1083 const vsi_l_offset nLine =
1084 static_cast<vsi_l_offset>(nYOff) +
1085 static_cast<vsi_l_offset>(iLine * dfSrcYInc);
1086 vsi_l_offset nOffset = nImgOffset;
1087 if( nLineOffset >= 0 )
1088 nOffset += nLine * nLineOffset;
1089 else
1090 nOffset -= nLine * static_cast<vsi_l_offset>(-nLineOffset);
1091 if( nPixelOffset >= 0 )
1092 nOffset += nXOff * nPixelOffset;
1093 else
1094 nOffset -= nXOff * static_cast<vsi_l_offset>(-nPixelOffset);
1095 if ( AccessBlock(nOffset,
1096 nBytesToRW, pabyData) != CE_None )
1097 {
1098 CPLError(CE_Failure, CPLE_FileIO,
1099 "Failed to read " CPL_FRMT_GUIB
1100 " bytes at " CPL_FRMT_GUIB ".",
1101 static_cast<GUIntBig>(nBytesToRW), nOffset);
1102 CPLFree(pabyData);
1103 return CE_Failure;
1104 }
1105 // Copy data from disk buffer to user block buffer and
1106 // subsample, if needed.
1107 if ( nXSize == nBufXSize && nYSize == nBufYSize )
1108 {
1109 GDALCopyWords(
1110 pabyData, eDataType, nPixelOffset,
1111 static_cast<GByte *>(pData) +
1112 static_cast<vsi_l_offset>(iLine) * nLineSpace,
1113 eBufType, static_cast<int>(nPixelSpace), nXSize);
1114 }
1115 else
1116 {
1117 for ( int iPixel = 0; iPixel < nBufXSize; iPixel++ )
1118 {
1119 GDALCopyWords(
1120 pabyData +
1121 static_cast<vsi_l_offset>(iPixel * dfSrcXInc) *
1122 nPixelOffset,
1123 eDataType, nPixelOffset,
1124 static_cast<GByte *>(pData) +
1125 static_cast<vsi_l_offset>(iLine) * nLineSpace +
1126 static_cast<vsi_l_offset>(iPixel) * nPixelSpace,
1127 eBufType, static_cast<int>(nPixelSpace), 1);
1128 }
1129 }
1130
1131 if( psExtraArg->pfnProgress != nullptr &&
1132 !psExtraArg->pfnProgress(1.0 * (iLine + 1) / nBufYSize, "",
1133 psExtraArg->pProgressData) )
1134 {
1135 CPLFree(pabyData);
1136 return CE_Failure;
1137 }
1138 }
1139
1140 CPLFree(pabyData);
1141 }
1142 }
1143 // Write data.
1144 else
1145 {
1146 // 1. Simplest case when we should write contiguous block of
1147 // uninterleaved pixels.
1148 if ( nXSize == GetXSize()
1149 && nXSize == nBufXSize
1150 && nYSize == nBufYSize
1151 && eBufType == eDataType
1152 && nPixelOffset == nBandDataSize
1153 && nPixelSpace == nBufDataSize
1154 && nLineSpace == nPixelSpace * nXSize )
1155 {
1156 // Byte swap the data buffer, if required.
1157 if( NeedsByteOrderChange() )
1158 {
1159 DoByteSwap(pData, nXSize, std::abs(nPixelOffset), false);
1160 }
1161
1162 // Seek to the correct block.
1163 vsi_l_offset nOffset = nImgOffset;
1164 if( nLineOffset >= 0 )
1165 nOffset += nYOff * nLineOffset;
1166 else
1167 nOffset -= nYOff * static_cast<vsi_l_offset>(-nLineOffset);
1168
1169 if( Seek(nOffset, SEEK_SET) == -1 )
1170 {
1171 CPLError(CE_Failure, CPLE_FileIO,
1172 "Failed to seek to " CPL_FRMT_GUIB " to write data.",
1173 nOffset);
1174
1175 return CE_Failure;
1176 }
1177
1178 // Write the block.
1179 const size_t nBytesToRW =
1180 static_cast<size_t>(nXSize) * nYSize * nBandDataSize;
1181
1182 const size_t nBytesActuallyWritten = Write(pData, 1, nBytesToRW);
1183 if( nBytesActuallyWritten < nBytesToRW )
1184 {
1185 CPLError(CE_Failure, CPLE_FileIO,
1186 "Failed to write " CPL_FRMT_GUIB
1187 " bytes to file. " CPL_FRMT_GUIB " bytes written",
1188 static_cast<GUIntBig>(nBytesToRW),
1189 static_cast<GUIntBig>(nBytesActuallyWritten));
1190
1191 return CE_Failure;
1192 }
1193
1194 // Byte swap (if necessary) back into machine order so the
1195 // buffer is still usable for reading purposes.
1196 if( NeedsByteOrderChange() )
1197 {
1198 DoByteSwap(pData, nXSize, std::abs(nPixelOffset), true);
1199 }
1200 }
1201 // 2. Case when we need deinterleave and/or subsample data.
1202 else
1203 {
1204 const double dfSrcXInc = static_cast<double>(nXSize) / nBufXSize;
1205 const double dfSrcYInc = static_cast<double>(nYSize) / nBufYSize;
1206
1207 const size_t nBytesToRW =
1208 static_cast<size_t>(nPixelOffset) * nXSize;
1209 GByte *pabyData =
1210 static_cast<GByte *>(VSI_MALLOC_VERBOSE(nBytesToRW));
1211 if( pabyData == nullptr )
1212 return CE_Failure;
1213
1214 for ( int iLine = 0; iLine < nBufYSize; iLine++ )
1215 {
1216 const vsi_l_offset nLine =
1217 static_cast<vsi_l_offset>(nYOff) +
1218 static_cast<vsi_l_offset>(iLine * dfSrcYInc);
1219 vsi_l_offset nOffset = nImgOffset;
1220 if( nLineOffset >= 0 )
1221 nOffset += nLine * nLineOffset;
1222 else
1223 nOffset -= nLine * static_cast<vsi_l_offset>(-nLineOffset);
1224 if( nPixelOffset >= 0 )
1225 nOffset += nXOff * nPixelOffset;
1226 else
1227 nOffset -= nXOff * static_cast<vsi_l_offset>(-nPixelOffset);
1228
1229 // If the data for this band is completely contiguous we don't
1230 // have to worry about pre-reading from disk.
1231 if( nPixelOffset > nBandDataSize )
1232 AccessBlock(nOffset, nBytesToRW, pabyData);
1233
1234 // Copy data from user block buffer to disk buffer and
1235 // subsample, if needed.
1236 if ( nXSize == nBufXSize && nYSize == nBufYSize )
1237 {
1238 GDALCopyWords(static_cast<GByte *>(pData) +
1239 static_cast<vsi_l_offset>(iLine) *
1240 nLineSpace,
1241 eBufType, static_cast<int>(nPixelSpace),
1242 pabyData, eDataType, nPixelOffset, nXSize);
1243 }
1244 else
1245 {
1246 for ( int iPixel = 0; iPixel < nBufXSize; iPixel++ )
1247 {
1248 GDALCopyWords(
1249 static_cast<GByte *>(pData) +
1250 static_cast<vsi_l_offset>(iLine) * nLineSpace +
1251 static_cast<vsi_l_offset>(iPixel) * nPixelSpace,
1252 eBufType, static_cast<int>(nPixelSpace),
1253 pabyData +
1254 static_cast<vsi_l_offset>(iPixel * dfSrcXInc) *
1255 nPixelOffset,
1256 eDataType, nPixelOffset, 1);
1257 }
1258 }
1259
1260 // Byte swap the data buffer, if required.
1261 if( NeedsByteOrderChange() )
1262 {
1263 if( GDALDataTypeIsComplex(eDataType) )
1264 {
1265 const int nWordSize =
1266 GDALGetDataTypeSize(eDataType) / 16;
1267 GDALSwapWords(pabyData, nWordSize, nXSize,
1268 nPixelOffset);
1269 GDALSwapWords(static_cast<GByte *>(pabyData) +
1270 nWordSize,
1271 nWordSize, nXSize, nPixelOffset);
1272 }
1273 else
1274 {
1275 GDALSwapWords(pabyData, nBandDataSize, nXSize,
1276 nPixelOffset);
1277 }
1278 }
1279
1280 // Seek to the right line in block.
1281 if( Seek(nOffset, SEEK_SET) == -1 )
1282 {
1283 CPLError(CE_Failure, CPLE_FileIO,
1284 "Failed to seek to " CPL_FRMT_GUIB " to read.",
1285 nOffset);
1286 CPLFree(pabyData);
1287 return CE_Failure;
1288 }
1289
1290 // Write the line of block.
1291 const size_t nBytesActuallyWritten =
1292 Write(pabyData, 1, nBytesToRW);
1293 if( nBytesActuallyWritten < nBytesToRW )
1294 {
1295 CPLError(CE_Failure, CPLE_FileIO,
1296 "Failed to write " CPL_FRMT_GUIB
1297 " bytes to file. " CPL_FRMT_GUIB " bytes written",
1298 static_cast<GUIntBig>(nBytesToRW),
1299 static_cast<GUIntBig>(nBytesActuallyWritten));
1300 CPLFree(pabyData);
1301 return CE_Failure;
1302 }
1303
1304
1305 // Byte swap (if necessary) back into machine order so the
1306 // buffer is still usable for reading purposes.
1307 if( NeedsByteOrderChange() )
1308 {
1309 if( GDALDataTypeIsComplex(eDataType) )
1310 {
1311 const int nWordSize =
1312 GDALGetDataTypeSize(eDataType) / 16;
1313 GDALSwapWords(pabyData, nWordSize, nXSize,
1314 nPixelOffset);
1315 GDALSwapWords(static_cast<GByte *>(pabyData) +
1316 nWordSize,
1317 nWordSize, nXSize, nPixelOffset);
1318 }
1319 else
1320 {
1321 GDALSwapWords(pabyData, nBandDataSize, nXSize,
1322 nPixelOffset);
1323 }
1324 }
1325 }
1326
1327 bNeedFileFlush = TRUE;
1328 CPLFree(pabyData);
1329 }
1330 }
1331
1332 return CE_None;
1333 }
1334
1335 /************************************************************************/
1336 /* Seek() */
1337 /************************************************************************/
1338
Seek(vsi_l_offset nOffset,int nSeekMode)1339 int RawRasterBand::Seek( vsi_l_offset nOffset, int nSeekMode )
1340
1341 {
1342 return VSIFSeekL(fpRawL, nOffset, nSeekMode);
1343 }
1344
1345 /************************************************************************/
1346 /* Read() */
1347 /************************************************************************/
1348
Read(void * pBuffer,size_t nSize,size_t nCount)1349 size_t RawRasterBand::Read( void *pBuffer, size_t nSize, size_t nCount )
1350
1351 {
1352 return VSIFReadL(pBuffer, nSize, nCount, fpRawL);
1353 }
1354
1355 /************************************************************************/
1356 /* Write() */
1357 /************************************************************************/
1358
Write(void * pBuffer,size_t nSize,size_t nCount)1359 size_t RawRasterBand::Write( void *pBuffer, size_t nSize, size_t nCount )
1360
1361 {
1362 return VSIFWriteL(pBuffer, nSize, nCount, fpRawL);
1363 }
1364
1365 /************************************************************************/
1366 /* StoreNoDataValue() */
1367 /* */
1368 /* This is a helper function for datasets to associate a no */
1369 /* data value with this band, it isn't intended to be called by */
1370 /* applications. */
1371 /************************************************************************/
1372
StoreNoDataValue(double dfValue)1373 void RawRasterBand::StoreNoDataValue( double dfValue )
1374
1375 {
1376 SetNoDataValue(dfValue);
1377 }
1378
1379 /************************************************************************/
1380 /* GetCategoryNames() */
1381 /************************************************************************/
1382
GetCategoryNames()1383 char **RawRasterBand::GetCategoryNames() { return papszCategoryNames; }
1384
1385 /************************************************************************/
1386 /* SetCategoryNames() */
1387 /************************************************************************/
1388
SetCategoryNames(char ** papszNewNames)1389 CPLErr RawRasterBand::SetCategoryNames( char **papszNewNames )
1390
1391 {
1392 CSLDestroy(papszCategoryNames);
1393 papszCategoryNames = CSLDuplicate(papszNewNames);
1394
1395 return CE_None;
1396 }
1397
1398 /************************************************************************/
1399 /* SetColorTable() */
1400 /************************************************************************/
1401
SetColorTable(GDALColorTable * poNewCT)1402 CPLErr RawRasterBand::SetColorTable( GDALColorTable *poNewCT )
1403
1404 {
1405 if( poCT )
1406 delete poCT;
1407 if( poNewCT == nullptr )
1408 poCT = nullptr;
1409 else
1410 poCT = poNewCT->Clone();
1411
1412 return CE_None;
1413 }
1414
1415 /************************************************************************/
1416 /* GetColorTable() */
1417 /************************************************************************/
1418
GetColorTable()1419 GDALColorTable *RawRasterBand::GetColorTable() { return poCT; }
1420
1421 /************************************************************************/
1422 /* SetColorInterpretation() */
1423 /************************************************************************/
1424
SetColorInterpretation(GDALColorInterp eNewInterp)1425 CPLErr RawRasterBand::SetColorInterpretation( GDALColorInterp eNewInterp )
1426
1427 {
1428 eInterp = eNewInterp;
1429
1430 return CE_None;
1431 }
1432
1433 /************************************************************************/
1434 /* GetColorInterpretation() */
1435 /************************************************************************/
1436
GetColorInterpretation()1437 GDALColorInterp RawRasterBand::GetColorInterpretation() { return eInterp; }
1438
1439 /************************************************************************/
1440 /* GetVirtualMemAuto() */
1441 /************************************************************************/
1442
GetVirtualMemAuto(GDALRWFlag eRWFlag,int * pnPixelSpace,GIntBig * pnLineSpace,char ** papszOptions)1443 CPLVirtualMem *RawRasterBand::GetVirtualMemAuto( GDALRWFlag eRWFlag,
1444 int *pnPixelSpace,
1445 GIntBig *pnLineSpace,
1446 char **papszOptions )
1447 {
1448 CPLAssert(pnPixelSpace);
1449 CPLAssert(pnLineSpace);
1450
1451 const vsi_l_offset nSize =
1452 static_cast<vsi_l_offset>(nRasterYSize - 1) * nLineOffset +
1453 (nRasterXSize - 1) * nPixelOffset + GDALGetDataTypeSizeBytes(eDataType);
1454
1455 const char *pszImpl = CSLFetchNameValueDef(
1456 papszOptions, "USE_DEFAULT_IMPLEMENTATION", "AUTO");
1457 if( VSIFGetNativeFileDescriptorL(fpRawL) == nullptr ||
1458 !CPLIsVirtualMemFileMapAvailable() ||
1459 NeedsByteOrderChange() ||
1460 static_cast<size_t>(nSize) != nSize ||
1461 nPixelOffset < 0 ||
1462 nLineOffset < 0 ||
1463 EQUAL(pszImpl, "YES") || EQUAL(pszImpl, "ON") ||
1464 EQUAL(pszImpl, "1") || EQUAL(pszImpl, "TRUE") )
1465 {
1466 return GDALRasterBand::GetVirtualMemAuto(eRWFlag, pnPixelSpace,
1467 pnLineSpace, papszOptions);
1468 }
1469
1470 FlushCache();
1471
1472 CPLVirtualMem *pVMem = CPLVirtualMemFileMapNew(
1473 fpRawL, nImgOffset, nSize,
1474 (eRWFlag == GF_Write) ? VIRTUALMEM_READWRITE : VIRTUALMEM_READONLY,
1475 nullptr, nullptr);
1476 if( pVMem == nullptr )
1477 {
1478 if( EQUAL(pszImpl, "NO") || EQUAL(pszImpl, "OFF") ||
1479 EQUAL(pszImpl, "0") || EQUAL(pszImpl, "FALSE") )
1480 {
1481 return nullptr;
1482 }
1483 return GDALRasterBand::GetVirtualMemAuto(eRWFlag, pnPixelSpace,
1484 pnLineSpace, papszOptions);
1485 }
1486
1487 *pnPixelSpace = nPixelOffset;
1488 *pnLineSpace = nLineOffset;
1489 return pVMem;
1490 }
1491
1492 /************************************************************************/
1493 /* ==================================================================== */
1494 /* RawDataset */
1495 /* ==================================================================== */
1496 /************************************************************************/
1497
1498 /************************************************************************/
1499 /* RawDataset() */
1500 /************************************************************************/
1501
RawDataset()1502 RawDataset::RawDataset() {}
1503
1504 /************************************************************************/
1505 /* ~RawDataset() */
1506 /************************************************************************/
1507
1508 // It's pure virtual function but must be defined, even if empty.
~RawDataset()1509 RawDataset::~RawDataset() {}
1510
1511 /************************************************************************/
1512 /* IRasterIO() */
1513 /* */
1514 /* Multi-band raster io handler. */
1515 /************************************************************************/
1516
IRasterIO(GDALRWFlag eRWFlag,int nXOff,int nYOff,int nXSize,int nYSize,void * pData,int nBufXSize,int nBufYSize,GDALDataType eBufType,int nBandCount,int * panBandMap,GSpacing nPixelSpace,GSpacing nLineSpace,GSpacing nBandSpace,GDALRasterIOExtraArg * psExtraArg)1517 CPLErr RawDataset::IRasterIO( GDALRWFlag eRWFlag,
1518 int nXOff, int nYOff, int nXSize, int nYSize,
1519 void *pData, int nBufXSize, int nBufYSize,
1520 GDALDataType eBufType,
1521 int nBandCount, int *panBandMap,
1522 GSpacing nPixelSpace, GSpacing nLineSpace,
1523 GSpacing nBandSpace,
1524 GDALRasterIOExtraArg* psExtraArg )
1525
1526 {
1527 const char* pszInterleave = nullptr;
1528
1529 // The default GDALDataset::IRasterIO() implementation would go to
1530 // BlockBasedRasterIO if the dataset is interleaved. However if the
1531 // access pattern is compatible with DirectIO() we don't want to go
1532 // BlockBasedRasterIO, but rather used our optimized path in
1533 // RawRasterBand::IRasterIO().
1534 if (nXSize == nBufXSize && nYSize == nBufYSize && nBandCount > 1 &&
1535 (pszInterleave = GetMetadataItem("INTERLEAVE",
1536 "IMAGE_STRUCTURE")) != nullptr &&
1537 EQUAL(pszInterleave, "PIXEL"))
1538 {
1539 int iBandIndex = 0;
1540 for( ; iBandIndex < nBandCount; iBandIndex++ )
1541 {
1542 RawRasterBand *poBand = dynamic_cast<RawRasterBand *>(
1543 GetRasterBand(panBandMap[iBandIndex]));
1544 if( poBand == nullptr ||
1545 !poBand->CanUseDirectIO(nXOff, nYOff,
1546 nXSize, nYSize, eBufType, psExtraArg) )
1547 {
1548 break;
1549 }
1550 }
1551 if( iBandIndex == nBandCount )
1552 {
1553 GDALProgressFunc pfnProgressGlobal = psExtraArg->pfnProgress;
1554 void *pProgressDataGlobal = psExtraArg->pProgressData;
1555
1556 CPLErr eErr = CE_None;
1557 for( iBandIndex = 0;
1558 iBandIndex < nBandCount && eErr == CE_None;
1559 iBandIndex++ )
1560 {
1561 GDALRasterBand *poBand = GetRasterBand(panBandMap[iBandIndex]);
1562
1563 if (poBand == nullptr)
1564 {
1565 eErr = CE_Failure;
1566 break;
1567 }
1568
1569 GByte *pabyBandData =
1570 static_cast<GByte *>(pData) + iBandIndex * nBandSpace;
1571
1572 psExtraArg->pfnProgress = GDALScaledProgress;
1573 psExtraArg->pProgressData = GDALCreateScaledProgress(
1574 1.0 * iBandIndex / nBandCount,
1575 1.0 * (iBandIndex + 1) / nBandCount, pfnProgressGlobal,
1576 pProgressDataGlobal);
1577
1578 eErr = poBand->RasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
1579 static_cast<void *>(pabyBandData),
1580 nBufXSize, nBufYSize, eBufType,
1581 nPixelSpace, nLineSpace, psExtraArg);
1582
1583 GDALDestroyScaledProgress(psExtraArg->pProgressData);
1584 }
1585
1586 psExtraArg->pfnProgress = pfnProgressGlobal;
1587 psExtraArg->pProgressData = pProgressDataGlobal;
1588
1589 return eErr;
1590 }
1591 }
1592
1593 return GDALDataset::IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, pData,
1594 nBufXSize, nBufYSize, eBufType, nBandCount,
1595 panBandMap, nPixelSpace, nLineSpace,
1596 nBandSpace, psExtraArg);
1597 }
1598
1599
1600 /************************************************************************/
1601 /* RAWDatasetCheckMemoryUsage() */
1602 /************************************************************************/
1603
RAWDatasetCheckMemoryUsage(int nXSize,int nYSize,int nBands,int nDTSize,int nPixelOffset,int nLineOffset,vsi_l_offset nHeaderSize,vsi_l_offset nBandOffset,VSILFILE * fp)1604 bool RAWDatasetCheckMemoryUsage(int nXSize, int nYSize, int nBands,
1605 int nDTSize,
1606 int nPixelOffset,
1607 int nLineOffset,
1608 vsi_l_offset nHeaderSize,
1609 vsi_l_offset nBandOffset,
1610 VSILFILE* fp)
1611 {
1612 const GIntBig nTotalBufferSize =
1613 nPixelOffset == static_cast<GIntBig>(nDTSize) * nBands ? // BIP ?
1614 static_cast<GIntBig>(nPixelOffset) * nXSize :
1615 static_cast<GIntBig>(std::abs(nPixelOffset)) * nXSize * nBands;
1616
1617 // Currently each RawRasterBand allocates nPixelOffset * nRasterXSize bytes
1618 // so for a pixel interleaved scheme, this will allocate lots of memory!
1619 // Actually this is quadratic in the number of bands!
1620 // Do a few sanity checks to avoid excessive memory allocation on
1621 // small files.
1622 // But ultimately we should fix RawRasterBand to have a shared buffer
1623 // among bands.
1624 const char* pszCheck = CPLGetConfigOption("RAW_CHECK_FILE_SIZE", nullptr);
1625 if( (nBands > 10 || nTotalBufferSize > 20000 ||
1626 (pszCheck && CPLTestBool(pszCheck))) &&
1627 !(pszCheck && !CPLTestBool(pszCheck)) )
1628 {
1629 vsi_l_offset nExpectedFileSize;
1630 try
1631 {
1632 nExpectedFileSize =
1633 (CPLSM(static_cast<GUInt64>(nHeaderSize)) +
1634 CPLSM(static_cast<GUInt64>(nBandOffset)) * CPLSM(static_cast<GUInt64>(nBands - 1)) +
1635 (nLineOffset >= 0 ? CPLSM(static_cast<GUInt64>(nYSize-1)) * CPLSM(static_cast<GUInt64>(nLineOffset)) : CPLSM(static_cast<GUInt64>(0))) +
1636 (nPixelOffset >= 0 ? CPLSM(static_cast<GUInt64>(nXSize-1)) * CPLSM(static_cast<GUInt64>(nPixelOffset)) : CPLSM(static_cast<GUInt64>(0)))).v();
1637 }
1638 catch( ... )
1639 {
1640 CPLError(CE_Failure, CPLE_AppDefined, "Image file is too small");
1641 return false;
1642 }
1643 CPL_IGNORE_RET_VAL( VSIFSeekL(fp, 0, SEEK_END) );
1644 vsi_l_offset nFileSize = VSIFTellL(fp);
1645 // Do not strictly compare against nExpectedFileSize, but use an arbitrary
1646 // 50% margin, since some raw formats such as ENVI
1647 // allow for sparse files (see https://github.com/OSGeo/gdal/issues/915)
1648 if( nFileSize < nExpectedFileSize / 2 )
1649 {
1650 CPLError(CE_Failure, CPLE_AppDefined, "Image file is too small");
1651 return false;
1652 }
1653 }
1654
1655 #if SIZEOF_VOIDP == 8
1656 const char* pszDefault = "1024";
1657 #else
1658 const char* pszDefault = "512";
1659 #endif
1660 constexpr int MB_IN_BYTES = 1024 * 1024;
1661 const GIntBig nMAX_BUFFER_MEM =
1662 static_cast<GIntBig>(atoi(CPLGetConfigOption("RAW_MEM_ALLOC_LIMIT_MB", pszDefault))) * MB_IN_BYTES;
1663 if( nTotalBufferSize > nMAX_BUFFER_MEM )
1664 {
1665 CPLError(CE_Failure, CPLE_OutOfMemory,
1666 CPL_FRMT_GIB " MB of RAM would be needed to open the dataset. If you are "
1667 "comfortable with this, you can set the RAW_MEM_ALLOC_LIMIT_MB "
1668 "configuration option to that value or above",
1669 (nTotalBufferSize + MB_IN_BYTES - 1) / MB_IN_BYTES);
1670 return false;
1671 }
1672
1673 return true;
1674 }
1675
1676 /************************************************************************/
1677 /* GetRawBinaryLayout() */
1678 /************************************************************************/
1679
GetRawBinaryLayout(GDALDataset::RawBinaryLayout & sLayout)1680 bool RawDataset::GetRawBinaryLayout(GDALDataset::RawBinaryLayout& sLayout)
1681 {
1682 vsi_l_offset nImgOffset = 0;
1683 GIntBig nBandOffset = 0;
1684 int nPixelOffset = 0;
1685 int nLineOffset = 0;
1686 RawRasterBand::ByteOrder eByteOrder = RawRasterBand::ByteOrder::ORDER_LITTLE_ENDIAN;
1687 GDALDataType eDT = GDT_Unknown;
1688 for( int i = 1; i <= nBands; i++ )
1689 {
1690 auto poBand = dynamic_cast<RawRasterBand*>(GetRasterBand(i));
1691 if( poBand == nullptr )
1692 return false;
1693 if( i == 1 )
1694 {
1695 nImgOffset = poBand->nImgOffset;
1696 nPixelOffset = poBand->nPixelOffset;
1697 nLineOffset = poBand->nLineOffset;
1698 eByteOrder = poBand->eByteOrder;
1699 if( eByteOrder == RawRasterBand::ByteOrder::ORDER_VAX )
1700 return false;
1701 eDT = poBand->GetRasterDataType();
1702 }
1703 else if( nPixelOffset != poBand->nPixelOffset ||
1704 nLineOffset != poBand->nLineOffset ||
1705 eByteOrder != poBand->eByteOrder ||
1706 eDT != poBand->GetRasterDataType() )
1707 {
1708 return false;
1709 }
1710 else if( i == 2 )
1711 {
1712 nBandOffset = static_cast<GIntBig>(poBand->nImgOffset) -
1713 static_cast<GIntBig>(nImgOffset);
1714 }
1715 else if( nBandOffset * (i - 1) !=
1716 static_cast<GIntBig>(poBand->nImgOffset) -
1717 static_cast<GIntBig>(nImgOffset) )
1718 {
1719 return false;
1720 }
1721 }
1722
1723 sLayout.eInterleaving = RawBinaryLayout::Interleaving::UNKNOWN;
1724 const int nDTSize = GDALGetDataTypeSizeBytes(eDT);
1725 if( nBands > 1 )
1726 {
1727 if( nPixelOffset == nBands * nDTSize &&
1728 nLineOffset == nPixelOffset * nRasterXSize &&
1729 nBandOffset == nDTSize )
1730 {
1731 sLayout.eInterleaving = RawBinaryLayout::Interleaving::BIP;
1732 }
1733 else if( nPixelOffset == nDTSize &&
1734 nLineOffset == nDTSize * nBands * nRasterXSize &&
1735 nBandOffset == static_cast<GIntBig>(nDTSize) * nRasterXSize )
1736 {
1737 sLayout.eInterleaving = RawBinaryLayout::Interleaving::BIL;
1738 }
1739 else if( nPixelOffset == nDTSize &&
1740 nLineOffset == nDTSize * nRasterXSize &&
1741 nBandOffset == static_cast<GIntBig>(nLineOffset) * nRasterYSize )
1742 {
1743 sLayout.eInterleaving = RawBinaryLayout::Interleaving::BSQ;
1744 }
1745 }
1746
1747 sLayout.eDataType = eDT;
1748 sLayout.bLittleEndianOrder = eByteOrder == RawRasterBand::ByteOrder::ORDER_LITTLE_ENDIAN;
1749 sLayout.nImageOffset = nImgOffset;
1750 sLayout.nPixelOffset = nPixelOffset;
1751 sLayout.nLineOffset = nLineOffset;
1752 sLayout.nBandOffset = nBandOffset;
1753
1754 return true;
1755 }
1756