1/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield
2 *
3 * This library is open source and may be redistributed and/or modified under
4 * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
5 * (at your option) any later version.  The full license is in LICENSE file
6 * included with this distribution, and on the openscenegraph.org website.
7 *
8 * This library is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 * OpenSceneGraph Public License for more details.
12*/
13
14#ifndef OSGDB_DATABASEPAGER
15#define OSGDB_DATABASEPAGER 1
16
17#include <osg/NodeVisitor>
18#include <osg/Group>
19#include <osg/PagedLOD>
20#include <osg/Drawable>
21#include <osg/GraphicsThread>
22#include <osg/FrameStamp>
23#include <osg/ObserverNodePath>
24#include <osg/observer_ptr>
25
26#include <OpenThreads/Thread>
27#include <OpenThreads/Mutex>
28#include <OpenThreads/ScopedLock>
29#include <OpenThreads/Condition>
30
31#include <osgUtil/IncrementalCompileOperation>
32
33#include <osgDB/SharedStateManager>
34#include <osgDB/ReaderWriter>
35#include <osgDB/Options>
36
37
38#include <map>
39#include <list>
40#include <algorithm>
41#include <functional>
42
43namespace osgDB {
44
45
46
47/** Database paging class which manages the loading of files in a background thread,
48  * and synchronizing of loaded models with the main scene graph.*/
49class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandler
50{
51    public :
52
53        typedef OpenThreads::Thread::ThreadPriority ThreadPriority;
54
55        DatabasePager();
56
57        DatabasePager(const DatabasePager& rhs);
58
59        virtual const char* className() const { return "DatabasePager"; }
60
61        /** Create a shallow copy on the DatabasePager.*/
62        virtual DatabasePager* clone() const { return new DatabasePager(*this); }
63
64        /** get the prototype singleton used by DatabasePager::create().*/
65        static osg::ref_ptr<DatabasePager>& prototype();
66
67        /** create a DatabasePager by cloning DatabasePager::prototype().*/
68        static DatabasePager* create();
69
70
71
72        /** Add a request to load a node file to end the database request list.*/
73        virtual void requestNodeFile(const std::string& fileName, osg::NodePath& nodePath,
74                                     float priority, const osg::FrameStamp* framestamp,
75                                     osg::ref_ptr<osg::Referenced>& databaseRequest,
76                                     const osg::Referenced* options);
77
78        /** Set the priority of the database pager thread(s).*/
79        int setSchedulePriority(OpenThreads::Thread::ThreadPriority priority);
80
81        /** Cancel the database pager thread(s).*/
82        virtual int cancel();
83
84        virtual bool isRunning() const;
85
86        /** Clear all internally cached structures.*/
87        virtual void clear();
88
89        class OSGDB_EXPORT DatabaseThread : public osg::Referenced, public OpenThreads::Thread
90        {
91        public:
92
93            enum Mode
94            {
95                HANDLE_ALL_REQUESTS,
96                HANDLE_NON_HTTP,
97                HANDLE_ONLY_HTTP
98            };
99
100            DatabaseThread(DatabasePager* pager, Mode mode, const std::string& name);
101
102            DatabaseThread(const DatabaseThread& dt, DatabasePager* pager);
103
104            void setName(const std::string& name) { _name = name; }
105            const std::string& getName() const { return _name; }
106
107            void setDone(bool done) { _done.exchange(done?1:0); }
108            bool getDone() const { return _done!=0; }
109
110            void setActive(bool active) { _active = active; }
111            bool getActive() const { return _active; }
112
113            virtual int cancel();
114
115            virtual void run();
116
117        protected:
118
119            virtual ~DatabaseThread();
120
121            OpenThreads::Atomic _done;
122            volatile bool       _active;
123            DatabasePager*      _pager;
124            Mode                _mode;
125            std::string         _name;
126
127        };
128
129        virtual void setProcessorAffinity(const OpenThreads::Affinity& affinity);
130        OpenThreads::Affinity& getProcessorAffinity() { return _affinity; }
131        const OpenThreads::Affinity& getProcessorAffinity() const { return _affinity; }
132
133        void setUpThreads(unsigned int totalNumThreads=2, unsigned int numHttpThreads=1);
134
135        virtual unsigned int addDatabaseThread(DatabaseThread::Mode mode, const std::string& name);
136
137        DatabaseThread* getDatabaseThread(unsigned int i) { return _databaseThreads[i].get(); }
138
139        const DatabaseThread* getDatabaseThread(unsigned int i) const { return _databaseThreads[i].get(); }
140
141        unsigned int getNumDatabaseThreads() const { return static_cast<unsigned int>(_databaseThreads.size()); }
142
143        /** Set whether the database pager thread should be paused or not.*/
144        void setDatabasePagerThreadPause(bool pause);
145
146        /** Get whether the database pager thread should is paused or not.*/
147        bool getDatabasePagerThreadPause() const { return _databasePagerThreadPaused; }
148
149        /** Set whether new database request calls are accepted or ignored.*/
150        void setAcceptNewDatabaseRequests(bool acceptNewRequests) { _acceptNewRequests = acceptNewRequests; }
151
152        /** Get whether new database request calls are accepted or ignored.*/
153        bool getAcceptNewDatabaseRequests() const { return _acceptNewRequests; }
154
155        /** Get the number of frames that are currently active.*/
156        int getNumFramesActive() const { return _numFramesActive; }
157
158        /** Signal the database thread that the update, cull and draw has begun for a new frame.
159          * Note, this is called by the application so that the database pager can go to sleep while the CPU is busy on the main rendering threads. */
160        virtual void signalBeginFrame(const osg::FrameStamp* framestamp);
161
162        /** Signal the database thread that the update, cull and draw dispatch has completed.
163          * Note, this is called by the application so that the database pager can go to wake back up now the main rendering threads are iddle waiting for the next frame.*/
164        virtual void signalEndFrame();
165
166
167        /** Find all PagedLOD nodes in a subgraph and register them with
168          * the DatabasePager so it can keep track of expired nodes.
169          * note, should be only be called from the update thread. */
170        virtual void registerPagedLODs(osg::Node* subgraph, unsigned int frameNumber = 0);
171
172        /** Set the incremental compile operation.
173          * Used to manage the OpenGL object compilation and merging of subgraphs in a way that avoids overloading
174          * the rendering of frame with too many new objects in one frame. */
175        void setIncrementalCompileOperation(osgUtil::IncrementalCompileOperation* ico);
176
177        /** Get the incremental compile operation. */
178        osgUtil::IncrementalCompileOperation* getIncrementalCompileOperation() { return _incrementalCompileOperation.get(); }
179
180
181        /** Set whether the database pager should pre compile OpenGL objects before allowing
182          * them to be merged into the scene graph.
183          * Pre compilation helps reduce the chances of frame drops, but also slows the
184          * speed at which tiles are merged as they have to be compiled first.*/
185        void setDoPreCompile(bool flag) { _doPreCompile = flag; }
186
187        /** Get whether the database pager should pre compile OpenGL objects before allowing
188          * them to be merged into the scene graph.*/
189        bool getDoPreCompile() const { return _doPreCompile; }
190
191
192
193        /** Set the target maximum number of PagedLOD to maintain in memory.
194          * Note, if more than the target number are required for rendering of a frame then these active PagedLOD are excempt from being expiried.
195          * But once the number of active drops back below the target the inactive PagedLOD will be trimmed back to the target number.*/
196        void setTargetMaximumNumberOfPageLOD(unsigned int target) { _targetMaximumNumberOfPageLOD = target; }
197
198        /** Get the target maximum number of PagedLOD to maintain in memory.*/
199        unsigned int getTargetMaximumNumberOfPageLOD() const { return _targetMaximumNumberOfPageLOD; }
200
201
202        /** Set whether the removed subgraphs should be deleted in the database thread or not.*/
203        void setDeleteRemovedSubgraphsInDatabaseThread(bool flag) { _deleteRemovedSubgraphsInDatabaseThread = flag; }
204
205        /** Get whether the removed subgraphs should be deleted in the database thread or not.*/
206        bool getDeleteRemovedSubgraphsInDatabaseThread() const { return _deleteRemovedSubgraphsInDatabaseThread; }
207
208        enum DrawablePolicy
209        {
210            DO_NOT_MODIFY_DRAWABLE_SETTINGS,
211            USE_DISPLAY_LISTS,
212            USE_VERTEX_BUFFER_OBJECTS,
213            USE_VERTEX_ARRAYS
214        };
215
216        /** Set how loaded drawables should be handled w.r.t their display list/vertex buffer object/vertex array settings.*/
217        void setDrawablePolicy(DrawablePolicy policy) { _drawablePolicy = policy; }
218
219        /** Get how loaded drawables should be handled w.r.t their display list/vertex buffer object/vertex array settings.*/
220        DrawablePolicy getDrawablePolicy() const { return _drawablePolicy; }
221
222
223        /** Set whether newly loaded textures should have a PixelBufferObject assigned to them to aid download to the GPU.*/
224        void setApplyPBOToImages(bool assignPBOToImages) { _assignPBOToImages = assignPBOToImages; }
225
226        /** Get whether newly loaded textures should have a PixelBufferObject assigned to them.*/
227        bool getApplyPBOToImages() const { return _assignPBOToImages; }
228
229
230        /** Set whether newly loaded textures should have their UnrefImageDataAfterApply set to a specified value.*/
231        void setUnrefImageDataAfterApplyPolicy(bool changeAutoUnRef, bool valueAutoUnRef) { _changeAutoUnRef = changeAutoUnRef; _valueAutoUnRef = valueAutoUnRef; }
232
233        /** Get whether newly loaded textures should have their UnrefImageDataAfterApply set to a specified value.*/
234        void getUnrefImageDataAfterApplyPolicy(bool& changeAutoUnRef, bool& valueAutoUnRef) const { changeAutoUnRef = _changeAutoUnRef; valueAutoUnRef = _valueAutoUnRef; }
235
236
237        /** Set whether newly loaded textures should have their MaxAnisotopy set to a specified value.*/
238        void setMaxAnisotropyPolicy(bool changeAnisotropy, float valueAnisotropy) { _changeAnisotropy = changeAnisotropy; _valueAnisotropy = valueAnisotropy; }
239
240        /** Set whether newly loaded textures should have their MaxAnisotopy set to a specified value.*/
241        void getMaxAnisotropyPolicy(bool& changeAnisotropy, float& valueAnisotropy) const { changeAnisotropy = _changeAnisotropy; valueAnisotropy = _valueAnisotropy; }
242
243
244        /** Return true if there are pending updates to the scene graph that require a call to updateSceneGraph(double). */
245        bool requiresUpdateSceneGraph() const;
246
247        /** Merge the changes to the scene graph by calling calling removeExpiredSubgraphs then addLoadedDataToSceneGraph.
248          * Note, must only be called from single thread update phase. */
249        virtual void updateSceneGraph(const osg::FrameStamp& frameStamp);
250
251        /** Return true if there are GL objects that need to be compiled by a draw traversal. */
252        bool requiresRedraw() const;
253
254        /** Report how many items are in the _fileRequestList queue */
255        unsigned int getFileRequestListSize() const { return static_cast<unsigned int>(_fileRequestQueue->size() + _httpRequestQueue->size()); }
256
257        /** Report how many items are in the _dataToCompileList queue */
258        unsigned int getDataToCompileListSize() const { return static_cast<unsigned int>(_dataToCompileList->size()); }
259
260        /** Report how many items are in the _dataToMergeList queue */
261        unsigned int getDataToMergeListSize() const { return static_cast<unsigned int>(_dataToMergeList->size()); }
262
263        /** Report whether any requests are in the pager.*/
264        bool getRequestsInProgress() const;
265
266        /** Get the minimum time between the first request for a tile to be loaded and the time of its merge into the main scene graph.*/
267        double getMinimumTimeToMergeTile() const { return _minimumTimeToMergeTile; }
268
269        /** Get the maximum time between the first request for a tile to be loaded and the time of its merge into the main scene graph.*/
270        double getMaximumTimeToMergeTile() const { return _maximumTimeToMergeTile; }
271
272        /** Get the average time between the first request for a tile to be loaded and the time of its merge into the main scene graph.*/
273        double getAverageTimeToMergeTiles() const { return (_numTilesMerges > 0) ? _totalTimeToMergeTiles/static_cast<double>(_numTilesMerges) : 0; }
274
275        /** Reset the Stats variables.*/
276        void resetStats();
277
278        typedef std::set< osg::ref_ptr<osg::StateSet> >                 StateSetList;
279        typedef std::vector< osg::ref_ptr<osg::Drawable> >              DrawableList;
280
281        class ExpirePagedLODsVisitor;
282
283        typedef std::list<  osg::ref_ptr<osg::Object> > ObjectList;
284
285        struct PagedLODList : public osg::Referenced
286        {
287            virtual PagedLODList* clone() = 0;
288            virtual void clear() = 0;
289            virtual unsigned int size() = 0;
290            virtual void removeExpiredChildren(int numberChildrenToRemove, double expiryTime, unsigned int expiryFrame, ObjectList& childrenRemoved, bool visitActive) = 0;
291            virtual void removeNodes(osg::NodeList& nodesToRemove) = 0;
292            virtual void insertPagedLOD(const osg::observer_ptr<osg::PagedLOD>& plod) = 0;
293            virtual bool containsPagedLOD(const osg::observer_ptr<osg::PagedLOD>& plod) const = 0;
294        };
295
296        void setMarkerObject(osg::Object* mo) { _markerObject = mo; }
297        osg::Object* getMarkerObject() { return _markerObject.get(); }
298        const osg::Object* getMarkerObject() const { return _markerObject.get(); }
299
300    protected:
301
302        virtual ~DatabasePager();
303
304        friend class DatabaseThread;
305        friend struct DatabaseRequest;
306
307        struct RequestQueue;
308
309        struct OSGDB_EXPORT DatabaseRequest : public osg::Referenced
310        {
311            DatabaseRequest():
312                osg::Referenced(true),
313                _valid(false),
314                _frameNumberFirstRequest(0),
315                _timestampFirstRequest(0.0),
316                _priorityFirstRequest(0.f),
317                _frameNumberLastRequest(0),
318                _timestampLastRequest(0.0),
319                _priorityLastRequest(0.0f),
320                _numOfRequests(0),
321                _groupExpired(false)
322            {}
323
324            void invalidate();
325
326            bool valid() const { return _valid; }
327
328            bool isRequestCurrent (int frameNumber) const
329            {
330                return _valid && (frameNumber - _frameNumberLastRequest <= 1);
331            }
332
333            bool                                _valid;
334            std::string                         _fileName;
335            unsigned int                        _frameNumberFirstRequest;
336            double                              _timestampFirstRequest;
337            float                               _priorityFirstRequest;
338            unsigned int                        _frameNumberLastRequest;
339            double                              _timestampLastRequest;
340            float                               _priorityLastRequest;
341            unsigned int                        _numOfRequests;
342
343            osg::observer_ptr<osg::Node>        _terrain;
344            osg::observer_ptr<osg::Group>       _group;
345
346            osg::ref_ptr<osg::Node>             _loadedModel;
347            osg::ref_ptr<Options>               _loadOptions;
348            osg::ref_ptr<ObjectCache>           _objectCache;
349
350            osg::observer_ptr<osgUtil::IncrementalCompileOperation::CompileSet> _compileSet;
351            bool                                _groupExpired; // flag used only in update thread
352        };
353
354
355        struct OSGDB_EXPORT RequestQueue : public osg::Referenced
356        {
357        public:
358
359            RequestQueue(DatabasePager* pager);
360
361            void add(DatabaseRequest* databaseRequest);
362            void remove(DatabaseRequest* databaseRequest);
363
364            void addNoLock(DatabaseRequest* databaseRequest);
365
366            void takeFirst(osg::ref_ptr<DatabaseRequest>& databaseRequest);
367
368            /// prune all the old requests and then return true if requestList left empty
369            bool pruneOldRequestsAndCheckIfEmpty();
370
371            virtual void updateBlock() {}
372
373            void invalidate(DatabaseRequest* dr);
374
375            bool empty();
376
377            unsigned int size();
378
379            void clear();
380
381
382            typedef std::list< osg::ref_ptr<DatabaseRequest> > RequestList;
383            void swap(RequestList& requestList);
384
385            DatabasePager*              _pager;
386            RequestList                 _requestList;
387            OpenThreads::Mutex          _requestMutex;
388            unsigned int                _frameNumberLastPruned;
389
390        protected:
391            virtual ~RequestQueue();
392        };
393
394
395        typedef std::vector< osg::ref_ptr<DatabaseThread> > DatabaseThreadList;
396
397        struct OSGDB_EXPORT ReadQueue : public RequestQueue
398        {
399            ReadQueue(DatabasePager* pager, const std::string& name);
400
401            void block() { _block->block(); }
402
403            void release() { _block->release(); }
404
405            virtual void updateBlock();
406
407
408            osg::ref_ptr<osg::RefBlock> _block;
409
410            std::string                 _name;
411
412            OpenThreads::Mutex          _childrenToDeleteListMutex;
413            ObjectList                  _childrenToDeleteList;
414        };
415
416        // forward declare inner helper classes
417        class FindCompileableGLObjectsVisitor;
418        friend class FindCompileableGLObjectsVisitor;
419
420        struct DatabasePagerCompileCompletedCallback;
421        friend struct DatabasePagerCompileCompletedCallback;
422
423        class FindPagedLODsVisitor;
424        friend class FindPagedLODsVisitor;
425
426        struct SortFileRequestFunctor;
427        friend struct SortFileRequestFunctor;
428
429
430        OpenThreads::Mutex              _run_mutex;
431        OpenThreads::Mutex              _dr_mutex;
432        bool                            _startThreadCalled;
433
434        void compileCompleted(DatabaseRequest* databaseRequest);
435
436        /** Iterate through the active PagedLOD nodes children removing
437          * children which haven't been visited since specified expiryTime.
438          * note, should be only be called from the update thread. */
439        virtual void removeExpiredSubgraphs(const osg::FrameStamp &frameStamp);
440
441        /** Add the loaded data to the scene graph.*/
442        void addLoadedDataToSceneGraph(const osg::FrameStamp &frameStamp);
443
444
445        OpenThreads::Affinity           _affinity;
446
447        bool                            _done;
448        bool                            _acceptNewRequests;
449        bool                            _databasePagerThreadPaused;
450
451        DatabaseThreadList              _databaseThreads;
452
453        int                             _numFramesActive;
454        mutable OpenThreads::Mutex      _numFramesActiveMutex;
455        OpenThreads::Atomic             _frameNumber;
456
457        osg::ref_ptr<ReadQueue>         _fileRequestQueue;
458        osg::ref_ptr<ReadQueue>         _httpRequestQueue;
459        osg::ref_ptr<RequestQueue>      _dataToCompileList;
460        osg::ref_ptr<RequestQueue>      _dataToMergeList;
461
462        DrawablePolicy                  _drawablePolicy;
463
464        bool                            _assignPBOToImages;
465        bool                            _changeAutoUnRef;
466        bool                            _valueAutoUnRef;
467        bool                            _changeAnisotropy;
468        float                           _valueAnisotropy;
469
470        bool                            _deleteRemovedSubgraphsInDatabaseThread;
471
472
473        osg::ref_ptr<PagedLODList>      _activePagedLODList;
474
475        unsigned int                    _targetMaximumNumberOfPageLOD;
476
477        bool                            _doPreCompile;
478        osg::ref_ptr<osgUtil::IncrementalCompileOperation>  _incrementalCompileOperation;
479
480
481        double                          _minimumTimeToMergeTile;
482        double                          _maximumTimeToMergeTile;
483        double                          _totalTimeToMergeTiles;
484        unsigned int                    _numTilesMerges;
485
486        osg::ref_ptr<osg::Object>       _markerObject;
487};
488
489}
490
491#endif
492