1 /*
2 Open Asset Import Library (assimp)
3 ----------------------------------------------------------------------
4 
5 Copyright (c) 2006-2019, assimp team
6 
7 
8 All rights reserved.
9 
10 Redistribution and use of this software in source and binary forms,
11 with or without modification, are permitted provided that the
12 following conditions are met:
13 
14 * Redistributions of source code must retain the above
15   copyright notice, this list of conditions and the
16   following disclaimer.
17 
18 * Redistributions in binary form must reproduce the above
19   copyright notice, this list of conditions and the
20   following disclaimer in the documentation and/or other
21   materials provided with the distribution.
22 
23 * Neither the name of the assimp team, nor the names of its
24   contributors may be used to endorse or promote products
25   derived from this software without specific prior
26   written permission of the assimp team.
27 
28 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 
40 ----------------------------------------------------------------------
41 */
42 
43 /** @file  FBXTokenizer.h
44  *  @brief FBX lexer
45  */
46 #ifndef INCLUDED_AI_FBX_TOKENIZER_H
47 #define INCLUDED_AI_FBX_TOKENIZER_H
48 
49 #include "FBXCompileConfig.h"
50 #include <assimp/ai_assert.h>
51 #include <vector>
52 #include <string>
53 
54 namespace Assimp {
55 namespace FBX {
56 
57 /** Rough classification for text FBX tokens used for constructing the
58  *  basic scope hierarchy. */
59 enum TokenType
60 {
61     // {
62     TokenType_OPEN_BRACKET = 0,
63 
64     // }
65     TokenType_CLOSE_BRACKET,
66 
67     // '"blablubb"', '2', '*14' - very general token class,
68     // further processing happens at a later stage.
69     TokenType_DATA,
70 
71     //
72     TokenType_BINARY_DATA,
73 
74     // ,
75     TokenType_COMMA,
76 
77     // blubb:
78     TokenType_KEY
79 };
80 
81 
82 /** Represents a single token in a FBX file. Tokens are
83  *  classified by the #TokenType enumerated types.
84  *
85  *  Offers iterator protocol. Tokens are immutable. */
86 class Token
87 {
88 private:
89     static const unsigned int BINARY_MARKER = static_cast<unsigned int>(-1);
90 
91 public:
92     /** construct a textual token */
93     Token(const char* sbegin, const char* send, TokenType type, unsigned int line, unsigned int column);
94 
95     /** construct a binary token */
96     Token(const char* sbegin, const char* send, TokenType type, size_t offset);
97 
98     ~Token();
99 
100 public:
StringContents()101     std::string StringContents() const {
102         return std::string(begin(),end());
103     }
104 
IsBinary()105     bool IsBinary() const {
106         return column == BINARY_MARKER;
107     }
108 
begin()109     const char* begin() const {
110         return sbegin;
111     }
112 
end()113     const char* end() const {
114         return send;
115     }
116 
Type()117     TokenType Type() const {
118         return type;
119     }
120 
Offset()121     size_t Offset() const {
122         ai_assert(IsBinary());
123         return offset;
124     }
125 
Line()126     unsigned int Line() const {
127         ai_assert(!IsBinary());
128         return static_cast<unsigned int>(line);
129     }
130 
Column()131     unsigned int Column() const {
132         ai_assert(!IsBinary());
133         return column;
134     }
135 
136 private:
137 
138 #ifdef DEBUG
139     // full string copy for the sole purpose that it nicely appears
140     // in msvc's debugger window.
141     const std::string contents;
142 #endif
143 
144 
145     const char* const sbegin;
146     const char* const send;
147     const TokenType type;
148 
149     union {
150         size_t line;
151         size_t offset;
152     };
153     const unsigned int column;
154 };
155 
156 // XXX should use C++11's unique_ptr - but assimp's need to keep working with 03
157 typedef const Token* TokenPtr;
158 typedef std::vector< TokenPtr > TokenList;
159 
160 #define new_Token new Token
161 
162 
163 /** Main FBX tokenizer function. Transform input buffer into a list of preprocessed tokens.
164  *
165  *  Skips over comments and generates line and column numbers.
166  *
167  * @param output_tokens Receives a list of all tokens in the input data.
168  * @param input_buffer Textual input buffer to be processed, 0-terminated.
169  * @throw DeadlyImportError if something goes wrong */
170 void Tokenize(TokenList& output_tokens, const char* input);
171 
172 
173 /** Tokenizer function for binary FBX files.
174  *
175  *  Emits a token list suitable for direct parsing.
176  *
177  * @param output_tokens Receives a list of all tokens in the input data.
178  * @param input_buffer Binary input buffer to be processed.
179  * @param length Length of input buffer, in bytes. There is no 0-terminal.
180  * @throw DeadlyImportError if something goes wrong */
181 void TokenizeBinary(TokenList& output_tokens, const char* input, size_t length);
182 
183 
184 } // ! FBX
185 } // ! Assimp
186 
187 #endif // ! INCLUDED_AI_FBX_PARSER_H
188