1#
2#
3#           The Nim Compiler
4#        (c) Copyright 2020 Andreas Rumpf
5#
6#    See the file "copying.txt", included in this
7#    distribution, for details about the copyright.
8#
9
10## Low level binary format used by the compiler to store and load various AST
11## and related data.
12##
13## NB: this is incredibly low level and if you're interested in how the
14##     compiler works and less a storage format, you're probably looking for
15##     the `ic` or `packed_ast` modules to understand the logical format.
16
17from typetraits import supportsCopyMem
18
19## Overview
20## ========
21## `RodFile` represents a Rod File (versioned binary format), and the
22## associated data for common interactions such as IO and error tracking
23## (`RodFileError`). The file format broken up into sections (`RodSection`)
24## and preceeded by a header (see: `cookie`). The precise layout, section
25## ordering and data following the section are determined by the user. See
26## `ic.loadRodFile`.
27##
28## A basic but "wrong" example of the lifecycle:
29## ---------------------------------------------
30## 1. `create` or `open`        - create a new one or open an existing
31## 2. `storeHeader`             - header info
32## 3. `storePrim` or `storeSeq` - save your stuff
33## 4. `close`                   - and we're done
34##
35## Now read the bits below to understand what's missing.
36##
37## Issues with the Example
38## ```````````````````````
39## Missing Sections:
40## This is a low level API, so headers and sections need to be stored and
41## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` &
42## `loadSection`, respectively.
43##
44## No Error Handling:
45## The API is centered around IO and prone to error, each operation checks or
46## sets the `RodFile.err` field. A user of this API needs to handle these
47## appropriately.
48##
49## API Notes
50## =========
51##
52## Valid inputs for Rod files
53## --------------------------
54## ASTs, hopes, dreams, and anything as long as it and any children it may have
55## support `copyMem`. This means anything that is not a pointer and that does not contain a pointer. At a glance these are:
56## * string
57## * objects & tuples (fields are recursed)
58## * sequences AKA `seq[T]`
59##
60## Note on error handling style
61## ----------------------------
62## A flag based approach is used where operations no-op in case of a
63## preexisting error and set the flag if they encounter one.
64##
65## Misc
66## ----
67## * 'Prim' is short for 'primitive', as in a non-sequence type
68
69type
70  RodSection* = enum
71    versionSection
72    configSection
73    stringsSection
74    checkSumsSection
75    depsSection
76    numbersSection
77    exportsSection
78    hiddenSection
79    reexportsSection
80    compilerProcsSection
81    trmacrosSection
82    convertersSection
83    methodsSection
84    pureEnumsSection
85    macroUsagesSection
86    toReplaySection
87    topLevelSection
88    bodiesSection
89    symsSection
90    typesSection
91    typeInstCacheSection
92    procInstCacheSection
93    attachedOpsSection
94    methodsPerTypeSection
95    enumToStringProcsSection
96    typeInfoSection  # required by the backend
97    backendFlagsSection
98    aliveSymsSection # beware, this is stored in a `.alivesyms` file.
99
100  RodFileError* = enum
101    ok, tooBig, cannotOpen, ioFailure, wrongHeader, wrongSection, configMismatch,
102    includeFileChanged
103
104  RodFile* = object
105    f*: File
106    currentSection*: RodSection # for error checking
107    err*: RodFileError # little experiment to see if this works
108                       # better than exceptions.
109
110const
111  RodVersion = 1
112  cookie = [byte(0), byte('R'), byte('O'), byte('D'),
113            byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)]
114
115proc setError(f: var RodFile; err: RodFileError) {.inline.} =
116  f.err = err
117  #raise newException(IOError, "IO error")
118
119proc storePrim*(f: var RodFile; s: string) =
120  ## Stores a string.
121  ## The len is prefixed to allow for later retreival.
122  if f.err != ok: return
123  if s.len >= high(int32):
124    setError f, tooBig
125    return
126  var lenPrefix = int32(s.len)
127  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
128    setError f, ioFailure
129  else:
130    if s.len != 0:
131      if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
132        setError f, ioFailure
133
134proc storePrim*[T](f: var RodFile; x: T) =
135  ## Stores a non-sequence/string `T`.
136  ## If `T` doesn't support `copyMem` and is an object or tuple then the fields
137  ## are written -- the user from context will need to know which `T` to load.
138  if f.err != ok: return
139  when supportsCopyMem(T):
140    if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
141      setError f, ioFailure
142  elif T is tuple:
143    for y in fields(x):
144      storePrim(f, y)
145  elif T is object:
146    for y in fields(x):
147      when y is seq:
148        storeSeq(f, y)
149      else:
150        storePrim(f, y)
151  else:
152    {.error: "unsupported type for 'storePrim'".}
153
154proc storeSeq*[T](f: var RodFile; s: seq[T]) =
155  ## Stores a sequence of `T`s, with the len as a prefix for later retrieval.
156  if f.err != ok: return
157  if s.len >= high(int32):
158    setError f, tooBig
159    return
160  var lenPrefix = int32(s.len)
161  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
162    setError f, ioFailure
163  else:
164    for i in 0..<s.len:
165      storePrim(f, s[i])
166
167proc loadPrim*(f: var RodFile; s: var string) =
168  ## Read a string, the length was stored as a prefix
169  if f.err != ok: return
170  var lenPrefix = int32(0)
171  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
172    setError f, ioFailure
173  else:
174    s = newString(lenPrefix)
175    if lenPrefix > 0:
176      if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
177        setError f, ioFailure
178
179proc loadPrim*[T](f: var RodFile; x: var T) =
180  ## Load a non-sequence/string `T`.
181  if f.err != ok: return
182  when supportsCopyMem(T):
183    if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
184      setError f, ioFailure
185  elif T is tuple:
186    for y in fields(x):
187      loadPrim(f, y)
188  elif T is object:
189    for y in fields(x):
190      when y is seq:
191        loadSeq(f, y)
192      else:
193        loadPrim(f, y)
194  else:
195    {.error: "unsupported type for 'loadPrim'".}
196
197proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
198  ## `T` must be compatible with `copyMem`, see `loadPrim`
199  if f.err != ok: return
200  var lenPrefix = int32(0)
201  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
202    setError f, ioFailure
203  else:
204    s = newSeq[T](lenPrefix)
205    for i in 0..<lenPrefix:
206      loadPrim(f, s[i])
207
208proc storeHeader*(f: var RodFile) =
209  ## stores the header which is described by `cookie`.
210  if f.err != ok: return
211  if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len:
212    setError f, ioFailure
213
214proc loadHeader*(f: var RodFile) =
215  ## Loads the header which is described by `cookie`.
216  if f.err != ok: return
217  var thisCookie: array[cookie.len, byte]
218  if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len:
219    setError f, ioFailure
220  elif thisCookie != cookie:
221    setError f, wrongHeader
222
223proc storeSection*(f: var RodFile; s: RodSection) =
224  ## update `currentSection` and writes the bytes value of s.
225  if f.err != ok: return
226  assert f.currentSection < s
227  f.currentSection = s
228  storePrim(f, s)
229
230proc loadSection*(f: var RodFile; expected: RodSection) =
231  ## read the bytes value of s, sets and error if the section is incorrect.
232  if f.err != ok: return
233  var s: RodSection
234  loadPrim(f, s)
235  if expected != s and f.err == ok:
236    setError f, wrongSection
237
238proc create*(filename: string): RodFile =
239  ## create the file and open it for writing
240  if not open(result.f, filename, fmWrite):
241    setError result, cannotOpen
242
243proc close*(f: var RodFile) = close(f.f)
244
245proc open*(filename: string): RodFile =
246  ## open the file for reading
247  if not open(result.f, filename, fmRead):
248    setError result, cannotOpen
249