1# 2# 3# The Nim Compiler 4# (c) Copyright 2020 Andreas Rumpf 5# 6# See the file "copying.txt", included in this 7# distribution, for details about the copyright. 8# 9 10## Low level binary format used by the compiler to store and load various AST 11## and related data. 12## 13## NB: this is incredibly low level and if you're interested in how the 14## compiler works and less a storage format, you're probably looking for 15## the `ic` or `packed_ast` modules to understand the logical format. 16 17from typetraits import supportsCopyMem 18 19## Overview 20## ======== 21## `RodFile` represents a Rod File (versioned binary format), and the 22## associated data for common interactions such as IO and error tracking 23## (`RodFileError`). The file format broken up into sections (`RodSection`) 24## and preceeded by a header (see: `cookie`). The precise layout, section 25## ordering and data following the section are determined by the user. See 26## `ic.loadRodFile`. 27## 28## A basic but "wrong" example of the lifecycle: 29## --------------------------------------------- 30## 1. `create` or `open` - create a new one or open an existing 31## 2. `storeHeader` - header info 32## 3. `storePrim` or `storeSeq` - save your stuff 33## 4. `close` - and we're done 34## 35## Now read the bits below to understand what's missing. 36## 37## Issues with the Example 38## ``````````````````````` 39## Missing Sections: 40## This is a low level API, so headers and sections need to be stored and 41## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` & 42## `loadSection`, respectively. 43## 44## No Error Handling: 45## The API is centered around IO and prone to error, each operation checks or 46## sets the `RodFile.err` field. A user of this API needs to handle these 47## appropriately. 48## 49## API Notes 50## ========= 51## 52## Valid inputs for Rod files 53## -------------------------- 54## ASTs, hopes, dreams, and anything as long as it and any children it may have 55## support `copyMem`. This means anything that is not a pointer and that does not contain a pointer. At a glance these are: 56## * string 57## * objects & tuples (fields are recursed) 58## * sequences AKA `seq[T]` 59## 60## Note on error handling style 61## ---------------------------- 62## A flag based approach is used where operations no-op in case of a 63## preexisting error and set the flag if they encounter one. 64## 65## Misc 66## ---- 67## * 'Prim' is short for 'primitive', as in a non-sequence type 68 69type 70 RodSection* = enum 71 versionSection 72 configSection 73 stringsSection 74 checkSumsSection 75 depsSection 76 numbersSection 77 exportsSection 78 hiddenSection 79 reexportsSection 80 compilerProcsSection 81 trmacrosSection 82 convertersSection 83 methodsSection 84 pureEnumsSection 85 macroUsagesSection 86 toReplaySection 87 topLevelSection 88 bodiesSection 89 symsSection 90 typesSection 91 typeInstCacheSection 92 procInstCacheSection 93 attachedOpsSection 94 methodsPerTypeSection 95 enumToStringProcsSection 96 typeInfoSection # required by the backend 97 backendFlagsSection 98 aliveSymsSection # beware, this is stored in a `.alivesyms` file. 99 100 RodFileError* = enum 101 ok, tooBig, cannotOpen, ioFailure, wrongHeader, wrongSection, configMismatch, 102 includeFileChanged 103 104 RodFile* = object 105 f*: File 106 currentSection*: RodSection # for error checking 107 err*: RodFileError # little experiment to see if this works 108 # better than exceptions. 109 110const 111 RodVersion = 1 112 cookie = [byte(0), byte('R'), byte('O'), byte('D'), 113 byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)] 114 115proc setError(f: var RodFile; err: RodFileError) {.inline.} = 116 f.err = err 117 #raise newException(IOError, "IO error") 118 119proc storePrim*(f: var RodFile; s: string) = 120 ## Stores a string. 121 ## The len is prefixed to allow for later retreival. 122 if f.err != ok: return 123 if s.len >= high(int32): 124 setError f, tooBig 125 return 126 var lenPrefix = int32(s.len) 127 if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): 128 setError f, ioFailure 129 else: 130 if s.len != 0: 131 if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len: 132 setError f, ioFailure 133 134proc storePrim*[T](f: var RodFile; x: T) = 135 ## Stores a non-sequence/string `T`. 136 ## If `T` doesn't support `copyMem` and is an object or tuple then the fields 137 ## are written -- the user from context will need to know which `T` to load. 138 if f.err != ok: return 139 when supportsCopyMem(T): 140 if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): 141 setError f, ioFailure 142 elif T is tuple: 143 for y in fields(x): 144 storePrim(f, y) 145 elif T is object: 146 for y in fields(x): 147 when y is seq: 148 storeSeq(f, y) 149 else: 150 storePrim(f, y) 151 else: 152 {.error: "unsupported type for 'storePrim'".} 153 154proc storeSeq*[T](f: var RodFile; s: seq[T]) = 155 ## Stores a sequence of `T`s, with the len as a prefix for later retrieval. 156 if f.err != ok: return 157 if s.len >= high(int32): 158 setError f, tooBig 159 return 160 var lenPrefix = int32(s.len) 161 if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): 162 setError f, ioFailure 163 else: 164 for i in 0..<s.len: 165 storePrim(f, s[i]) 166 167proc loadPrim*(f: var RodFile; s: var string) = 168 ## Read a string, the length was stored as a prefix 169 if f.err != ok: return 170 var lenPrefix = int32(0) 171 if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): 172 setError f, ioFailure 173 else: 174 s = newString(lenPrefix) 175 if lenPrefix > 0: 176 if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len: 177 setError f, ioFailure 178 179proc loadPrim*[T](f: var RodFile; x: var T) = 180 ## Load a non-sequence/string `T`. 181 if f.err != ok: return 182 when supportsCopyMem(T): 183 if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): 184 setError f, ioFailure 185 elif T is tuple: 186 for y in fields(x): 187 loadPrim(f, y) 188 elif T is object: 189 for y in fields(x): 190 when y is seq: 191 loadSeq(f, y) 192 else: 193 loadPrim(f, y) 194 else: 195 {.error: "unsupported type for 'loadPrim'".} 196 197proc loadSeq*[T](f: var RodFile; s: var seq[T]) = 198 ## `T` must be compatible with `copyMem`, see `loadPrim` 199 if f.err != ok: return 200 var lenPrefix = int32(0) 201 if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): 202 setError f, ioFailure 203 else: 204 s = newSeq[T](lenPrefix) 205 for i in 0..<lenPrefix: 206 loadPrim(f, s[i]) 207 208proc storeHeader*(f: var RodFile) = 209 ## stores the header which is described by `cookie`. 210 if f.err != ok: return 211 if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len: 212 setError f, ioFailure 213 214proc loadHeader*(f: var RodFile) = 215 ## Loads the header which is described by `cookie`. 216 if f.err != ok: return 217 var thisCookie: array[cookie.len, byte] 218 if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len: 219 setError f, ioFailure 220 elif thisCookie != cookie: 221 setError f, wrongHeader 222 223proc storeSection*(f: var RodFile; s: RodSection) = 224 ## update `currentSection` and writes the bytes value of s. 225 if f.err != ok: return 226 assert f.currentSection < s 227 f.currentSection = s 228 storePrim(f, s) 229 230proc loadSection*(f: var RodFile; expected: RodSection) = 231 ## read the bytes value of s, sets and error if the section is incorrect. 232 if f.err != ok: return 233 var s: RodSection 234 loadPrim(f, s) 235 if expected != s and f.err == ok: 236 setError f, wrongSection 237 238proc create*(filename: string): RodFile = 239 ## create the file and open it for writing 240 if not open(result.f, filename, fmWrite): 241 setError result, cannotOpen 242 243proc close*(f: var RodFile) = close(f.f) 244 245proc open*(filename: string): RodFile = 246 ## open the file for reading 247 if not open(result.f, filename, fmRead): 248 setError result, cannotOpen 249