1IOWOW - The C11 persistent key/value database engine based on [skip list](https://en.wikipedia.org/wiki/Skip_list) 2================================================================================================================== 3 4[![Join ejdb2 telegram](https://img.shields.io/badge/join-ejdb2%20telegram-0088cc.svg)](https://t.me/ejdb2) 5[![license](https://img.shields.io/github/license/Softmotions/ejdb.svg)](https://github.com/Softmotions/iowow/blob/master/LICENSE) 6![Maintained](https://img.shields.io/maintenance/yes/2021.svg) 7 8Website http://iowow.io 9 10# Key components 11 12* [iwkv.h](https://github.com/Softmotions/iowow/blob/master/src/kv/iwkv.h) Persistent key/value database engine 13* [iwfsmfile.h](https://github.com/Softmotions/iowow/blob/master/src/fs/iwfsmfile.h) File blocks allocation manager like `malloc()` on files 14 15# IWKV 16 17## Features 18 19* Support of multiple key-value databases within a single file 20* Online database backups 21* Native support of integer keys 22* [Write Ahead Logging (WAL) support](http://iowow.io/wal) 23* Ultra-fast traversal of database records 24* Compound keys support 25* Good performance comparing its main competitors: `lmdb`, `leveldb`, `kyoto cabinet` 26* Tiny C11 library (200Kb) can be easily embedded into any software 27 28[![Presentation](https://iowow.io/articles/iowow-presentation-cover-small.png)](https://iowow.io/articles/intro/) 29 30 31## Used by 32 33* EJDB - Embeddable JSON database engine. http://ejdb.org 34 35## Limitations 36 37* Maximum iwkv storage file size: `512 GB (0x7fffffff80)` 38* Total size of a single key+value record must be not greater than `255Mb (0xfffffff)` 39* In-memory cache for every opened database takes `~130Kb`, cache can be disposed by `iwkv_db_cache_release()` 40 41# Supported platforms 42 43## Linux 44### Ubuntu/Debian 45#### PPA repository 46 47```sh 48sudo add-apt-repository ppa:adamansky/iwowow 49sudo apt-get update 50sudo apt-get install iowow 51``` 52 53#### Building debian packages 54 55```sh 56mkdir build && cd build 57cmake .. -DCMAKE_BUILD_TYPE=Release -DPACKAGE_DEB=ON 58make package 59``` 60 61### RPM based Linux distributions 62```sh 63mkdir build && cd build 64cmake .. -DCMAKE_BUILD_TYPE=Release -DPACKAGE_RPM=ON 65make package 66``` 67 68## FreeBSD 69 70Successfully tested on FreeBSD 10/11 71 72## OSX 73 74Successfully tested on OSX 10.12/10.13 75 76## Windows 77 78[Cross-compilation for windows](http://iowow.io/iw/win) 79 80 81## MIPS based systems (+big-endian) 82 83Successfully tested on Debian 9.4, MIPS 32, gcc 6.x compiler. 84 85# Examples 86 87[src/kv/examples](https://github.com/Softmotions/iowow/tree/master/src/kv/examples) 88 89## Store and retrieve records 90 91```c 92#include <iowow/iwkv.h> 93#include <string.h> 94#include <stdlib.h> 95 96int main() { 97 IWKV_OPTS opts = { 98 .path = "example1.db", 99 .oflags = IWKV_TRUNC // Cleanup database before open 100 }; 101 IWKV iwkv; 102 IWDB mydb; 103 iwrc rc = iwkv_open(&opts, &iwkv); 104 if (rc) { 105 iwlog_ecode_error3(rc); 106 return 1; 107 } 108 // Now open mydb 109 // - Database id: 1 110 rc = iwkv_db(iwkv, 1, 0, &mydb); 111 if (rc) { 112 iwlog_ecode_error2(rc, "Failed to open mydb"); 113 return 1; 114 } 115 // Work with db: put/get value 116 IWKV_val key, val; 117 key.data = "foo"; 118 key.size = strlen(key.data); 119 val.data = "bar"; 120 val.size = strlen(val.data); 121 122 fprintf(stdout, "put: %.*s => %.*s\n", 123 (int) key.size, (char *) key.data, 124 (int) val.size, (char *) val.data); 125 126 rc = iwkv_put(mydb, &key, &val, 0); 127 if (rc) { 128 iwlog_ecode_error3(rc); 129 return rc; 130 } 131 // Retrieve value associated with `foo` key 132 val.data = 0; 133 val.size = 0; 134 rc = iwkv_get(mydb, &key, &val); 135 if (rc) { 136 iwlog_ecode_error3(rc); 137 return rc; 138 } 139 140 fprintf(stdout, "get: %.*s => %.*s\n", 141 (int) key.size, (char *) key.data, 142 (int) val.size, (char *) val.data); 143 144 iwkv_val_dispose(&val); 145 iwkv_close(&iwkv); 146 return 0; 147} 148``` 149**Compile and run:** 150 151```sh 152gcc -std=gnu11 -Wall -pedantic -c -o example1.o example1.c 153gcc -o example1 example1.o -liowow 154 155./example1 156 put: foo => bar 157 get: foo => bar 158``` 159 160## Cursor iteration example 161 162```c 163/// 164/// Fills database with a set of football table records 165/// then traverse records according to club name in ascending and descending orders. 166/// 167 168#include "iwkv.h" 169#include <string.h> 170#include <stdlib.h> 171#include <stdint.h> 172 173static struct data_s { 174 const char *club; 175 uint8_t points; 176} _points[] = { 177 178 { "Aston Villa", 25 }, 179 { "Manchester City", 57 }, 180 { "Arsenal", 40 }, 181 { "Everton", 37 }, 182 { "West Ham United", 27 }, 183 { "Tottenham Hotspur", 41 }, 184 { "Wolverhampton Wanderers", 43 }, 185 { "Norwich City", 21 }, 186 { "Leicester City", 53 }, 187 { "Manchester United", 45 }, 188 { "Newcastle United", 35 }, 189 { "Brighton & Hove Albion", 29 }, 190 { "AFC Bournemouth", 27 }, 191 { "Crystal Palace", 39 }, 192 { "Sheffield United", 43 }, 193 { "Burnley", 39 }, 194 { "Southampton", 34 }, 195 { "Watford", 27 }, 196 { "Chelsea", 48 }, 197 { "Liverpool", 82 }, 198}; 199 200static iwrc run(void) { 201 IWKV_OPTS opts = { 202 .path = "cursor1.db", 203 .oflags = IWKV_TRUNC // Cleanup database before open 204 }; 205 IWKV iwkv; 206 IWDB db; 207 IWKV_cursor cur = 0; 208 iwrc rc = iwkv_open(&opts, &iwkv); 209 RCRET(rc); 210 211 rc = iwkv_db(iwkv, 1, 0, &db); 212 RCGO(rc, finish); 213 214 for (int i = 0; i < sizeof(_points) / sizeof(_points[0]); ++i) { 215 struct data_s *n = &_points[i]; 216 IWKV_val key = { .data = (void *) n->club, .size = strlen(n->club) }; 217 IWKV_val val = { .data = &n->points, .size = sizeof(n->points) }; 218 RCC(rc, finish, iwkv_put(db, &key, &val, 0)); 219 } 220 221 fprintf(stdout, ">>>> Traverse in descending order\n"); 222 RCC(rc, finish, iwkv_cursor_open(db, &cur, IWKV_CURSOR_BEFORE_FIRST, 0)); 223 while ((rc = iwkv_cursor_to(cur, IWKV_CURSOR_NEXT)) == 0) { 224 IWKV_val key, val; 225 RCC(rc, finish, iwkv_cursor_get(cur, &key, &val)); 226 fprintf(stdout, "%.*s: %u\n", 227 (int) key.size, (char *) key.data, 228 *(uint8_t *) val.data); 229 iwkv_kv_dispose(&key, &val); 230 } 231 rc = 0; 232 iwkv_cursor_close(&cur); 233 234 fprintf(stdout, "\n>>>> Traverse in ascending order\n"); 235 RCC(rc, finish, iwkv_cursor_open(db, &cur, IWKV_CURSOR_AFTER_LAST, 0)); 236 while ((rc = iwkv_cursor_to(cur, IWKV_CURSOR_PREV)) == 0) { 237 IWKV_val key, val; 238 RCC(rc, finish, iwkv_cursor_get(cur, &key, &val)); 239 fprintf(stdout, "%.*s: %u\n", 240 (int) key.size, (char *) key.data, 241 *(uint8_t *) val.data); 242 iwkv_kv_dispose(&key, &val); 243 } 244 rc = 0; 245 iwkv_cursor_close(&cur); 246 247 // Select all keys greater or equal than: Manchester United 248 { 249 fprintf(stdout, "\n>>>> Records GE: %s\n", _points[9].club); 250 IWKV_val key = { .data = (void *) _points[9].club, .size = strlen(_points[9].club) }, val; 251 RCC(rc, finish, iwkv_cursor_open(db, &cur, IWKV_CURSOR_GE, &key)); 252 do { 253 RCC(rc, finish, iwkv_cursor_get(cur, &key, &val)); 254 fprintf(stdout, "%.*s: %u\n", 255 (int) key.size, (char *) key.data, 256 *(uint8_t *) val.data); 257 iwkv_kv_dispose(&key, &val); 258 } while ((rc = iwkv_cursor_to(cur, IWKV_CURSOR_NEXT)) == 0); 259 rc = 0; 260 } 261 iwkv_cursor_close(&cur); 262 263finish: 264 if (cur) { 265 iwkv_cursor_close(&cur); 266 } 267 iwkv_close(&iwkv); 268 return rc; 269} 270 271int main() { 272 iwrc rc = run(); 273 if (rc) { 274 iwlog_ecode_error3(rc); 275 return 1; 276 } 277 return 0; 278} 279``` 280 281Output: 282``` 283>>>> Traverse in descending order 284Wolverhampton Wanderers: 43 285West Ham United: 27 286Watford: 27 287Tottenham Hotspur: 41 288Southampton: 34 289Sheffield United: 43 290Norwich City: 21 291Newcastle United: 35 292Manchester United: 45 293Manchester City: 57 294Liverpool: 82 295Leicester City: 53 296Everton: 37 297Crystal Palace: 39 298Chelsea: 48 299Burnley: 39 300Brighton & Hove Albion: 29 301Aston Villa: 25 302Arsenal: 40 303AFC Bournemouth: 27 304 305>>>> Traverse in ascending order 306AFC Bournemouth: 27 307Arsenal: 40 308Aston Villa: 25 309Brighton & Hove Albion: 29 310Burnley: 39 311Chelsea: 48 312Crystal Palace: 39 313Everton: 37 314Leicester City: 53 315Liverpool: 82 316Manchester City: 57 317Manchester United: 45 318Newcastle United: 35 319Norwich City: 21 320Sheffield United: 43 321Southampton: 34 322Tottenham Hotspur: 41 323Watford: 27 324West Ham United: 27 325Wolverhampton Wanderers: 43 326 327>>>> Records GE: Manchester United 328Manchester United: 45 329Manchester City: 57 330Liverpool: 82 331Leicester City: 53 332Everton: 37 333Crystal Palace: 39 334Chelsea: 48 335Burnley: 39 336Brighton & Hove Albion: 29 337Aston Villa: 25 338Arsenal: 40 339AFC Bournemouth: 27 340``` 341 342 343