1// Copyright 2011 Google Inc. All rights reserved. 2// Use of this source code is governed by the Apache 2.0 3// license that can be found in the LICENSE file. 4 5// Package memcache provides a client for App Engine's distributed in-memory 6// key-value store for small chunks of arbitrary data. 7// 8// The fundamental operations get and set items, keyed by a string. 9// 10// item0, err := memcache.Get(c, "key") 11// if err != nil && err != memcache.ErrCacheMiss { 12// return err 13// } 14// if err == nil { 15// fmt.Fprintf(w, "memcache hit: Key=%q Val=[% x]\n", item0.Key, item0.Value) 16// } else { 17// fmt.Fprintf(w, "memcache miss\n") 18// } 19// 20// and 21// 22// item1 := &memcache.Item{ 23// Key: "foo", 24// Value: []byte("bar"), 25// } 26// if err := memcache.Set(c, item1); err != nil { 27// return err 28// } 29package memcache 30 31import ( 32 "bytes" 33 "encoding/gob" 34 "encoding/json" 35 "errors" 36 "time" 37 38 "appengine" 39 "appengine_internal" 40 "appengine_internal/github.com/golang/protobuf/proto" 41 42 pb "appengine_internal/memcache" 43) 44 45var ( 46 // ErrCacheMiss means that an operation failed 47 // because the item wasn't present. 48 ErrCacheMiss = errors.New("memcache: cache miss") 49 // ErrCASConflict means that a CompareAndSwap call failed due to the 50 // cached value being modified between the Get and the CompareAndSwap. 51 // If the cached value was simply evicted rather than replaced, 52 // ErrNotStored will be returned instead. 53 ErrCASConflict = errors.New("memcache: compare-and-swap conflict") 54 // ErrNoStats means that no statistics were available. 55 ErrNoStats = errors.New("memcache: no statistics available") 56 // ErrNotStored means that a conditional write operation (i.e. Add or 57 // CompareAndSwap) failed because the condition was not satisfied. 58 ErrNotStored = errors.New("memcache: item not stored") 59 // ErrServerError means that a server error occurred. 60 ErrServerError = errors.New("memcache: server error") 61) 62 63// Item is the unit of memcache gets and sets. 64type Item struct { 65 // Key is the Item's key (250 bytes maximum). 66 Key string 67 // Value is the Item's value. 68 Value []byte 69 // Object is the Item's value for use with a Codec. 70 Object interface{} 71 // Flags are server-opaque flags whose semantics are entirely up to the 72 // App Engine app. 73 Flags uint32 74 // Expiration is the maximum duration that the item will stay 75 // in the cache. 76 // The zero value means the Item has no expiration time. 77 // Subsecond precision is ignored. 78 // This is not set when getting items. 79 Expiration time.Duration 80 // casID is a client-opaque value used for compare-and-swap operations. 81 // Zero means that compare-and-swap is not used. 82 casID uint64 83} 84 85const ( 86 secondsIn30Years = 60 * 60 * 24 * 365 * 30 // from memcache server code 87 thirtyYears = time.Duration(secondsIn30Years) * time.Second 88) 89 90// protoToItem converts a protocol buffer item to a Go struct. 91func protoToItem(p *pb.MemcacheGetResponse_Item) *Item { 92 return &Item{ 93 Key: string(p.Key), 94 Value: p.Value, 95 Flags: p.GetFlags(), 96 casID: p.GetCasId(), 97 } 98} 99 100// If err is an appengine.MultiError, return its first element. Otherwise, return err. 101func singleError(err error) error { 102 if me, ok := err.(appengine.MultiError); ok { 103 return me[0] 104 } 105 return err 106} 107 108// Get gets the item for the given key. ErrCacheMiss is returned for a memcache 109// cache miss. The key must be at most 250 bytes in length. 110func Get(c appengine.Context, key string) (*Item, error) { 111 m, err := GetMulti(c, []string{key}) 112 if err != nil { 113 return nil, err 114 } 115 if _, ok := m[key]; !ok { 116 return nil, ErrCacheMiss 117 } 118 return m[key], nil 119} 120 121// GetMulti is a batch version of Get. The returned map from keys to items may 122// have fewer elements than the input slice, due to memcache cache misses. 123// Each key must be at most 250 bytes in length. 124func GetMulti(c appengine.Context, key []string) (map[string]*Item, error) { 125 if len(key) == 0 { 126 return nil, nil 127 } 128 keyAsBytes := make([][]byte, len(key)) 129 for i, k := range key { 130 keyAsBytes[i] = []byte(k) 131 } 132 req := &pb.MemcacheGetRequest{ 133 Key: keyAsBytes, 134 ForCas: proto.Bool(true), 135 } 136 res := &pb.MemcacheGetResponse{} 137 if err := c.Call("memcache", "Get", req, res, nil); err != nil { 138 return nil, err 139 } 140 m := make(map[string]*Item, len(res.Item)) 141 for _, p := range res.Item { 142 t := protoToItem(p) 143 m[t.Key] = t 144 } 145 return m, nil 146} 147 148// Delete deletes the item for the given key. 149// ErrCacheMiss is returned if the specified item can not be found. 150// The key must be at most 250 bytes in length. 151func Delete(c appengine.Context, key string) error { 152 return singleError(DeleteMulti(c, []string{key})) 153} 154 155// DeleteMulti is a batch version of Delete. 156// If any keys cannot be found, an appengine.MultiError is returned. 157// Each key must be at most 250 bytes in length. 158func DeleteMulti(c appengine.Context, key []string) error { 159 if len(key) == 0 { 160 return nil 161 } 162 req := &pb.MemcacheDeleteRequest{ 163 Item: make([]*pb.MemcacheDeleteRequest_Item, len(key)), 164 } 165 for i, k := range key { 166 req.Item[i] = &pb.MemcacheDeleteRequest_Item{Key: []byte(k)} 167 } 168 res := &pb.MemcacheDeleteResponse{} 169 if err := c.Call("memcache", "Delete", req, res, nil); err != nil { 170 return err 171 } 172 if len(res.DeleteStatus) != len(key) { 173 return ErrServerError 174 } 175 me, any := make(appengine.MultiError, len(key)), false 176 for i, s := range res.DeleteStatus { 177 switch s { 178 case pb.MemcacheDeleteResponse_DELETED: 179 // OK 180 case pb.MemcacheDeleteResponse_NOT_FOUND: 181 me[i] = ErrCacheMiss 182 any = true 183 default: 184 me[i] = ErrServerError 185 any = true 186 } 187 } 188 if any { 189 return me 190 } 191 return nil 192} 193 194// Increment atomically increments the decimal value in the given key 195// by delta and returns the new value. The value must fit in a uint64. 196// Overflow wraps around, and underflow is capped to zero. The 197// provided delta may be negative. If the key doesn't exist in 198// memcache, the provided initial value is used to atomically 199// populate it before the delta is applied. 200// The key must be at most 250 bytes in length. 201func Increment(c appengine.Context, key string, delta int64, initialValue uint64) (newValue uint64, err error) { 202 return incr(c, key, delta, &initialValue) 203} 204 205// IncrementExisting works like Increment but assumes that the key 206// already exists in memcache and doesn't take an initial value. 207// IncrementExisting can save work if calculating the initial value is 208// expensive. 209// An error is returned if the specified item can not be found. 210func IncrementExisting(c appengine.Context, key string, delta int64) (newValue uint64, err error) { 211 return incr(c, key, delta, nil) 212} 213 214func incr(c appengine.Context, key string, delta int64, initialValue *uint64) (newValue uint64, err error) { 215 req := &pb.MemcacheIncrementRequest{ 216 Key: []byte(key), 217 InitialValue: initialValue, 218 } 219 if delta >= 0 { 220 req.Delta = proto.Uint64(uint64(delta)) 221 } else { 222 req.Delta = proto.Uint64(uint64(-delta)) 223 req.Direction = pb.MemcacheIncrementRequest_DECREMENT.Enum() 224 } 225 res := &pb.MemcacheIncrementResponse{} 226 err = c.Call("memcache", "Increment", req, res, nil) 227 if err != nil { 228 return 229 } 230 if res.NewValue == nil { 231 return 0, ErrCacheMiss 232 } 233 return *res.NewValue, nil 234} 235 236// set sets the given items using the given conflict resolution policy. 237// appengine.MultiError may be returned. 238func set(c appengine.Context, item []*Item, value [][]byte, policy pb.MemcacheSetRequest_SetPolicy) error { 239 if len(item) == 0 { 240 return nil 241 } 242 req := &pb.MemcacheSetRequest{ 243 Item: make([]*pb.MemcacheSetRequest_Item, len(item)), 244 } 245 for i, t := range item { 246 p := &pb.MemcacheSetRequest_Item{ 247 Key: []byte(t.Key), 248 } 249 if value == nil { 250 p.Value = t.Value 251 } else { 252 p.Value = value[i] 253 } 254 if t.Flags != 0 { 255 p.Flags = proto.Uint32(t.Flags) 256 } 257 if t.Expiration != 0 { 258 // In the .proto file, MemcacheSetRequest_Item uses a fixed32 (i.e. unsigned) 259 // for expiration time, while MemcacheGetRequest_Item uses int32 (i.e. signed). 260 // Throughout this .go file, we use int32. 261 // Also, in the proto, the expiration value is either a duration (in seconds) 262 // or an absolute Unix timestamp (in seconds), depending on whether the 263 // value is less than or greater than or equal to 30 years, respectively. 264 if t.Expiration < time.Second { 265 // Because an Expiration of 0 means no expiration, we take 266 // care here to translate an item with an expiration 267 // Duration between 0-1 seconds as immediately expiring 268 // (saying it expired a few seconds ago), rather than 269 // rounding it down to 0 and making it live forever. 270 p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) - 5) 271 } else if t.Expiration >= thirtyYears { 272 p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) + uint32(t.Expiration/time.Second)) 273 } else { 274 p.ExpirationTime = proto.Uint32(uint32(t.Expiration / time.Second)) 275 } 276 } 277 if t.casID != 0 { 278 p.CasId = proto.Uint64(t.casID) 279 p.ForCas = proto.Bool(true) 280 } 281 p.SetPolicy = policy.Enum() 282 req.Item[i] = p 283 } 284 res := &pb.MemcacheSetResponse{} 285 if err := c.Call("memcache", "Set", req, res, nil); err != nil { 286 return err 287 } 288 if len(res.SetStatus) != len(item) { 289 return ErrServerError 290 } 291 me, any := make(appengine.MultiError, len(item)), false 292 for i, st := range res.SetStatus { 293 var err error 294 switch st { 295 case pb.MemcacheSetResponse_STORED: 296 // OK 297 case pb.MemcacheSetResponse_NOT_STORED: 298 err = ErrNotStored 299 case pb.MemcacheSetResponse_EXISTS: 300 err = ErrCASConflict 301 default: 302 err = ErrServerError 303 } 304 if err != nil { 305 me[i] = err 306 any = true 307 } 308 } 309 if any { 310 return me 311 } 312 return nil 313} 314 315// Set writes the given item, unconditionally. 316func Set(c appengine.Context, item *Item) error { 317 return singleError(set(c, []*Item{item}, nil, pb.MemcacheSetRequest_SET)) 318} 319 320// SetMulti is a batch version of Set. 321// appengine.MultiError may be returned. 322func SetMulti(c appengine.Context, item []*Item) error { 323 return set(c, item, nil, pb.MemcacheSetRequest_SET) 324} 325 326// Add writes the given item, if no value already exists for its key. 327// ErrNotStored is returned if that condition is not met. 328func Add(c appengine.Context, item *Item) error { 329 return singleError(set(c, []*Item{item}, nil, pb.MemcacheSetRequest_ADD)) 330} 331 332// AddMulti is a batch version of Add. 333// appengine.MultiError may be returned. 334func AddMulti(c appengine.Context, item []*Item) error { 335 return set(c, item, nil, pb.MemcacheSetRequest_ADD) 336} 337 338// CompareAndSwap writes the given item that was previously returned by Get, 339// if the value was neither modified or evicted between the Get and the 340// CompareAndSwap calls. The item's Key should not change between calls but 341// all other item fields may differ. 342// ErrCASConflict is returned if the value was modified in between the calls. 343// ErrNotStored is returned if the value was evicted in between the calls. 344func CompareAndSwap(c appengine.Context, item *Item) error { 345 return singleError(set(c, []*Item{item}, nil, pb.MemcacheSetRequest_CAS)) 346} 347 348// CompareAndSwapMulti is a batch version of CompareAndSwap. 349// appengine.MultiError may be returned. 350func CompareAndSwapMulti(c appengine.Context, item []*Item) error { 351 return set(c, item, nil, pb.MemcacheSetRequest_CAS) 352} 353 354// Codec represents a symmetric pair of functions that implement a codec. 355// Items stored into or retrieved from memcache using a Codec have their 356// values marshaled or unmarshaled. 357// 358// All the methods provided for Codec behave analogously to the package level 359// function with same name. 360type Codec struct { 361 Marshal func(interface{}) ([]byte, error) 362 Unmarshal func([]byte, interface{}) error 363} 364 365// Get gets the item for the given key and decodes the obtained value into v. 366// ErrCacheMiss is returned for a memcache cache miss. 367// The key must be at most 250 bytes in length. 368func (cd Codec) Get(c appengine.Context, key string, v interface{}) (*Item, error) { 369 i, err := Get(c, key) 370 if err != nil { 371 return nil, err 372 } 373 if err := cd.Unmarshal(i.Value, v); err != nil { 374 return nil, err 375 } 376 return i, nil 377} 378 379func (cd Codec) set(c appengine.Context, items []*Item, policy pb.MemcacheSetRequest_SetPolicy) error { 380 var vs [][]byte 381 var me appengine.MultiError 382 for i, item := range items { 383 v, err := cd.Marshal(item.Object) 384 if err != nil { 385 if me == nil { 386 me = make(appengine.MultiError, len(items)) 387 } 388 me[i] = err 389 continue 390 } 391 if me == nil { 392 vs = append(vs, v) 393 } 394 } 395 if me != nil { 396 return me 397 } 398 399 return set(c, items, vs, policy) 400} 401 402// Set writes the given item, unconditionally. 403func (cd Codec) Set(c appengine.Context, item *Item) error { 404 return singleError(cd.set(c, []*Item{item}, pb.MemcacheSetRequest_SET)) 405} 406 407// SetMulti is a batch version of Set. 408// appengine.MultiError may be returned. 409func (cd Codec) SetMulti(c appengine.Context, items []*Item) error { 410 return cd.set(c, items, pb.MemcacheSetRequest_SET) 411} 412 413// Add writes the given item, if no value already exists for its key. 414// ErrNotStored is returned if that condition is not met. 415func (cd Codec) Add(c appengine.Context, item *Item) error { 416 return singleError(cd.set(c, []*Item{item}, pb.MemcacheSetRequest_ADD)) 417} 418 419// AddMulti is a batch version of Add. 420// appengine.MultiError may be returned. 421func (cd Codec) AddMulti(c appengine.Context, items []*Item) error { 422 return cd.set(c, items, pb.MemcacheSetRequest_ADD) 423} 424 425// CompareAndSwap writes the given item that was previously returned by Get, 426// if the value was neither modified or evicted between the Get and the 427// CompareAndSwap calls. The item's Key should not change between calls but 428// all other item fields may differ. 429// ErrCASConflict is returned if the value was modified in between the calls. 430// ErrNotStored is returned if the value was evicted in between the calls. 431func (cd Codec) CompareAndSwap(c appengine.Context, item *Item) error { 432 return singleError(cd.set(c, []*Item{item}, pb.MemcacheSetRequest_CAS)) 433} 434 435// CompareAndSwapMulti is a batch version of CompareAndSwap. 436// appengine.MultiError may be returned. 437func (cd Codec) CompareAndSwapMulti(c appengine.Context, items []*Item) error { 438 return cd.set(c, items, pb.MemcacheSetRequest_CAS) 439} 440 441var ( 442 // Gob is a Codec that uses the gob package. 443 Gob = Codec{gobMarshal, gobUnmarshal} 444 // JSON is a Codec that uses the json package. 445 JSON = Codec{json.Marshal, json.Unmarshal} 446) 447 448func gobMarshal(v interface{}) ([]byte, error) { 449 var buf bytes.Buffer 450 if err := gob.NewEncoder(&buf).Encode(v); err != nil { 451 return nil, err 452 } 453 return buf.Bytes(), nil 454} 455 456func gobUnmarshal(data []byte, v interface{}) error { 457 return gob.NewDecoder(bytes.NewBuffer(data)).Decode(v) 458} 459 460// Statistics represents a set of statistics about the memcache cache. 461// This may include items that have expired but have not yet been removed from the cache. 462type Statistics struct { 463 Hits uint64 // Counter of cache hits 464 Misses uint64 // Counter of cache misses 465 ByteHits uint64 // Counter of bytes transferred for gets 466 467 Items uint64 // Items currently in the cache 468 Bytes uint64 // Size of all items currently in the cache 469 470 Oldest int64 // Age of access of the oldest item, in seconds 471} 472 473// Stats retrieves the current memcache statistics. 474func Stats(c appengine.Context) (*Statistics, error) { 475 req := &pb.MemcacheStatsRequest{} 476 res := &pb.MemcacheStatsResponse{} 477 if err := c.Call("memcache", "Stats", req, res, nil); err != nil { 478 return nil, err 479 } 480 if res.Stats == nil { 481 return nil, ErrNoStats 482 } 483 return &Statistics{ 484 Hits: *res.Stats.Hits, 485 Misses: *res.Stats.Misses, 486 ByteHits: *res.Stats.ByteHits, 487 Items: *res.Stats.Items, 488 Bytes: *res.Stats.Bytes, 489 Oldest: int64(*res.Stats.OldestItemAge), 490 }, nil 491} 492 493// Flush flushes all items from memcache. 494func Flush(c appengine.Context) error { 495 req := &pb.MemcacheFlushRequest{} 496 res := &pb.MemcacheFlushResponse{} 497 return c.Call("memcache", "FlushAll", req, res, nil) 498} 499 500func namespaceMod(m appengine_internal.ProtoMessage, namespace string) { 501 switch m := m.(type) { 502 case *pb.MemcacheDeleteRequest: 503 if m.NameSpace == nil { 504 m.NameSpace = &namespace 505 } 506 case *pb.MemcacheGetRequest: 507 if m.NameSpace == nil { 508 m.NameSpace = &namespace 509 } 510 case *pb.MemcacheIncrementRequest: 511 if m.NameSpace == nil { 512 m.NameSpace = &namespace 513 } 514 case *pb.MemcacheSetRequest: 515 if m.NameSpace == nil { 516 m.NameSpace = &namespace 517 } 518 // MemcacheFlushRequest, MemcacheStatsRequest do not apply namespace. 519 } 520} 521 522func init() { 523 appengine_internal.RegisterErrorCodeMap("memcache", pb.MemcacheServiceError_ErrorCode_name) 524 appengine_internal.NamespaceMods["memcache"] = namespaceMod 525} 526