1// Package models implements basic objects used throughout the TICK stack. 2package models // import "github.com/influxdata/influxdb/models" 3 4import ( 5 "bytes" 6 "encoding/binary" 7 "errors" 8 "fmt" 9 "io" 10 "math" 11 "sort" 12 "strconv" 13 "strings" 14 "time" 15 "unicode" 16 "unicode/utf8" 17 18 "github.com/influxdata/influxdb/pkg/escape" 19) 20 21type escapeSet struct { 22 k [1]byte 23 esc [2]byte 24} 25 26var ( 27 measurementEscapeCodes = [...]escapeSet{ 28 {k: [1]byte{','}, esc: [2]byte{'\\', ','}}, 29 {k: [1]byte{' '}, esc: [2]byte{'\\', ' '}}, 30 } 31 32 tagEscapeCodes = [...]escapeSet{ 33 {k: [1]byte{','}, esc: [2]byte{'\\', ','}}, 34 {k: [1]byte{' '}, esc: [2]byte{'\\', ' '}}, 35 {k: [1]byte{'='}, esc: [2]byte{'\\', '='}}, 36 } 37 38 // ErrPointMustHaveAField is returned when operating on a point that does not have any fields. 39 ErrPointMustHaveAField = errors.New("point without fields is unsupported") 40 41 // ErrInvalidNumber is returned when a number is expected but not provided. 42 ErrInvalidNumber = errors.New("invalid number") 43 44 // ErrInvalidPoint is returned when a point cannot be parsed correctly. 45 ErrInvalidPoint = errors.New("point is invalid") 46) 47 48const ( 49 // MaxKeyLength is the largest allowed size of the combined measurement and tag keys. 50 MaxKeyLength = 65535 51) 52 53// enableUint64Support will enable uint64 support if set to true. 54var enableUint64Support = false 55 56// EnableUintSupport manually enables uint support for the point parser. 57// This function will be removed in the future and only exists for unit tests during the 58// transition. 59func EnableUintSupport() { 60 enableUint64Support = true 61} 62 63// Point defines the values that will be written to the database. 64type Point interface { 65 // Name return the measurement name for the point. 66 Name() []byte 67 68 // SetName updates the measurement name for the point. 69 SetName(string) 70 71 // Tags returns the tag set for the point. 72 Tags() Tags 73 74 // ForEachTag iterates over each tag invoking fn. If fn return false, iteration stops. 75 ForEachTag(fn func(k, v []byte) bool) 76 77 // AddTag adds or replaces a tag value for a point. 78 AddTag(key, value string) 79 80 // SetTags replaces the tags for the point. 81 SetTags(tags Tags) 82 83 // HasTag returns true if the tag exists for the point. 84 HasTag(tag []byte) bool 85 86 // Fields returns the fields for the point. 87 Fields() (Fields, error) 88 89 // Time return the timestamp for the point. 90 Time() time.Time 91 92 // SetTime updates the timestamp for the point. 93 SetTime(t time.Time) 94 95 // UnixNano returns the timestamp of the point as nanoseconds since Unix epoch. 96 UnixNano() int64 97 98 // HashID returns a non-cryptographic checksum of the point's key. 99 HashID() uint64 100 101 // Key returns the key (measurement joined with tags) of the point. 102 Key() []byte 103 104 // String returns a string representation of the point. If there is a 105 // timestamp associated with the point then it will be specified with the default 106 // precision of nanoseconds. 107 String() string 108 109 // MarshalBinary returns a binary representation of the point. 110 MarshalBinary() ([]byte, error) 111 112 // PrecisionString returns a string representation of the point. If there 113 // is a timestamp associated with the point then it will be specified in the 114 // given unit. 115 PrecisionString(precision string) string 116 117 // RoundedString returns a string representation of the point. If there 118 // is a timestamp associated with the point, then it will be rounded to the 119 // given duration. 120 RoundedString(d time.Duration) string 121 122 // Split will attempt to return multiple points with the same timestamp whose 123 // string representations are no longer than size. Points with a single field or 124 // a point without a timestamp may exceed the requested size. 125 Split(size int) []Point 126 127 // Round will round the timestamp of the point to the given duration. 128 Round(d time.Duration) 129 130 // StringSize returns the length of the string that would be returned by String(). 131 StringSize() int 132 133 // AppendString appends the result of String() to the provided buffer and returns 134 // the result, potentially reducing string allocations. 135 AppendString(buf []byte) []byte 136 137 // FieldIterator retuns a FieldIterator that can be used to traverse the 138 // fields of a point without constructing the in-memory map. 139 FieldIterator() FieldIterator 140} 141 142// FieldType represents the type of a field. 143type FieldType int 144 145const ( 146 // Integer indicates the field's type is integer. 147 Integer FieldType = iota 148 149 // Float indicates the field's type is float. 150 Float 151 152 // Boolean indicates the field's type is boolean. 153 Boolean 154 155 // String indicates the field's type is string. 156 String 157 158 // Empty is used to indicate that there is no field. 159 Empty 160 161 // Unsigned indicates the field's type is an unsigned integer. 162 Unsigned 163) 164 165// FieldIterator provides a low-allocation interface to iterate through a point's fields. 166type FieldIterator interface { 167 // Next indicates whether there any fields remaining. 168 Next() bool 169 170 // FieldKey returns the key of the current field. 171 FieldKey() []byte 172 173 // Type returns the FieldType of the current field. 174 Type() FieldType 175 176 // StringValue returns the string value of the current field. 177 StringValue() string 178 179 // IntegerValue returns the integer value of the current field. 180 IntegerValue() (int64, error) 181 182 // UnsignedValue returns the unsigned value of the current field. 183 UnsignedValue() (uint64, error) 184 185 // BooleanValue returns the boolean value of the current field. 186 BooleanValue() (bool, error) 187 188 // FloatValue returns the float value of the current field. 189 FloatValue() (float64, error) 190 191 // Reset resets the iterator to its initial state. 192 Reset() 193} 194 195// Points represents a sortable list of points by timestamp. 196type Points []Point 197 198// Len implements sort.Interface. 199func (a Points) Len() int { return len(a) } 200 201// Less implements sort.Interface. 202func (a Points) Less(i, j int) bool { return a[i].Time().Before(a[j].Time()) } 203 204// Swap implements sort.Interface. 205func (a Points) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 206 207// point is the default implementation of Point. 208type point struct { 209 time time.Time 210 211 // text encoding of measurement and tags 212 // key must always be stored sorted by tags, if the original line was not sorted, 213 // we need to resort it 214 key []byte 215 216 // text encoding of field data 217 fields []byte 218 219 // text encoding of timestamp 220 ts []byte 221 222 // cached version of parsed fields from data 223 cachedFields map[string]interface{} 224 225 // cached version of parsed name from key 226 cachedName string 227 228 // cached version of parsed tags 229 cachedTags Tags 230 231 it fieldIterator 232} 233 234// type assertions 235var ( 236 _ Point = (*point)(nil) 237 _ FieldIterator = (*point)(nil) 238) 239 240const ( 241 // the number of characters for the largest possible int64 (9223372036854775807) 242 maxInt64Digits = 19 243 244 // the number of characters for the smallest possible int64 (-9223372036854775808) 245 minInt64Digits = 20 246 247 // the number of characters for the largest possible uint64 (18446744073709551615) 248 maxUint64Digits = 20 249 250 // the number of characters required for the largest float64 before a range check 251 // would occur during parsing 252 maxFloat64Digits = 25 253 254 // the number of characters required for smallest float64 before a range check occur 255 // would occur during parsing 256 minFloat64Digits = 27 257) 258 259// ParsePoints returns a slice of Points from a text representation of a point 260// with each point separated by newlines. If any points fail to parse, a non-nil error 261// will be returned in addition to the points that parsed successfully. 262func ParsePoints(buf []byte) ([]Point, error) { 263 return ParsePointsWithPrecision(buf, time.Now().UTC(), "n") 264} 265 266// ParsePointsString is identical to ParsePoints but accepts a string. 267func ParsePointsString(buf string) ([]Point, error) { 268 return ParsePoints([]byte(buf)) 269} 270 271// ParseKey returns the measurement name and tags from a point. 272// 273// NOTE: to minimize heap allocations, the returned Tags will refer to subslices of buf. 274// This can have the unintended effect preventing buf from being garbage collected. 275func ParseKey(buf []byte) (string, Tags) { 276 name, tags := ParseKeyBytes(buf) 277 return string(name), tags 278} 279 280func ParseKeyBytes(buf []byte) ([]byte, Tags) { 281 return ParseKeyBytesWithTags(buf, nil) 282} 283 284func ParseKeyBytesWithTags(buf []byte, tags Tags) ([]byte, Tags) { 285 // Ignore the error because scanMeasurement returns "missing fields" which we ignore 286 // when just parsing a key 287 state, i, _ := scanMeasurement(buf, 0) 288 289 var name []byte 290 if state == tagKeyState { 291 tags = parseTags(buf, tags) 292 // scanMeasurement returns the location of the comma if there are tags, strip that off 293 name = buf[:i-1] 294 } else { 295 name = buf[:i] 296 } 297 return unescapeMeasurement(name), tags 298} 299 300func ParseTags(buf []byte) Tags { 301 return parseTags(buf, nil) 302} 303 304func ParseName(buf []byte) []byte { 305 // Ignore the error because scanMeasurement returns "missing fields" which we ignore 306 // when just parsing a key 307 state, i, _ := scanMeasurement(buf, 0) 308 var name []byte 309 if state == tagKeyState { 310 name = buf[:i-1] 311 } else { 312 name = buf[:i] 313 } 314 315 return unescapeMeasurement(name) 316} 317 318// ParsePointsWithPrecision is similar to ParsePoints, but allows the 319// caller to provide a precision for time. 320// 321// NOTE: to minimize heap allocations, the returned Points will refer to subslices of buf. 322// This can have the unintended effect preventing buf from being garbage collected. 323func ParsePointsWithPrecision(buf []byte, defaultTime time.Time, precision string) ([]Point, error) { 324 points := make([]Point, 0, bytes.Count(buf, []byte{'\n'})+1) 325 var ( 326 pos int 327 block []byte 328 failed []string 329 ) 330 for pos < len(buf) { 331 pos, block = scanLine(buf, pos) 332 pos++ 333 334 if len(block) == 0 { 335 continue 336 } 337 338 // lines which start with '#' are comments 339 start := skipWhitespace(block, 0) 340 341 // If line is all whitespace, just skip it 342 if start >= len(block) { 343 continue 344 } 345 346 if block[start] == '#' { 347 continue 348 } 349 350 // strip the newline if one is present 351 if block[len(block)-1] == '\n' { 352 block = block[:len(block)-1] 353 } 354 355 pt, err := parsePoint(block[start:], defaultTime, precision) 356 if err != nil { 357 failed = append(failed, fmt.Sprintf("unable to parse '%s': %v", string(block[start:]), err)) 358 } else { 359 points = append(points, pt) 360 } 361 362 } 363 if len(failed) > 0 { 364 return points, fmt.Errorf("%s", strings.Join(failed, "\n")) 365 } 366 return points, nil 367 368} 369 370func parsePoint(buf []byte, defaultTime time.Time, precision string) (Point, error) { 371 // scan the first block which is measurement[,tag1=value1,tag2=value=2...] 372 pos, key, err := scanKey(buf, 0) 373 if err != nil { 374 return nil, err 375 } 376 377 // measurement name is required 378 if len(key) == 0 { 379 return nil, fmt.Errorf("missing measurement") 380 } 381 382 if len(key) > MaxKeyLength { 383 return nil, fmt.Errorf("max key length exceeded: %v > %v", len(key), MaxKeyLength) 384 } 385 386 // scan the second block is which is field1=value1[,field2=value2,...] 387 pos, fields, err := scanFields(buf, pos) 388 if err != nil { 389 return nil, err 390 } 391 392 // at least one field is required 393 if len(fields) == 0 { 394 return nil, fmt.Errorf("missing fields") 395 } 396 397 var maxKeyErr error 398 walkFields(fields, func(k, v []byte) bool { 399 if sz := seriesKeySize(key, k); sz > MaxKeyLength { 400 maxKeyErr = fmt.Errorf("max key length exceeded: %v > %v", sz, MaxKeyLength) 401 return false 402 } 403 return true 404 }) 405 406 if maxKeyErr != nil { 407 return nil, maxKeyErr 408 } 409 410 // scan the last block which is an optional integer timestamp 411 pos, ts, err := scanTime(buf, pos) 412 if err != nil { 413 return nil, err 414 } 415 416 pt := &point{ 417 key: key, 418 fields: fields, 419 ts: ts, 420 } 421 422 if len(ts) == 0 { 423 pt.time = defaultTime 424 pt.SetPrecision(precision) 425 } else { 426 ts, err := parseIntBytes(ts, 10, 64) 427 if err != nil { 428 return nil, err 429 } 430 pt.time, err = SafeCalcTime(ts, precision) 431 if err != nil { 432 return nil, err 433 } 434 435 // Determine if there are illegal non-whitespace characters after the 436 // timestamp block. 437 for pos < len(buf) { 438 if buf[pos] != ' ' { 439 return nil, ErrInvalidPoint 440 } 441 pos++ 442 } 443 } 444 return pt, nil 445} 446 447// GetPrecisionMultiplier will return a multiplier for the precision specified. 448func GetPrecisionMultiplier(precision string) int64 { 449 d := time.Nanosecond 450 switch precision { 451 case "u": 452 d = time.Microsecond 453 case "ms": 454 d = time.Millisecond 455 case "s": 456 d = time.Second 457 case "m": 458 d = time.Minute 459 case "h": 460 d = time.Hour 461 } 462 return int64(d) 463} 464 465// scanKey scans buf starting at i for the measurement and tag portion of the point. 466// It returns the ending position and the byte slice of key within buf. If there 467// are tags, they will be sorted if they are not already. 468func scanKey(buf []byte, i int) (int, []byte, error) { 469 start := skipWhitespace(buf, i) 470 471 i = start 472 473 // Determines whether the tags are sort, assume they are 474 sorted := true 475 476 // indices holds the indexes within buf of the start of each tag. For example, 477 // a buf of 'cpu,host=a,region=b,zone=c' would have indices slice of [4,11,20] 478 // which indicates that the first tag starts at buf[4], seconds at buf[11], and 479 // last at buf[20] 480 indices := make([]int, 100) 481 482 // tracks how many commas we've seen so we know how many values are indices. 483 // Since indices is an arbitrarily large slice, 484 // we need to know how many values in the buffer are in use. 485 commas := 0 486 487 // First scan the Point's measurement. 488 state, i, err := scanMeasurement(buf, i) 489 if err != nil { 490 return i, buf[start:i], err 491 } 492 493 // Optionally scan tags if needed. 494 if state == tagKeyState { 495 i, commas, indices, err = scanTags(buf, i, indices) 496 if err != nil { 497 return i, buf[start:i], err 498 } 499 } 500 501 // Now we know where the key region is within buf, and the location of tags, we 502 // need to determine if duplicate tags exist and if the tags are sorted. This iterates 503 // over the list comparing each tag in the sequence with each other. 504 for j := 0; j < commas-1; j++ { 505 // get the left and right tags 506 _, left := scanTo(buf[indices[j]:indices[j+1]-1], 0, '=') 507 _, right := scanTo(buf[indices[j+1]:indices[j+2]-1], 0, '=') 508 509 // If left is greater than right, the tags are not sorted. We do not have to 510 // continue because the short path no longer works. 511 // If the tags are equal, then there are duplicate tags, and we should abort. 512 // If the tags are not sorted, this pass may not find duplicate tags and we 513 // need to do a more exhaustive search later. 514 if cmp := bytes.Compare(left, right); cmp > 0 { 515 sorted = false 516 break 517 } else if cmp == 0 { 518 return i, buf[start:i], fmt.Errorf("duplicate tags") 519 } 520 } 521 522 // If the tags are not sorted, then sort them. This sort is inline and 523 // uses the tag indices we created earlier. The actual buffer is not sorted, the 524 // indices are using the buffer for value comparison. After the indices are sorted, 525 // the buffer is reconstructed from the sorted indices. 526 if !sorted && commas > 0 { 527 // Get the measurement name for later 528 measurement := buf[start : indices[0]-1] 529 530 // Sort the indices 531 indices := indices[:commas] 532 insertionSort(0, commas, buf, indices) 533 534 // Create a new key using the measurement and sorted indices 535 b := make([]byte, len(buf[start:i])) 536 pos := copy(b, measurement) 537 for _, i := range indices { 538 b[pos] = ',' 539 pos++ 540 _, v := scanToSpaceOr(buf, i, ',') 541 pos += copy(b[pos:], v) 542 } 543 544 // Check again for duplicate tags now that the tags are sorted. 545 for j := 0; j < commas-1; j++ { 546 // get the left and right tags 547 _, left := scanTo(buf[indices[j]:], 0, '=') 548 _, right := scanTo(buf[indices[j+1]:], 0, '=') 549 550 // If the tags are equal, then there are duplicate tags, and we should abort. 551 // If the tags are not sorted, this pass may not find duplicate tags and we 552 // need to do a more exhaustive search later. 553 if bytes.Equal(left, right) { 554 return i, b, fmt.Errorf("duplicate tags") 555 } 556 } 557 558 return i, b, nil 559 } 560 561 return i, buf[start:i], nil 562} 563 564// The following constants allow us to specify which state to move to 565// next, when scanning sections of a Point. 566const ( 567 tagKeyState = iota 568 tagValueState 569 fieldsState 570) 571 572// scanMeasurement examines the measurement part of a Point, returning 573// the next state to move to, and the current location in the buffer. 574func scanMeasurement(buf []byte, i int) (int, int, error) { 575 // Check first byte of measurement, anything except a comma is fine. 576 // It can't be a space, since whitespace is stripped prior to this 577 // function call. 578 if i >= len(buf) || buf[i] == ',' { 579 return -1, i, fmt.Errorf("missing measurement") 580 } 581 582 for { 583 i++ 584 if i >= len(buf) { 585 // cpu 586 return -1, i, fmt.Errorf("missing fields") 587 } 588 589 if buf[i-1] == '\\' { 590 // Skip character (it's escaped). 591 continue 592 } 593 594 // Unescaped comma; move onto scanning the tags. 595 if buf[i] == ',' { 596 return tagKeyState, i + 1, nil 597 } 598 599 // Unescaped space; move onto scanning the fields. 600 if buf[i] == ' ' { 601 // cpu value=1.0 602 return fieldsState, i, nil 603 } 604 } 605} 606 607// scanTags examines all the tags in a Point, keeping track of and 608// returning the updated indices slice, number of commas and location 609// in buf where to start examining the Point fields. 610func scanTags(buf []byte, i int, indices []int) (int, int, []int, error) { 611 var ( 612 err error 613 commas int 614 state = tagKeyState 615 ) 616 617 for { 618 switch state { 619 case tagKeyState: 620 // Grow our indices slice if we have too many tags. 621 if commas >= len(indices) { 622 newIndics := make([]int, cap(indices)*2) 623 copy(newIndics, indices) 624 indices = newIndics 625 } 626 indices[commas] = i 627 commas++ 628 629 i, err = scanTagsKey(buf, i) 630 state = tagValueState // tag value always follows a tag key 631 case tagValueState: 632 state, i, err = scanTagsValue(buf, i) 633 case fieldsState: 634 indices[commas] = i + 1 635 return i, commas, indices, nil 636 } 637 638 if err != nil { 639 return i, commas, indices, err 640 } 641 } 642} 643 644// scanTagsKey scans each character in a tag key. 645func scanTagsKey(buf []byte, i int) (int, error) { 646 // First character of the key. 647 if i >= len(buf) || buf[i] == ' ' || buf[i] == ',' || buf[i] == '=' { 648 // cpu,{'', ' ', ',', '='} 649 return i, fmt.Errorf("missing tag key") 650 } 651 652 // Examine each character in the tag key until we hit an unescaped 653 // equals (the tag value), or we hit an error (i.e., unescaped 654 // space or comma). 655 for { 656 i++ 657 658 // Either we reached the end of the buffer or we hit an 659 // unescaped comma or space. 660 if i >= len(buf) || 661 ((buf[i] == ' ' || buf[i] == ',') && buf[i-1] != '\\') { 662 // cpu,tag{'', ' ', ','} 663 return i, fmt.Errorf("missing tag value") 664 } 665 666 if buf[i] == '=' && buf[i-1] != '\\' { 667 // cpu,tag= 668 return i + 1, nil 669 } 670 } 671} 672 673// scanTagsValue scans each character in a tag value. 674func scanTagsValue(buf []byte, i int) (int, int, error) { 675 // Tag value cannot be empty. 676 if i >= len(buf) || buf[i] == ',' || buf[i] == ' ' { 677 // cpu,tag={',', ' '} 678 return -1, i, fmt.Errorf("missing tag value") 679 } 680 681 // Examine each character in the tag value until we hit an unescaped 682 // comma (move onto next tag key), an unescaped space (move onto 683 // fields), or we error out. 684 for { 685 i++ 686 if i >= len(buf) { 687 // cpu,tag=value 688 return -1, i, fmt.Errorf("missing fields") 689 } 690 691 // An unescaped equals sign is an invalid tag value. 692 if buf[i] == '=' && buf[i-1] != '\\' { 693 // cpu,tag={'=', 'fo=o'} 694 return -1, i, fmt.Errorf("invalid tag format") 695 } 696 697 if buf[i] == ',' && buf[i-1] != '\\' { 698 // cpu,tag=foo, 699 return tagKeyState, i + 1, nil 700 } 701 702 // cpu,tag=foo value=1.0 703 // cpu, tag=foo\= value=1.0 704 if buf[i] == ' ' && buf[i-1] != '\\' { 705 return fieldsState, i, nil 706 } 707 } 708} 709 710func insertionSort(l, r int, buf []byte, indices []int) { 711 for i := l + 1; i < r; i++ { 712 for j := i; j > l && less(buf, indices, j, j-1); j-- { 713 indices[j], indices[j-1] = indices[j-1], indices[j] 714 } 715 } 716} 717 718func less(buf []byte, indices []int, i, j int) bool { 719 // This grabs the tag names for i & j, it ignores the values 720 _, a := scanTo(buf, indices[i], '=') 721 _, b := scanTo(buf, indices[j], '=') 722 return bytes.Compare(a, b) < 0 723} 724 725// scanFields scans buf, starting at i for the fields section of a point. It returns 726// the ending position and the byte slice of the fields within buf. 727func scanFields(buf []byte, i int) (int, []byte, error) { 728 start := skipWhitespace(buf, i) 729 i = start 730 quoted := false 731 732 // tracks how many '=' we've seen 733 equals := 0 734 735 // tracks how many commas we've seen 736 commas := 0 737 738 for { 739 // reached the end of buf? 740 if i >= len(buf) { 741 break 742 } 743 744 // escaped characters? 745 if buf[i] == '\\' && i+1 < len(buf) { 746 i += 2 747 continue 748 } 749 750 // If the value is quoted, scan until we get to the end quote 751 // Only quote values in the field value since quotes are not significant 752 // in the field key 753 if buf[i] == '"' && equals > commas { 754 quoted = !quoted 755 i++ 756 continue 757 } 758 759 // If we see an =, ensure that there is at least on char before and after it 760 if buf[i] == '=' && !quoted { 761 equals++ 762 763 // check for "... =123" but allow "a\ =123" 764 if buf[i-1] == ' ' && buf[i-2] != '\\' { 765 return i, buf[start:i], fmt.Errorf("missing field key") 766 } 767 768 // check for "...a=123,=456" but allow "a=123,a\,=456" 769 if buf[i-1] == ',' && buf[i-2] != '\\' { 770 return i, buf[start:i], fmt.Errorf("missing field key") 771 } 772 773 // check for "... value=" 774 if i+1 >= len(buf) { 775 return i, buf[start:i], fmt.Errorf("missing field value") 776 } 777 778 // check for "... value=,value2=..." 779 if buf[i+1] == ',' || buf[i+1] == ' ' { 780 return i, buf[start:i], fmt.Errorf("missing field value") 781 } 782 783 if isNumeric(buf[i+1]) || buf[i+1] == '-' || buf[i+1] == 'N' || buf[i+1] == 'n' { 784 var err error 785 i, err = scanNumber(buf, i+1) 786 if err != nil { 787 return i, buf[start:i], err 788 } 789 continue 790 } 791 // If next byte is not a double-quote, the value must be a boolean 792 if buf[i+1] != '"' { 793 var err error 794 i, _, err = scanBoolean(buf, i+1) 795 if err != nil { 796 return i, buf[start:i], err 797 } 798 continue 799 } 800 } 801 802 if buf[i] == ',' && !quoted { 803 commas++ 804 } 805 806 // reached end of block? 807 if buf[i] == ' ' && !quoted { 808 break 809 } 810 i++ 811 } 812 813 if quoted { 814 return i, buf[start:i], fmt.Errorf("unbalanced quotes") 815 } 816 817 // check that all field sections had key and values (e.g. prevent "a=1,b" 818 if equals == 0 || commas != equals-1 { 819 return i, buf[start:i], fmt.Errorf("invalid field format") 820 } 821 822 return i, buf[start:i], nil 823} 824 825// scanTime scans buf, starting at i for the time section of a point. It 826// returns the ending position and the byte slice of the timestamp within buf 827// and and error if the timestamp is not in the correct numeric format. 828func scanTime(buf []byte, i int) (int, []byte, error) { 829 start := skipWhitespace(buf, i) 830 i = start 831 832 for { 833 // reached the end of buf? 834 if i >= len(buf) { 835 break 836 } 837 838 // Reached end of block or trailing whitespace? 839 if buf[i] == '\n' || buf[i] == ' ' { 840 break 841 } 842 843 // Handle negative timestamps 844 if i == start && buf[i] == '-' { 845 i++ 846 continue 847 } 848 849 // Timestamps should be integers, make sure they are so we don't need 850 // to actually parse the timestamp until needed. 851 if buf[i] < '0' || buf[i] > '9' { 852 return i, buf[start:i], fmt.Errorf("bad timestamp") 853 } 854 i++ 855 } 856 return i, buf[start:i], nil 857} 858 859func isNumeric(b byte) bool { 860 return (b >= '0' && b <= '9') || b == '.' 861} 862 863// scanNumber returns the end position within buf, start at i after 864// scanning over buf for an integer, or float. It returns an 865// error if a invalid number is scanned. 866func scanNumber(buf []byte, i int) (int, error) { 867 start := i 868 var isInt, isUnsigned bool 869 870 // Is negative number? 871 if i < len(buf) && buf[i] == '-' { 872 i++ 873 // There must be more characters now, as just '-' is illegal. 874 if i == len(buf) { 875 return i, ErrInvalidNumber 876 } 877 } 878 879 // how many decimal points we've see 880 decimal := false 881 882 // indicates the number is float in scientific notation 883 scientific := false 884 885 for { 886 if i >= len(buf) { 887 break 888 } 889 890 if buf[i] == ',' || buf[i] == ' ' { 891 break 892 } 893 894 if buf[i] == 'i' && i > start && !(isInt || isUnsigned) { 895 isInt = true 896 i++ 897 continue 898 } else if buf[i] == 'u' && i > start && !(isInt || isUnsigned) { 899 isUnsigned = true 900 i++ 901 continue 902 } 903 904 if buf[i] == '.' { 905 // Can't have more than 1 decimal (e.g. 1.1.1 should fail) 906 if decimal { 907 return i, ErrInvalidNumber 908 } 909 decimal = true 910 } 911 912 // `e` is valid for floats but not as the first char 913 if i > start && (buf[i] == 'e' || buf[i] == 'E') { 914 scientific = true 915 i++ 916 continue 917 } 918 919 // + and - are only valid at this point if they follow an e (scientific notation) 920 if (buf[i] == '+' || buf[i] == '-') && (buf[i-1] == 'e' || buf[i-1] == 'E') { 921 i++ 922 continue 923 } 924 925 // NaN is an unsupported value 926 if i+2 < len(buf) && (buf[i] == 'N' || buf[i] == 'n') { 927 return i, ErrInvalidNumber 928 } 929 930 if !isNumeric(buf[i]) { 931 return i, ErrInvalidNumber 932 } 933 i++ 934 } 935 936 if (isInt || isUnsigned) && (decimal || scientific) { 937 return i, ErrInvalidNumber 938 } 939 940 numericDigits := i - start 941 if isInt { 942 numericDigits-- 943 } 944 if decimal { 945 numericDigits-- 946 } 947 if buf[start] == '-' { 948 numericDigits-- 949 } 950 951 if numericDigits == 0 { 952 return i, ErrInvalidNumber 953 } 954 955 // It's more common that numbers will be within min/max range for their type but we need to prevent 956 // out or range numbers from being parsed successfully. This uses some simple heuristics to decide 957 // if we should parse the number to the actual type. It does not do it all the time because it incurs 958 // extra allocations and we end up converting the type again when writing points to disk. 959 if isInt { 960 // Make sure the last char is an 'i' for integers (e.g. 9i10 is not valid) 961 if buf[i-1] != 'i' { 962 return i, ErrInvalidNumber 963 } 964 // Parse the int to check bounds the number of digits could be larger than the max range 965 // We subtract 1 from the index to remove the `i` from our tests 966 if len(buf[start:i-1]) >= maxInt64Digits || len(buf[start:i-1]) >= minInt64Digits { 967 if _, err := parseIntBytes(buf[start:i-1], 10, 64); err != nil { 968 return i, fmt.Errorf("unable to parse integer %s: %s", buf[start:i-1], err) 969 } 970 } 971 } else if isUnsigned { 972 // Return an error if uint64 support has not been enabled. 973 if !enableUint64Support { 974 return i, ErrInvalidNumber 975 } 976 // Make sure the last char is a 'u' for unsigned 977 if buf[i-1] != 'u' { 978 return i, ErrInvalidNumber 979 } 980 // Make sure the first char is not a '-' for unsigned 981 if buf[start] == '-' { 982 return i, ErrInvalidNumber 983 } 984 // Parse the uint to check bounds the number of digits could be larger than the max range 985 // We subtract 1 from the index to remove the `u` from our tests 986 if len(buf[start:i-1]) >= maxUint64Digits { 987 if _, err := parseUintBytes(buf[start:i-1], 10, 64); err != nil { 988 return i, fmt.Errorf("unable to parse unsigned %s: %s", buf[start:i-1], err) 989 } 990 } 991 } else { 992 // Parse the float to check bounds if it's scientific or the number of digits could be larger than the max range 993 if scientific || len(buf[start:i]) >= maxFloat64Digits || len(buf[start:i]) >= minFloat64Digits { 994 if _, err := parseFloatBytes(buf[start:i], 10); err != nil { 995 return i, fmt.Errorf("invalid float") 996 } 997 } 998 } 999 1000 return i, nil 1001} 1002 1003// scanBoolean returns the end position within buf, start at i after 1004// scanning over buf for boolean. Valid values for a boolean are 1005// t, T, true, TRUE, f, F, false, FALSE. It returns an error if a invalid boolean 1006// is scanned. 1007func scanBoolean(buf []byte, i int) (int, []byte, error) { 1008 start := i 1009 1010 if i < len(buf) && (buf[i] != 't' && buf[i] != 'f' && buf[i] != 'T' && buf[i] != 'F') { 1011 return i, buf[start:i], fmt.Errorf("invalid boolean") 1012 } 1013 1014 i++ 1015 for { 1016 if i >= len(buf) { 1017 break 1018 } 1019 1020 if buf[i] == ',' || buf[i] == ' ' { 1021 break 1022 } 1023 i++ 1024 } 1025 1026 // Single char bool (t, T, f, F) is ok 1027 if i-start == 1 { 1028 return i, buf[start:i], nil 1029 } 1030 1031 // length must be 4 for true or TRUE 1032 if (buf[start] == 't' || buf[start] == 'T') && i-start != 4 { 1033 return i, buf[start:i], fmt.Errorf("invalid boolean") 1034 } 1035 1036 // length must be 5 for false or FALSE 1037 if (buf[start] == 'f' || buf[start] == 'F') && i-start != 5 { 1038 return i, buf[start:i], fmt.Errorf("invalid boolean") 1039 } 1040 1041 // Otherwise 1042 valid := false 1043 switch buf[start] { 1044 case 't': 1045 valid = bytes.Equal(buf[start:i], []byte("true")) 1046 case 'f': 1047 valid = bytes.Equal(buf[start:i], []byte("false")) 1048 case 'T': 1049 valid = bytes.Equal(buf[start:i], []byte("TRUE")) || bytes.Equal(buf[start:i], []byte("True")) 1050 case 'F': 1051 valid = bytes.Equal(buf[start:i], []byte("FALSE")) || bytes.Equal(buf[start:i], []byte("False")) 1052 } 1053 1054 if !valid { 1055 return i, buf[start:i], fmt.Errorf("invalid boolean") 1056 } 1057 1058 return i, buf[start:i], nil 1059 1060} 1061 1062// skipWhitespace returns the end position within buf, starting at i after 1063// scanning over spaces in tags. 1064func skipWhitespace(buf []byte, i int) int { 1065 for i < len(buf) { 1066 if buf[i] != ' ' && buf[i] != '\t' && buf[i] != 0 { 1067 break 1068 } 1069 i++ 1070 } 1071 return i 1072} 1073 1074// scanLine returns the end position in buf and the next line found within 1075// buf. 1076func scanLine(buf []byte, i int) (int, []byte) { 1077 start := i 1078 quoted := false 1079 fields := false 1080 1081 // tracks how many '=' and commas we've seen 1082 // this duplicates some of the functionality in scanFields 1083 equals := 0 1084 commas := 0 1085 for { 1086 // reached the end of buf? 1087 if i >= len(buf) { 1088 break 1089 } 1090 1091 // skip past escaped characters 1092 if buf[i] == '\\' && i+2 < len(buf) { 1093 i += 2 1094 continue 1095 } 1096 1097 if buf[i] == ' ' { 1098 fields = true 1099 } 1100 1101 // If we see a double quote, makes sure it is not escaped 1102 if fields { 1103 if !quoted && buf[i] == '=' { 1104 i++ 1105 equals++ 1106 continue 1107 } else if !quoted && buf[i] == ',' { 1108 i++ 1109 commas++ 1110 continue 1111 } else if buf[i] == '"' && equals > commas { 1112 i++ 1113 quoted = !quoted 1114 continue 1115 } 1116 } 1117 1118 if buf[i] == '\n' && !quoted { 1119 break 1120 } 1121 1122 i++ 1123 } 1124 1125 return i, buf[start:i] 1126} 1127 1128// scanTo returns the end position in buf and the next consecutive block 1129// of bytes, starting from i and ending with stop byte, where stop byte 1130// has not been escaped. 1131// 1132// If there are leading spaces, they are skipped. 1133func scanTo(buf []byte, i int, stop byte) (int, []byte) { 1134 start := i 1135 for { 1136 // reached the end of buf? 1137 if i >= len(buf) { 1138 break 1139 } 1140 1141 // Reached unescaped stop value? 1142 if buf[i] == stop && (i == 0 || buf[i-1] != '\\') { 1143 break 1144 } 1145 i++ 1146 } 1147 1148 return i, buf[start:i] 1149} 1150 1151// scanTo returns the end position in buf and the next consecutive block 1152// of bytes, starting from i and ending with stop byte. If there are leading 1153// spaces, they are skipped. 1154func scanToSpaceOr(buf []byte, i int, stop byte) (int, []byte) { 1155 start := i 1156 if buf[i] == stop || buf[i] == ' ' { 1157 return i, buf[start:i] 1158 } 1159 1160 for { 1161 i++ 1162 if buf[i-1] == '\\' { 1163 continue 1164 } 1165 1166 // reached the end of buf? 1167 if i >= len(buf) { 1168 return i, buf[start:i] 1169 } 1170 1171 // reached end of block? 1172 if buf[i] == stop || buf[i] == ' ' { 1173 return i, buf[start:i] 1174 } 1175 } 1176} 1177 1178func scanTagValue(buf []byte, i int) (int, []byte) { 1179 start := i 1180 for { 1181 if i >= len(buf) { 1182 break 1183 } 1184 1185 if buf[i] == ',' && buf[i-1] != '\\' { 1186 break 1187 } 1188 i++ 1189 } 1190 if i > len(buf) { 1191 return i, nil 1192 } 1193 return i, buf[start:i] 1194} 1195 1196func scanFieldValue(buf []byte, i int) (int, []byte) { 1197 start := i 1198 quoted := false 1199 for i < len(buf) { 1200 // Only escape char for a field value is a double-quote and backslash 1201 if buf[i] == '\\' && i+1 < len(buf) && (buf[i+1] == '"' || buf[i+1] == '\\') { 1202 i += 2 1203 continue 1204 } 1205 1206 // Quoted value? (e.g. string) 1207 if buf[i] == '"' { 1208 i++ 1209 quoted = !quoted 1210 continue 1211 } 1212 1213 if buf[i] == ',' && !quoted { 1214 break 1215 } 1216 i++ 1217 } 1218 return i, buf[start:i] 1219} 1220 1221func EscapeMeasurement(in []byte) []byte { 1222 for _, c := range measurementEscapeCodes { 1223 if bytes.IndexByte(in, c.k[0]) != -1 { 1224 in = bytes.Replace(in, c.k[:], c.esc[:], -1) 1225 } 1226 } 1227 return in 1228} 1229 1230func unescapeMeasurement(in []byte) []byte { 1231 if bytes.IndexByte(in, '\\') == -1 { 1232 return in 1233 } 1234 1235 for i := range measurementEscapeCodes { 1236 c := &measurementEscapeCodes[i] 1237 if bytes.IndexByte(in, c.k[0]) != -1 { 1238 in = bytes.Replace(in, c.esc[:], c.k[:], -1) 1239 } 1240 } 1241 return in 1242} 1243 1244func escapeTag(in []byte) []byte { 1245 for i := range tagEscapeCodes { 1246 c := &tagEscapeCodes[i] 1247 if bytes.IndexByte(in, c.k[0]) != -1 { 1248 in = bytes.Replace(in, c.k[:], c.esc[:], -1) 1249 } 1250 } 1251 return in 1252} 1253 1254func unescapeTag(in []byte) []byte { 1255 if bytes.IndexByte(in, '\\') == -1 { 1256 return in 1257 } 1258 1259 for i := range tagEscapeCodes { 1260 c := &tagEscapeCodes[i] 1261 if bytes.IndexByte(in, c.k[0]) != -1 { 1262 in = bytes.Replace(in, c.esc[:], c.k[:], -1) 1263 } 1264 } 1265 return in 1266} 1267 1268// escapeStringFieldReplacer replaces double quotes and backslashes 1269// with the same character preceded by a backslash. 1270// As of Go 1.7 this benchmarked better in allocations and CPU time 1271// compared to iterating through a string byte-by-byte and appending to a new byte slice, 1272// calling strings.Replace twice, and better than (*Regex).ReplaceAllString. 1273var escapeStringFieldReplacer = strings.NewReplacer(`"`, `\"`, `\`, `\\`) 1274 1275// EscapeStringField returns a copy of in with any double quotes or 1276// backslashes with escaped values. 1277func EscapeStringField(in string) string { 1278 return escapeStringFieldReplacer.Replace(in) 1279} 1280 1281// unescapeStringField returns a copy of in with any escaped double-quotes 1282// or backslashes unescaped. 1283func unescapeStringField(in string) string { 1284 if strings.IndexByte(in, '\\') == -1 { 1285 return in 1286 } 1287 1288 var out []byte 1289 i := 0 1290 for { 1291 if i >= len(in) { 1292 break 1293 } 1294 // unescape backslashes 1295 if in[i] == '\\' && i+1 < len(in) && in[i+1] == '\\' { 1296 out = append(out, '\\') 1297 i += 2 1298 continue 1299 } 1300 // unescape double-quotes 1301 if in[i] == '\\' && i+1 < len(in) && in[i+1] == '"' { 1302 out = append(out, '"') 1303 i += 2 1304 continue 1305 } 1306 out = append(out, in[i]) 1307 i++ 1308 1309 } 1310 return string(out) 1311} 1312 1313// NewPoint returns a new point with the given measurement name, tags, fields and timestamp. If 1314// an unsupported field value (NaN, or +/-Inf) or out of range time is passed, this function 1315// returns an error. 1316func NewPoint(name string, tags Tags, fields Fields, t time.Time) (Point, error) { 1317 key, err := pointKey(name, tags, fields, t) 1318 if err != nil { 1319 return nil, err 1320 } 1321 1322 return &point{ 1323 key: key, 1324 time: t, 1325 fields: fields.MarshalBinary(), 1326 }, nil 1327} 1328 1329// pointKey checks some basic requirements for valid points, and returns the 1330// key, along with an possible error. 1331func pointKey(measurement string, tags Tags, fields Fields, t time.Time) ([]byte, error) { 1332 if len(fields) == 0 { 1333 return nil, ErrPointMustHaveAField 1334 } 1335 1336 if !t.IsZero() { 1337 if err := CheckTime(t); err != nil { 1338 return nil, err 1339 } 1340 } 1341 1342 for key, value := range fields { 1343 switch value := value.(type) { 1344 case float64: 1345 // Ensure the caller validates and handles invalid field values 1346 if math.IsInf(value, 0) { 1347 return nil, fmt.Errorf("+/-Inf is an unsupported value for field %s", key) 1348 } 1349 if math.IsNaN(value) { 1350 return nil, fmt.Errorf("NaN is an unsupported value for field %s", key) 1351 } 1352 case float32: 1353 // Ensure the caller validates and handles invalid field values 1354 if math.IsInf(float64(value), 0) { 1355 return nil, fmt.Errorf("+/-Inf is an unsupported value for field %s", key) 1356 } 1357 if math.IsNaN(float64(value)) { 1358 return nil, fmt.Errorf("NaN is an unsupported value for field %s", key) 1359 } 1360 } 1361 if len(key) == 0 { 1362 return nil, fmt.Errorf("all fields must have non-empty names") 1363 } 1364 } 1365 1366 key := MakeKey([]byte(measurement), tags) 1367 for field := range fields { 1368 sz := seriesKeySize(key, []byte(field)) 1369 if sz > MaxKeyLength { 1370 return nil, fmt.Errorf("max key length exceeded: %v > %v", sz, MaxKeyLength) 1371 } 1372 } 1373 1374 return key, nil 1375} 1376 1377func seriesKeySize(key, field []byte) int { 1378 // 4 is the length of the tsm1.fieldKeySeparator constant. It's inlined here to avoid a circular 1379 // dependency. 1380 return len(key) + 4 + len(field) 1381} 1382 1383// NewPointFromBytes returns a new Point from a marshalled Point. 1384func NewPointFromBytes(b []byte) (Point, error) { 1385 p := &point{} 1386 if err := p.UnmarshalBinary(b); err != nil { 1387 return nil, err 1388 } 1389 1390 // This does some basic validation to ensure there are fields and they 1391 // can be unmarshalled as well. 1392 iter := p.FieldIterator() 1393 var hasField bool 1394 for iter.Next() { 1395 if len(iter.FieldKey()) == 0 { 1396 continue 1397 } 1398 hasField = true 1399 switch iter.Type() { 1400 case Float: 1401 _, err := iter.FloatValue() 1402 if err != nil { 1403 return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) 1404 } 1405 case Integer: 1406 _, err := iter.IntegerValue() 1407 if err != nil { 1408 return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) 1409 } 1410 case Unsigned: 1411 _, err := iter.UnsignedValue() 1412 if err != nil { 1413 return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) 1414 } 1415 case String: 1416 // Skip since this won't return an error 1417 case Boolean: 1418 _, err := iter.BooleanValue() 1419 if err != nil { 1420 return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) 1421 } 1422 } 1423 } 1424 1425 if !hasField { 1426 return nil, ErrPointMustHaveAField 1427 } 1428 1429 return p, nil 1430} 1431 1432// MustNewPoint returns a new point with the given measurement name, tags, fields and timestamp. If 1433// an unsupported field value (NaN) is passed, this function panics. 1434func MustNewPoint(name string, tags Tags, fields Fields, time time.Time) Point { 1435 pt, err := NewPoint(name, tags, fields, time) 1436 if err != nil { 1437 panic(err.Error()) 1438 } 1439 return pt 1440} 1441 1442// Key returns the key (measurement joined with tags) of the point. 1443func (p *point) Key() []byte { 1444 return p.key 1445} 1446 1447func (p *point) name() []byte { 1448 _, name := scanTo(p.key, 0, ',') 1449 return name 1450} 1451 1452func (p *point) Name() []byte { 1453 return escape.Unescape(p.name()) 1454} 1455 1456// SetName updates the measurement name for the point. 1457func (p *point) SetName(name string) { 1458 p.cachedName = "" 1459 p.key = MakeKey([]byte(name), p.Tags()) 1460} 1461 1462// Time return the timestamp for the point. 1463func (p *point) Time() time.Time { 1464 return p.time 1465} 1466 1467// SetTime updates the timestamp for the point. 1468func (p *point) SetTime(t time.Time) { 1469 p.time = t 1470} 1471 1472// Round will round the timestamp of the point to the given duration. 1473func (p *point) Round(d time.Duration) { 1474 p.time = p.time.Round(d) 1475} 1476 1477// Tags returns the tag set for the point. 1478func (p *point) Tags() Tags { 1479 if p.cachedTags != nil { 1480 return p.cachedTags 1481 } 1482 p.cachedTags = parseTags(p.key, nil) 1483 return p.cachedTags 1484} 1485 1486func (p *point) ForEachTag(fn func(k, v []byte) bool) { 1487 walkTags(p.key, fn) 1488} 1489 1490func (p *point) HasTag(tag []byte) bool { 1491 if len(p.key) == 0 { 1492 return false 1493 } 1494 1495 var exists bool 1496 walkTags(p.key, func(key, value []byte) bool { 1497 if bytes.Equal(tag, key) { 1498 exists = true 1499 return false 1500 } 1501 return true 1502 }) 1503 1504 return exists 1505} 1506 1507func walkTags(buf []byte, fn func(key, value []byte) bool) { 1508 if len(buf) == 0 { 1509 return 1510 } 1511 1512 pos, name := scanTo(buf, 0, ',') 1513 1514 // it's an empty key, so there are no tags 1515 if len(name) == 0 { 1516 return 1517 } 1518 1519 hasEscape := bytes.IndexByte(buf, '\\') != -1 1520 i := pos + 1 1521 var key, value []byte 1522 for { 1523 if i >= len(buf) { 1524 break 1525 } 1526 i, key = scanTo(buf, i, '=') 1527 i, value = scanTagValue(buf, i+1) 1528 1529 if len(value) == 0 { 1530 continue 1531 } 1532 1533 if hasEscape { 1534 if !fn(unescapeTag(key), unescapeTag(value)) { 1535 return 1536 } 1537 } else { 1538 if !fn(key, value) { 1539 return 1540 } 1541 } 1542 1543 i++ 1544 } 1545} 1546 1547// walkFields walks each field key and value via fn. If fn returns false, the iteration 1548// is stopped. The values are the raw byte slices and not the converted types. 1549func walkFields(buf []byte, fn func(key, value []byte) bool) { 1550 var i int 1551 var key, val []byte 1552 for len(buf) > 0 { 1553 i, key = scanTo(buf, 0, '=') 1554 buf = buf[i+1:] 1555 i, val = scanFieldValue(buf, 0) 1556 buf = buf[i:] 1557 if !fn(key, val) { 1558 break 1559 } 1560 1561 // slice off comma 1562 if len(buf) > 0 { 1563 buf = buf[1:] 1564 } 1565 } 1566} 1567 1568// parseTags parses buf into the provided destination tags, returning destination 1569// Tags, which may have a different length and capacity. 1570func parseTags(buf []byte, dst Tags) Tags { 1571 if len(buf) == 0 { 1572 return nil 1573 } 1574 1575 n := bytes.Count(buf, []byte(",")) 1576 if cap(dst) < n { 1577 dst = make(Tags, n) 1578 } else { 1579 dst = dst[:n] 1580 } 1581 1582 // Ensure existing behaviour when point has no tags and nil slice passed in. 1583 if dst == nil { 1584 dst = Tags{} 1585 } 1586 1587 // Series keys can contain escaped commas, therefore the number of commas 1588 // in a series key only gives an estimation of the upper bound on the number 1589 // of tags. 1590 var i int 1591 walkTags(buf, func(key, value []byte) bool { 1592 dst[i].Key, dst[i].Value = key, value 1593 i++ 1594 return true 1595 }) 1596 return dst[:i] 1597} 1598 1599// MakeKey creates a key for a set of tags. 1600func MakeKey(name []byte, tags Tags) []byte { 1601 return AppendMakeKey(nil, name, tags) 1602} 1603 1604// AppendMakeKey appends the key derived from name and tags to dst and returns the extended buffer. 1605func AppendMakeKey(dst []byte, name []byte, tags Tags) []byte { 1606 // unescape the name and then re-escape it to avoid double escaping. 1607 // The key should always be stored in escaped form. 1608 dst = append(dst, EscapeMeasurement(unescapeMeasurement(name))...) 1609 dst = tags.AppendHashKey(dst) 1610 return dst 1611} 1612 1613// SetTags replaces the tags for the point. 1614func (p *point) SetTags(tags Tags) { 1615 p.key = MakeKey(p.Name(), tags) 1616 p.cachedTags = tags 1617} 1618 1619// AddTag adds or replaces a tag value for a point. 1620func (p *point) AddTag(key, value string) { 1621 tags := p.Tags() 1622 tags = append(tags, Tag{Key: []byte(key), Value: []byte(value)}) 1623 sort.Sort(tags) 1624 p.cachedTags = tags 1625 p.key = MakeKey(p.Name(), tags) 1626} 1627 1628// Fields returns the fields for the point. 1629func (p *point) Fields() (Fields, error) { 1630 if p.cachedFields != nil { 1631 return p.cachedFields, nil 1632 } 1633 cf, err := p.unmarshalBinary() 1634 if err != nil { 1635 return nil, err 1636 } 1637 p.cachedFields = cf 1638 return p.cachedFields, nil 1639} 1640 1641// SetPrecision will round a time to the specified precision. 1642func (p *point) SetPrecision(precision string) { 1643 switch precision { 1644 case "n": 1645 case "u": 1646 p.SetTime(p.Time().Truncate(time.Microsecond)) 1647 case "ms": 1648 p.SetTime(p.Time().Truncate(time.Millisecond)) 1649 case "s": 1650 p.SetTime(p.Time().Truncate(time.Second)) 1651 case "m": 1652 p.SetTime(p.Time().Truncate(time.Minute)) 1653 case "h": 1654 p.SetTime(p.Time().Truncate(time.Hour)) 1655 } 1656} 1657 1658// String returns the string representation of the point. 1659func (p *point) String() string { 1660 if p.Time().IsZero() { 1661 return string(p.Key()) + " " + string(p.fields) 1662 } 1663 return string(p.Key()) + " " + string(p.fields) + " " + strconv.FormatInt(p.UnixNano(), 10) 1664} 1665 1666// AppendString appends the string representation of the point to buf. 1667func (p *point) AppendString(buf []byte) []byte { 1668 buf = append(buf, p.key...) 1669 buf = append(buf, ' ') 1670 buf = append(buf, p.fields...) 1671 1672 if !p.time.IsZero() { 1673 buf = append(buf, ' ') 1674 buf = strconv.AppendInt(buf, p.UnixNano(), 10) 1675 } 1676 1677 return buf 1678} 1679 1680// StringSize returns the length of the string that would be returned by String(). 1681func (p *point) StringSize() int { 1682 size := len(p.key) + len(p.fields) + 1 1683 1684 if !p.time.IsZero() { 1685 digits := 1 // even "0" has one digit 1686 t := p.UnixNano() 1687 if t < 0 { 1688 // account for negative sign, then negate 1689 digits++ 1690 t = -t 1691 } 1692 for t > 9 { // already accounted for one digit 1693 digits++ 1694 t /= 10 1695 } 1696 size += digits + 1 // digits and a space 1697 } 1698 1699 return size 1700} 1701 1702// MarshalBinary returns a binary representation of the point. 1703func (p *point) MarshalBinary() ([]byte, error) { 1704 if len(p.fields) == 0 { 1705 return nil, ErrPointMustHaveAField 1706 } 1707 1708 tb, err := p.time.MarshalBinary() 1709 if err != nil { 1710 return nil, err 1711 } 1712 1713 b := make([]byte, 8+len(p.key)+len(p.fields)+len(tb)) 1714 i := 0 1715 1716 binary.BigEndian.PutUint32(b[i:], uint32(len(p.key))) 1717 i += 4 1718 1719 i += copy(b[i:], p.key) 1720 1721 binary.BigEndian.PutUint32(b[i:i+4], uint32(len(p.fields))) 1722 i += 4 1723 1724 i += copy(b[i:], p.fields) 1725 1726 copy(b[i:], tb) 1727 return b, nil 1728} 1729 1730// UnmarshalBinary decodes a binary representation of the point into a point struct. 1731func (p *point) UnmarshalBinary(b []byte) error { 1732 var n int 1733 1734 // Read key length. 1735 if len(b) < 4 { 1736 return io.ErrShortBuffer 1737 } 1738 n, b = int(binary.BigEndian.Uint32(b[:4])), b[4:] 1739 1740 // Read key. 1741 if len(b) < n { 1742 return io.ErrShortBuffer 1743 } 1744 p.key, b = b[:n], b[n:] 1745 1746 // Read fields length. 1747 if len(b) < 4 { 1748 return io.ErrShortBuffer 1749 } 1750 n, b = int(binary.BigEndian.Uint32(b[:4])), b[4:] 1751 1752 // Read fields. 1753 if len(b) < n { 1754 return io.ErrShortBuffer 1755 } 1756 p.fields, b = b[:n], b[n:] 1757 1758 // Read timestamp. 1759 return p.time.UnmarshalBinary(b) 1760} 1761 1762// PrecisionString returns a string representation of the point. If there 1763// is a timestamp associated with the point then it will be specified in the 1764// given unit. 1765func (p *point) PrecisionString(precision string) string { 1766 if p.Time().IsZero() { 1767 return fmt.Sprintf("%s %s", p.Key(), string(p.fields)) 1768 } 1769 return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields), 1770 p.UnixNano()/GetPrecisionMultiplier(precision)) 1771} 1772 1773// RoundedString returns a string representation of the point. If there 1774// is a timestamp associated with the point, then it will be rounded to the 1775// given duration. 1776func (p *point) RoundedString(d time.Duration) string { 1777 if p.Time().IsZero() { 1778 return fmt.Sprintf("%s %s", p.Key(), string(p.fields)) 1779 } 1780 return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields), 1781 p.time.Round(d).UnixNano()) 1782} 1783 1784func (p *point) unmarshalBinary() (Fields, error) { 1785 iter := p.FieldIterator() 1786 fields := make(Fields, 8) 1787 for iter.Next() { 1788 if len(iter.FieldKey()) == 0 { 1789 continue 1790 } 1791 switch iter.Type() { 1792 case Float: 1793 v, err := iter.FloatValue() 1794 if err != nil { 1795 return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) 1796 } 1797 fields[string(iter.FieldKey())] = v 1798 case Integer: 1799 v, err := iter.IntegerValue() 1800 if err != nil { 1801 return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) 1802 } 1803 fields[string(iter.FieldKey())] = v 1804 case Unsigned: 1805 v, err := iter.UnsignedValue() 1806 if err != nil { 1807 return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) 1808 } 1809 fields[string(iter.FieldKey())] = v 1810 case String: 1811 fields[string(iter.FieldKey())] = iter.StringValue() 1812 case Boolean: 1813 v, err := iter.BooleanValue() 1814 if err != nil { 1815 return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) 1816 } 1817 fields[string(iter.FieldKey())] = v 1818 } 1819 } 1820 return fields, nil 1821} 1822 1823// HashID returns a non-cryptographic checksum of the point's key. 1824func (p *point) HashID() uint64 { 1825 h := NewInlineFNV64a() 1826 h.Write(p.key) 1827 sum := h.Sum64() 1828 return sum 1829} 1830 1831// UnixNano returns the timestamp of the point as nanoseconds since Unix epoch. 1832func (p *point) UnixNano() int64 { 1833 return p.Time().UnixNano() 1834} 1835 1836// Split will attempt to return multiple points with the same timestamp whose 1837// string representations are no longer than size. Points with a single field or 1838// a point without a timestamp may exceed the requested size. 1839func (p *point) Split(size int) []Point { 1840 if p.time.IsZero() || p.StringSize() <= size { 1841 return []Point{p} 1842 } 1843 1844 // key string, timestamp string, spaces 1845 size -= len(p.key) + len(strconv.FormatInt(p.time.UnixNano(), 10)) + 2 1846 1847 var points []Point 1848 var start, cur int 1849 1850 for cur < len(p.fields) { 1851 end, _ := scanTo(p.fields, cur, '=') 1852 end, _ = scanFieldValue(p.fields, end+1) 1853 1854 if cur > start && end-start > size { 1855 points = append(points, &point{ 1856 key: p.key, 1857 time: p.time, 1858 fields: p.fields[start : cur-1], 1859 }) 1860 start = cur 1861 } 1862 1863 cur = end + 1 1864 } 1865 1866 points = append(points, &point{ 1867 key: p.key, 1868 time: p.time, 1869 fields: p.fields[start:], 1870 }) 1871 1872 return points 1873} 1874 1875// Tag represents a single key/value tag pair. 1876type Tag struct { 1877 Key []byte 1878 Value []byte 1879} 1880 1881// NewTag returns a new Tag. 1882func NewTag(key, value []byte) Tag { 1883 return Tag{ 1884 Key: key, 1885 Value: value, 1886 } 1887} 1888 1889// Size returns the size of the key and value. 1890func (t Tag) Size() int { return len(t.Key) + len(t.Value) } 1891 1892// Clone returns a shallow copy of Tag. 1893// 1894// Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. 1895// Use Clone to create a Tag with new byte slices that do not refer to the argument to ParsePointsWithPrecision. 1896func (t Tag) Clone() Tag { 1897 other := Tag{ 1898 Key: make([]byte, len(t.Key)), 1899 Value: make([]byte, len(t.Value)), 1900 } 1901 1902 copy(other.Key, t.Key) 1903 copy(other.Value, t.Value) 1904 1905 return other 1906} 1907 1908// String returns the string reprsentation of the tag. 1909func (t *Tag) String() string { 1910 var buf bytes.Buffer 1911 buf.WriteByte('{') 1912 buf.WriteString(string(t.Key)) 1913 buf.WriteByte(' ') 1914 buf.WriteString(string(t.Value)) 1915 buf.WriteByte('}') 1916 return buf.String() 1917} 1918 1919// Tags represents a sorted list of tags. 1920type Tags []Tag 1921 1922// NewTags returns a new Tags from a map. 1923func NewTags(m map[string]string) Tags { 1924 if len(m) == 0 { 1925 return nil 1926 } 1927 a := make(Tags, 0, len(m)) 1928 for k, v := range m { 1929 a = append(a, NewTag([]byte(k), []byte(v))) 1930 } 1931 sort.Sort(a) 1932 return a 1933} 1934 1935// Keys returns the list of keys for a tag set. 1936func (a Tags) Keys() []string { 1937 if len(a) == 0 { 1938 return nil 1939 } 1940 keys := make([]string, len(a)) 1941 for i, tag := range a { 1942 keys[i] = string(tag.Key) 1943 } 1944 return keys 1945} 1946 1947// Values returns the list of values for a tag set. 1948func (a Tags) Values() []string { 1949 if len(a) == 0 { 1950 return nil 1951 } 1952 values := make([]string, len(a)) 1953 for i, tag := range a { 1954 values[i] = string(tag.Value) 1955 } 1956 return values 1957} 1958 1959// String returns the string representation of the tags. 1960func (a Tags) String() string { 1961 var buf bytes.Buffer 1962 buf.WriteByte('[') 1963 for i := range a { 1964 buf.WriteString(a[i].String()) 1965 if i < len(a)-1 { 1966 buf.WriteByte(' ') 1967 } 1968 } 1969 buf.WriteByte(']') 1970 return buf.String() 1971} 1972 1973// Size returns the number of bytes needed to store all tags. Note, this is 1974// the number of bytes needed to store all keys and values and does not account 1975// for data structures or delimiters for example. 1976func (a Tags) Size() int { 1977 var total int 1978 for i := range a { 1979 total += a[i].Size() 1980 } 1981 return total 1982} 1983 1984// Clone returns a copy of the slice where the elements are a result of calling `Clone` on the original elements 1985// 1986// Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. 1987// Use Clone to create Tags with new byte slices that do not refer to the argument to ParsePointsWithPrecision. 1988func (a Tags) Clone() Tags { 1989 if len(a) == 0 { 1990 return nil 1991 } 1992 1993 others := make(Tags, len(a)) 1994 for i := range a { 1995 others[i] = a[i].Clone() 1996 } 1997 1998 return others 1999} 2000 2001func (a Tags) Len() int { return len(a) } 2002func (a Tags) Less(i, j int) bool { return bytes.Compare(a[i].Key, a[j].Key) == -1 } 2003func (a Tags) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 2004 2005// Equal returns true if a equals other. 2006func (a Tags) Equal(other Tags) bool { 2007 if len(a) != len(other) { 2008 return false 2009 } 2010 for i := range a { 2011 if !bytes.Equal(a[i].Key, other[i].Key) || !bytes.Equal(a[i].Value, other[i].Value) { 2012 return false 2013 } 2014 } 2015 return true 2016} 2017 2018// CompareTags returns -1 if a < b, 1 if a > b, and 0 if a == b. 2019func CompareTags(a, b Tags) int { 2020 // Compare each key & value until a mismatch. 2021 for i := 0; i < len(a) && i < len(b); i++ { 2022 if cmp := bytes.Compare(a[i].Key, b[i].Key); cmp != 0 { 2023 return cmp 2024 } 2025 if cmp := bytes.Compare(a[i].Value, b[i].Value); cmp != 0 { 2026 return cmp 2027 } 2028 } 2029 2030 // If all tags are equal up to this point then return shorter tagset. 2031 if len(a) < len(b) { 2032 return -1 2033 } else if len(a) > len(b) { 2034 return 1 2035 } 2036 2037 // All tags are equal. 2038 return 0 2039} 2040 2041// Get returns the value for a key. 2042func (a Tags) Get(key []byte) []byte { 2043 // OPTIMIZE: Use sort.Search if tagset is large. 2044 2045 for _, t := range a { 2046 if bytes.Equal(t.Key, key) { 2047 return t.Value 2048 } 2049 } 2050 return nil 2051} 2052 2053// GetString returns the string value for a string key. 2054func (a Tags) GetString(key string) string { 2055 return string(a.Get([]byte(key))) 2056} 2057 2058// Set sets the value for a key. 2059func (a *Tags) Set(key, value []byte) { 2060 for i, t := range *a { 2061 if bytes.Equal(t.Key, key) { 2062 (*a)[i].Value = value 2063 return 2064 } 2065 } 2066 *a = append(*a, Tag{Key: key, Value: value}) 2067 sort.Sort(*a) 2068} 2069 2070// SetString sets the string value for a string key. 2071func (a *Tags) SetString(key, value string) { 2072 a.Set([]byte(key), []byte(value)) 2073} 2074 2075// Delete removes a tag by key. 2076func (a *Tags) Delete(key []byte) { 2077 for i, t := range *a { 2078 if bytes.Equal(t.Key, key) { 2079 copy((*a)[i:], (*a)[i+1:]) 2080 (*a)[len(*a)-1] = Tag{} 2081 *a = (*a)[:len(*a)-1] 2082 return 2083 } 2084 } 2085} 2086 2087// Map returns a map representation of the tags. 2088func (a Tags) Map() map[string]string { 2089 m := make(map[string]string, len(a)) 2090 for _, t := range a { 2091 m[string(t.Key)] = string(t.Value) 2092 } 2093 return m 2094} 2095 2096// Merge merges the tags combining the two. If both define a tag with the 2097// same key, the merged value overwrites the old value. 2098// A new map is returned. 2099func (a Tags) Merge(other map[string]string) Tags { 2100 merged := make(map[string]string, len(a)+len(other)) 2101 for _, t := range a { 2102 merged[string(t.Key)] = string(t.Value) 2103 } 2104 for k, v := range other { 2105 merged[k] = v 2106 } 2107 return NewTags(merged) 2108} 2109 2110// HashKey hashes all of a tag's keys. 2111func (a Tags) HashKey() []byte { 2112 return a.AppendHashKey(nil) 2113} 2114 2115func (a Tags) needsEscape() bool { 2116 for i := range a { 2117 t := &a[i] 2118 for j := range tagEscapeCodes { 2119 c := &tagEscapeCodes[j] 2120 if bytes.IndexByte(t.Key, c.k[0]) != -1 || bytes.IndexByte(t.Value, c.k[0]) != -1 { 2121 return true 2122 } 2123 } 2124 } 2125 return false 2126} 2127 2128// AppendHashKey appends the result of hashing all of a tag's keys and values to dst and returns the extended buffer. 2129func (a Tags) AppendHashKey(dst []byte) []byte { 2130 // Empty maps marshal to empty bytes. 2131 if len(a) == 0 { 2132 return dst 2133 } 2134 2135 // Type invariant: Tags are sorted 2136 2137 sz := 0 2138 var escaped Tags 2139 if a.needsEscape() { 2140 var tmp [20]Tag 2141 if len(a) < len(tmp) { 2142 escaped = tmp[:len(a)] 2143 } else { 2144 escaped = make(Tags, len(a)) 2145 } 2146 2147 for i := range a { 2148 t := &a[i] 2149 nt := &escaped[i] 2150 nt.Key = escapeTag(t.Key) 2151 nt.Value = escapeTag(t.Value) 2152 sz += len(nt.Key) + len(nt.Value) 2153 } 2154 } else { 2155 sz = a.Size() 2156 escaped = a 2157 } 2158 2159 sz += len(escaped) + (len(escaped) * 2) // separators 2160 2161 // Generate marshaled bytes. 2162 if cap(dst)-len(dst) < sz { 2163 nd := make([]byte, len(dst), len(dst)+sz) 2164 copy(nd, dst) 2165 dst = nd 2166 } 2167 buf := dst[len(dst) : len(dst)+sz] 2168 idx := 0 2169 for i := range escaped { 2170 k := &escaped[i] 2171 if len(k.Value) == 0 { 2172 continue 2173 } 2174 buf[idx] = ',' 2175 idx++ 2176 copy(buf[idx:], k.Key) 2177 idx += len(k.Key) 2178 buf[idx] = '=' 2179 idx++ 2180 copy(buf[idx:], k.Value) 2181 idx += len(k.Value) 2182 } 2183 return dst[:len(dst)+idx] 2184} 2185 2186// CopyTags returns a shallow copy of tags. 2187func CopyTags(a Tags) Tags { 2188 other := make(Tags, len(a)) 2189 copy(other, a) 2190 return other 2191} 2192 2193// DeepCopyTags returns a deep copy of tags. 2194func DeepCopyTags(a Tags) Tags { 2195 // Calculate size of keys/values in bytes. 2196 var n int 2197 for _, t := range a { 2198 n += len(t.Key) + len(t.Value) 2199 } 2200 2201 // Build single allocation for all key/values. 2202 buf := make([]byte, n) 2203 2204 // Copy tags to new set. 2205 other := make(Tags, len(a)) 2206 for i, t := range a { 2207 copy(buf, t.Key) 2208 other[i].Key, buf = buf[:len(t.Key)], buf[len(t.Key):] 2209 2210 copy(buf, t.Value) 2211 other[i].Value, buf = buf[:len(t.Value)], buf[len(t.Value):] 2212 } 2213 2214 return other 2215} 2216 2217// Fields represents a mapping between a Point's field names and their 2218// values. 2219type Fields map[string]interface{} 2220 2221// FieldIterator retuns a FieldIterator that can be used to traverse the 2222// fields of a point without constructing the in-memory map. 2223func (p *point) FieldIterator() FieldIterator { 2224 p.Reset() 2225 return p 2226} 2227 2228type fieldIterator struct { 2229 start, end int 2230 key, keybuf []byte 2231 valueBuf []byte 2232 fieldType FieldType 2233} 2234 2235// Next indicates whether there any fields remaining. 2236func (p *point) Next() bool { 2237 p.it.start = p.it.end 2238 if p.it.start >= len(p.fields) { 2239 return false 2240 } 2241 2242 p.it.end, p.it.key = scanTo(p.fields, p.it.start, '=') 2243 if escape.IsEscaped(p.it.key) { 2244 p.it.keybuf = escape.AppendUnescaped(p.it.keybuf[:0], p.it.key) 2245 p.it.key = p.it.keybuf 2246 } 2247 2248 p.it.end, p.it.valueBuf = scanFieldValue(p.fields, p.it.end+1) 2249 p.it.end++ 2250 2251 if len(p.it.valueBuf) == 0 { 2252 p.it.fieldType = Empty 2253 return true 2254 } 2255 2256 c := p.it.valueBuf[0] 2257 2258 if c == '"' { 2259 p.it.fieldType = String 2260 return true 2261 } 2262 2263 if strings.IndexByte(`0123456789-.nNiIu`, c) >= 0 { 2264 if p.it.valueBuf[len(p.it.valueBuf)-1] == 'i' { 2265 p.it.fieldType = Integer 2266 p.it.valueBuf = p.it.valueBuf[:len(p.it.valueBuf)-1] 2267 } else if p.it.valueBuf[len(p.it.valueBuf)-1] == 'u' { 2268 p.it.fieldType = Unsigned 2269 p.it.valueBuf = p.it.valueBuf[:len(p.it.valueBuf)-1] 2270 } else { 2271 p.it.fieldType = Float 2272 } 2273 return true 2274 } 2275 2276 // to keep the same behavior that currently exists, default to boolean 2277 p.it.fieldType = Boolean 2278 return true 2279} 2280 2281// FieldKey returns the key of the current field. 2282func (p *point) FieldKey() []byte { 2283 return p.it.key 2284} 2285 2286// Type returns the FieldType of the current field. 2287func (p *point) Type() FieldType { 2288 return p.it.fieldType 2289} 2290 2291// StringValue returns the string value of the current field. 2292func (p *point) StringValue() string { 2293 return unescapeStringField(string(p.it.valueBuf[1 : len(p.it.valueBuf)-1])) 2294} 2295 2296// IntegerValue returns the integer value of the current field. 2297func (p *point) IntegerValue() (int64, error) { 2298 n, err := parseIntBytes(p.it.valueBuf, 10, 64) 2299 if err != nil { 2300 return 0, fmt.Errorf("unable to parse integer value %q: %v", p.it.valueBuf, err) 2301 } 2302 return n, nil 2303} 2304 2305// UnsignedValue returns the unsigned value of the current field. 2306func (p *point) UnsignedValue() (uint64, error) { 2307 n, err := parseUintBytes(p.it.valueBuf, 10, 64) 2308 if err != nil { 2309 return 0, fmt.Errorf("unable to parse unsigned value %q: %v", p.it.valueBuf, err) 2310 } 2311 return n, nil 2312} 2313 2314// BooleanValue returns the boolean value of the current field. 2315func (p *point) BooleanValue() (bool, error) { 2316 b, err := parseBoolBytes(p.it.valueBuf) 2317 if err != nil { 2318 return false, fmt.Errorf("unable to parse bool value %q: %v", p.it.valueBuf, err) 2319 } 2320 return b, nil 2321} 2322 2323// FloatValue returns the float value of the current field. 2324func (p *point) FloatValue() (float64, error) { 2325 f, err := parseFloatBytes(p.it.valueBuf, 64) 2326 if err != nil { 2327 return 0, fmt.Errorf("unable to parse floating point value %q: %v", p.it.valueBuf, err) 2328 } 2329 return f, nil 2330} 2331 2332// Reset resets the iterator to its initial state. 2333func (p *point) Reset() { 2334 p.it.fieldType = Empty 2335 p.it.key = nil 2336 p.it.valueBuf = nil 2337 p.it.start = 0 2338 p.it.end = 0 2339} 2340 2341// MarshalBinary encodes all the fields to their proper type and returns the binary 2342// represenation 2343// NOTE: uint64 is specifically not supported due to potential overflow when we decode 2344// again later to an int64 2345// NOTE2: uint is accepted, and may be 64 bits, and is for some reason accepted... 2346func (p Fields) MarshalBinary() []byte { 2347 var b []byte 2348 keys := make([]string, 0, len(p)) 2349 2350 for k := range p { 2351 keys = append(keys, k) 2352 } 2353 2354 // Not really necessary, can probably be removed. 2355 sort.Strings(keys) 2356 2357 for i, k := range keys { 2358 if i > 0 { 2359 b = append(b, ',') 2360 } 2361 b = appendField(b, k, p[k]) 2362 } 2363 2364 return b 2365} 2366 2367func appendField(b []byte, k string, v interface{}) []byte { 2368 b = append(b, []byte(escape.String(k))...) 2369 b = append(b, '=') 2370 2371 // check popular types first 2372 switch v := v.(type) { 2373 case float64: 2374 b = strconv.AppendFloat(b, v, 'f', -1, 64) 2375 case int64: 2376 b = strconv.AppendInt(b, v, 10) 2377 b = append(b, 'i') 2378 case string: 2379 b = append(b, '"') 2380 b = append(b, []byte(EscapeStringField(v))...) 2381 b = append(b, '"') 2382 case bool: 2383 b = strconv.AppendBool(b, v) 2384 case int32: 2385 b = strconv.AppendInt(b, int64(v), 10) 2386 b = append(b, 'i') 2387 case int16: 2388 b = strconv.AppendInt(b, int64(v), 10) 2389 b = append(b, 'i') 2390 case int8: 2391 b = strconv.AppendInt(b, int64(v), 10) 2392 b = append(b, 'i') 2393 case int: 2394 b = strconv.AppendInt(b, int64(v), 10) 2395 b = append(b, 'i') 2396 case uint64: 2397 b = strconv.AppendUint(b, v, 10) 2398 b = append(b, 'u') 2399 case uint32: 2400 b = strconv.AppendInt(b, int64(v), 10) 2401 b = append(b, 'i') 2402 case uint16: 2403 b = strconv.AppendInt(b, int64(v), 10) 2404 b = append(b, 'i') 2405 case uint8: 2406 b = strconv.AppendInt(b, int64(v), 10) 2407 b = append(b, 'i') 2408 case uint: 2409 // TODO: 'uint' should be converted to writing as an unsigned integer, 2410 // but we cannot since that would break backwards compatibility. 2411 b = strconv.AppendInt(b, int64(v), 10) 2412 b = append(b, 'i') 2413 case float32: 2414 b = strconv.AppendFloat(b, float64(v), 'f', -1, 32) 2415 case []byte: 2416 b = append(b, v...) 2417 case nil: 2418 // skip 2419 default: 2420 // Can't determine the type, so convert to string 2421 b = append(b, '"') 2422 b = append(b, []byte(EscapeStringField(fmt.Sprintf("%v", v)))...) 2423 b = append(b, '"') 2424 2425 } 2426 2427 return b 2428} 2429 2430// ValidKeyToken returns true if the token used for measurement, tag key, or tag 2431// value is a valid unicode string and only contains printable, non-replacement characters. 2432func ValidKeyToken(s string) bool { 2433 if !utf8.ValidString(s) { 2434 return false 2435 } 2436 for _, r := range s { 2437 if !unicode.IsPrint(r) || r == unicode.ReplacementChar { 2438 return false 2439 } 2440 } 2441 return true 2442} 2443 2444// ValidKeyTokens returns true if the measurement name and all tags are valid. 2445func ValidKeyTokens(name string, tags Tags) bool { 2446 if !ValidKeyToken(name) { 2447 return false 2448 } 2449 for _, tag := range tags { 2450 if !ValidKeyToken(string(tag.Key)) || !ValidKeyToken(string(tag.Value)) { 2451 return false 2452 } 2453 } 2454 return true 2455} 2456