appengine/search/search.go

// Copyright 2012 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package search // import "google.golang.org/appengine/search"

// TODO: let Put specify the document language: "en", "fr", etc. Also: order_id?? storage??
// TODO: Index.GetAll (or Iterator.GetAll)?
// TODO: struct <-> protobuf tests.
// TODO: enforce Python's MIN_NUMBER_VALUE and MIN_DATE (which would disallow a zero
// time.Time)? _MAXIMUM_STRING_LENGTH?

import (
	"errors"
	"fmt"
	"math"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"

	"google.golang.org/appengine"
	"google.golang.org/appengine/internal"
	pb "google.golang.org/appengine/internal/search"
)

const maxDocumentsPerPutDelete = 200

var (
	// ErrInvalidDocumentType is returned when methods like Put, Get or Next
	// are passed a dst or src argument of invalid type.
	ErrInvalidDocumentType = errors.New("search: invalid document type")

	// ErrNoSuchDocument is returned when no document was found for a given ID.
	ErrNoSuchDocument = errors.New("search: no such document")

	// ErrTooManyDocuments is returned when the user passes too many documents to
	// PutMulti or DeleteMulti.
	ErrTooManyDocuments = fmt.Errorf("search: too many documents given to put or delete (max is %d)", maxDocumentsPerPutDelete)
)

// Atom is a document field whose contents are indexed as a single indivisible
// string.
type Atom string

// HTML is a document field whose contents are indexed as HTML. Only text nodes
// are indexed: "foo<b>bar" will be treated as "foobar".
type HTML string

// validIndexNameOrDocID is the Go equivalent of Python's
// _ValidateVisiblePrintableAsciiNotReserved.
func validIndexNameOrDocID(s string) bool {
	if strings.HasPrefix(s, "!") {
		return false
	}
	for _, c := range s {
		if c < 0x21 || 0x7f <= c {
			return false
		}
	}
	return true
}

var (
	fieldNameRE = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`)
	languageRE  = regexp.MustCompile(`^[a-z]{2}$`)
)

// validFieldName is the Go equivalent of Python's _CheckFieldName. It checks
// the validity of both field and facet names.
func validFieldName(s string) bool {
	return len(s) <= 500 && fieldNameRE.MatchString(s)
}

// validDocRank checks that the ranks is in the range [0, 2^31).
func validDocRank(r int) bool {
	return 0 <= r && r <= (1<<31-1)
}

// validLanguage checks that a language looks like ISO 639-1.
func validLanguage(s string) bool {
	return languageRE.MatchString(s)
}

// validFloat checks that f is in the range [-2147483647, 2147483647].
func validFloat(f float64) bool {
	return -(1<<31-1) <= f && f <= (1<<31-1)
}

// Index is an index of documents.
type Index struct {
	spec pb.IndexSpec
}

// orderIDEpoch forms the basis for populating OrderId on documents.
var orderIDEpoch = time.Date(2011, 1, 1, 0, 0, 0, 0, time.UTC)

// Open opens the index with the given name. The index is created if it does
// not already exist.
//
// The name is a human-readable ASCII string. It must contain no whitespace
// characters and not start with "!".
func Open(name string) (*Index, error) {
	if !validIndexNameOrDocID(name) {
		return nil, fmt.Errorf("search: invalid index name %q", name)
	}
	return &Index{
		spec: pb.IndexSpec{
			Name: &name,
		},
	}, nil
}

// Put saves src to the index. If id is empty, a new ID is allocated by the
// service and returned. If id is not empty, any existing index entry for that
// ID is replaced.
//
// The ID is a human-readable ASCII string. It must contain no whitespace
// characters and not start with "!".
//
// src must be a non-nil struct pointer or implement the FieldLoadSaver
// interface.
func (x *Index) Put(c context.Context, id string, src interface{}) (string, error) {
	ids, err := x.PutMulti(c, []string{id}, []interface{}{src})
	if err != nil {
		return "", err
	}
	return ids[0], nil
}

// PutMulti is like Put, but is more efficient for adding multiple documents to
// the index at once.
//
// Up to 200 documents can be added at once. ErrTooManyDocuments is returned if
// you try to add more.
//
// ids can either be an empty slice (which means new IDs will be allocated for
// each of the documents added) or a slice the same size as srcs.
//
// The error may be an instance of appengine.MultiError, in which case it will
// be the same size as srcs and the individual errors inside will correspond
// with the items in srcs.
func (x *Index) PutMulti(c context.Context, ids []string, srcs []interface{}) ([]string, error) {
	if len(ids) != 0 && len(srcs) != len(ids) {
		return nil, fmt.Errorf("search: PutMulti expects ids and srcs slices of the same length")
	}
	if len(srcs) > maxDocumentsPerPutDelete {
		return nil, ErrTooManyDocuments
	}

	docs := make([]*pb.Document, len(srcs))
	for i, s := range srcs {
		var err error
		docs[i], err = saveDoc(s)
		if err != nil {
			return nil, err
		}

		if len(ids) != 0 && ids[i] != "" {
			if !validIndexNameOrDocID(ids[i]) {
				return nil, fmt.Errorf("search: invalid ID %q", ids[i])
			}
			docs[i].Id = proto.String(ids[i])
		}
	}

	// spec is modified by Call when applying the current Namespace, so copy it to
	// avoid retaining the namespace beyond the scope of the Call.
	spec := x.spec
	req := &pb.IndexDocumentRequest{
		Params: &pb.IndexDocumentParams{
			Document:  docs,
			IndexSpec: &spec,
		},
	}
	res := &pb.IndexDocumentResponse{}
	if err := internal.Call(c, "search", "IndexDocument", req, res); err != nil {
		return nil, err
	}
	multiErr, hasErr := make(appengine.MultiError, len(res.Status)), false
	for i, s := range res.Status {
		if s.GetCode() != pb.SearchServiceError_OK {
			multiErr[i] = fmt.Errorf("search: %s: %s", s.GetCode(), s.GetErrorDetail())
			hasErr = true
		}
	}
	if hasErr {
		return res.DocId, multiErr
	}

	if len(res.Status) != len(docs) || len(res.DocId) != len(docs) {
		return nil, fmt.Errorf("search: internal error: wrong number of results (%d Statuses, %d DocIDs, expected %d)",
			len(res.Status), len(res.DocId), len(docs))
	}
	return res.DocId, nil
}

// Get loads the document with the given ID into dst.
//
// The ID is a human-readable ASCII string. It must be non-empty, contain no
// whitespace characters and not start with "!".
//
// dst must be a non-nil struct pointer or implement the FieldLoadSaver
// interface.
//
// ErrFieldMismatch is returned when a field is to be loaded into a different
// type than the one it was stored from, or when a field is missing or
// unexported in the destination struct. ErrFieldMismatch is only returned if
// dst is a struct pointer. It is up to the callee to decide whether this error
// is fatal, recoverable, or ignorable.
func (x *Index) Get(c context.Context, id string, dst interface{}) error {
	if id == "" || !validIndexNameOrDocID(id) {
		return fmt.Errorf("search: invalid ID %q", id)
	}
	req := &pb.ListDocumentsRequest{
		Params: &pb.ListDocumentsParams{
			IndexSpec:  &x.spec,
			StartDocId: proto.String(id),
			Limit:      proto.Int32(1),
		},
	}
	res := &pb.ListDocumentsResponse{}
	if err := internal.Call(c, "search", "ListDocuments", req, res); err != nil {
		return err
	}
	if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
		return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
	}
	if len(res.Document) != 1 || res.Document[0].GetId() != id {
		return ErrNoSuchDocument
	}
	return loadDoc(dst, res.Document[0], nil)
}

// Delete deletes a document from the index.
func (x *Index) Delete(c context.Context, id string) error {
	return x.DeleteMulti(c, []string{id})
}

// DeleteMulti deletes multiple documents from the index.
//
// The returned error may be an instance of appengine.MultiError, in which case
// it will be the same size as srcs and the individual errors inside will
// correspond with the items in srcs.
func (x *Index) DeleteMulti(c context.Context, ids []string) error {
	if len(ids) > maxDocumentsPerPutDelete {
		return ErrTooManyDocuments
	}

	req := &pb.DeleteDocumentRequest{
		Params: &pb.DeleteDocumentParams{
			DocId:     ids,
			IndexSpec: &x.spec,
		},
	}
	res := &pb.DeleteDocumentResponse{}
	if err := internal.Call(c, "search", "DeleteDocument", req, res); err != nil {
		return err
	}
	if len(res.Status) != len(ids) {
		return fmt.Errorf("search: internal error: wrong number of results (%d, expected %d)",
			len(res.Status), len(ids))
	}
	multiErr, hasErr := make(appengine.MultiError, len(ids)), false
	for i, s := range res.Status {
		if s.GetCode() != pb.SearchServiceError_OK {
			multiErr[i] = fmt.Errorf("search: %s: %s", s.GetCode(), s.GetErrorDetail())
			hasErr = true
		}
	}
	if hasErr {
		return multiErr
	}
	return nil
}

// List lists all of the documents in an index. The documents are returned in
// increasing ID order.
func (x *Index) List(c context.Context, opts *ListOptions) *Iterator {
	t := &Iterator{
		c:             c,
		index:         x,
		count:         -1,
		listInclusive: true,
		more:          moreList,
	}
	if opts != nil {
		t.listStartID = opts.StartID
		t.limit = opts.Limit
		t.idsOnly = opts.IDsOnly
	}
	return t
}

func moreList(t *Iterator) error {
	req := &pb.ListDocumentsRequest{
		Params: &pb.ListDocumentsParams{
			IndexSpec: &t.index.spec,
		},
	}
	if t.listStartID != "" {
		req.Params.StartDocId = &t.listStartID
		req.Params.IncludeStartDoc = &t.listInclusive
	}
	if t.limit > 0 {
		req.Params.Limit = proto.Int32(int32(t.limit))
	}
	if t.idsOnly {
		req.Params.KeysOnly = &t.idsOnly
	}

	res := &pb.ListDocumentsResponse{}
	if err := internal.Call(t.c, "search", "ListDocuments", req, res); err != nil {
		return err
	}
	if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
		return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
	}
	t.listRes = res.Document
	t.listStartID, t.listInclusive, t.more = "", false, nil
	if len(res.Document) != 0 && t.limit <= 0 {
		if id := res.Document[len(res.Document)-1].GetId(); id != "" {
			t.listStartID, t.more = id, moreList
		}
	}
	return nil
}

// ListOptions are the options for listing documents in an index. Passing a nil
// *ListOptions is equivalent to using the default values.
type ListOptions struct {
	// StartID is the inclusive lower bound for the ID of the returned
	// documents. The zero value means all documents will be returned.
	StartID string

	// Limit is the maximum number of documents to return. The zero value
	// indicates no limit.
	Limit int

	// IDsOnly indicates that only document IDs should be returned for the list
	// operation; no document fields are populated.
	IDsOnly bool
}

// Search searches the index for the given query.
func (x *Index) Search(c context.Context, query string, opts *SearchOptions) *Iterator {
	t := &Iterator{
		c:           c,
		index:       x,
		searchQuery: query,
		more:        moreSearch,
	}
	if opts != nil {
		if opts.Cursor != "" {
			if opts.Offset != 0 {
				return errIter("at most one of Cursor and Offset may be specified")
			}
			t.searchCursor = proto.String(string(opts.Cursor))
		}
		t.limit = opts.Limit
		t.fields = opts.Fields
		t.idsOnly = opts.IDsOnly
		t.sort = opts.Sort
		t.exprs = opts.Expressions
		t.refinements = opts.Refinements
		t.facetOpts = opts.Facets
		t.searchOffset = opts.Offset
		t.countAccuracy = opts.CountAccuracy
	}
	return t
}

func moreSearch(t *Iterator) error {
	// We use per-result (rather than single/per-page) cursors since this
	// lets us return a Cursor for every iterator document. The two cursor
	// types are largely interchangeable: a page cursor is the same as the
	// last per-result cursor in a given search response.
	req := &pb.SearchRequest{
		Params: &pb.SearchParams{
			IndexSpec:  &t.index.spec,
			Query:      &t.searchQuery,
			Cursor:     t.searchCursor,
			CursorType: pb.SearchParams_PER_RESULT.Enum(),
			FieldSpec: &pb.FieldSpec{
				Name: t.fields,
			},
		},
	}
	if t.limit > 0 {
		req.Params.Limit = proto.Int32(int32(t.limit))
	}
	if t.searchOffset > 0 {
		req.Params.Offset = proto.Int32(int32(t.searchOffset))
		t.searchOffset = 0
	}
	if t.countAccuracy > 0 {
		req.Params.MatchedCountAccuracy = proto.Int32(int32(t.countAccuracy))
	}
	if t.idsOnly {
		req.Params.KeysOnly = &t.idsOnly
	}
	if t.sort != nil {
		if err := sortToProto(t.sort, req.Params); err != nil {
			return err
		}
	}
	if t.refinements != nil {
		if err := refinementsToProto(t.refinements, req.Params); err != nil {
			return err
		}
	}
	for _, e := range t.exprs {
		req.Params.FieldSpec.Expression = append(req.Params.FieldSpec.Expression, &pb.FieldSpec_Expression{
			Name:       proto.String(e.Name),
			Expression: proto.String(e.Expr),
		})
	}
	for _, f := range t.facetOpts {
		if err := f.setParams(req.Params); err != nil {
			return fmt.Errorf("bad FacetSearchOption: %v", err)
		}
	}
	// Don't repeat facet search.
	t.facetOpts = nil

	res := &pb.SearchResponse{}
	if err := internal.Call(t.c, "search", "Search", req, res); err != nil {
		return err
	}
	if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
		return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
	}
	t.searchRes = res.Result
	if len(res.FacetResult) > 0 {
		t.facetRes = res.FacetResult
	}
	t.count = int(*res.MatchedCount)
	if t.limit > 0 {
		t.more = nil
	} else {
		t.more = moreSearch
	}
	return nil
}

// SearchOptions are the options for searching an index. Passing a nil
// *SearchOptions is equivalent to using the default values.
type SearchOptions struct {
	// Limit is the maximum number of documents to return. The zero value
	// indicates no limit.
	Limit int

	// IDsOnly indicates that only document IDs should be returned for the search
	// operation; no document fields are populated.
	IDsOnly bool

	// Sort controls the ordering of search results.
	Sort *SortOptions

	// Fields specifies which document fields to include in the results. If omitted,
	// all document fields are returned. No more than 100 fields may be specified.
	Fields []string

	// Expressions specifies additional computed fields to add to each returned
	// document.
	Expressions []FieldExpression

	// Facets controls what facet information is returned for these search results.
	// If no options are specified, no facet results will be returned.
	Facets []FacetSearchOption

	// Refinements filters the returned documents by requiring them to contain facets
	// with specific values. Refinements are applied in conjunction for facets with
	// different names, and in disjunction otherwise.
	Refinements []Facet

	// Cursor causes the results to commence with the first document after
	// the document associated with the cursor.
	Cursor Cursor

	// Offset specifies the number of documents to skip over before returning results.
	// When specified, Cursor must be nil.
	Offset int

	// CountAccuracy specifies the maximum result count that can be expected to
	// be accurate. If zero, the count accuracy defaults to 20.
	CountAccuracy int
}

// Cursor represents an iterator's position.
//
// The string value of a cursor is web-safe. It can be saved and restored
// for later use.
type Cursor string

// FieldExpression defines a custom expression to evaluate for each result.
type FieldExpression struct {
	// Name is the name to use for the computed field.
	Name string

	// Expr is evaluated to provide a custom content snippet for each document.
	// See https://cloud.google.com/appengine/docs/standard/go/search/options for
	// the supported expression syntax.
	Expr string
}

// FacetSearchOption controls what facet information is returned in search results.
type FacetSearchOption interface {
	setParams(*pb.SearchParams) error
}

// AutoFacetDiscovery returns a FacetSearchOption which enables automatic facet
// discovery for the search. Automatic facet discovery looks for the facets
// which appear the most often in the aggregate in the matched documents.
//
// The maximum number of facets returned is controlled by facetLimit, and the
// maximum number of values per facet by facetLimit. A limit of zero indicates
// a default limit should be used.
func AutoFacetDiscovery(facetLimit, valueLimit int) FacetSearchOption {
	return &autoFacetOpt{facetLimit, valueLimit}
}

type autoFacetOpt struct {
	facetLimit, valueLimit int
}

const defaultAutoFacetLimit = 10 // As per python runtime search.py.

func (o *autoFacetOpt) setParams(params *pb.SearchParams) error {
	lim := int32(o.facetLimit)
	if lim == 0 {
		lim = defaultAutoFacetLimit
	}
	params.AutoDiscoverFacetCount = &lim
	if o.valueLimit > 0 {
		params.FacetAutoDetectParam = &pb.FacetAutoDetectParam{
			ValueLimit: proto.Int32(int32(o.valueLimit)),
		}
	}
	return nil
}

// FacetDiscovery returns a FacetSearchOption which selects a facet to be
// returned with the search results. By default, the most frequently
// occurring values for that facet will be returned. However, you can also
// specify a list of particular Atoms or specific Ranges to return.
func FacetDiscovery(name string, value ...interface{}) FacetSearchOption {
	return &facetOpt{name, value}
}

type facetOpt struct {
	name   string
	values []interface{}
}

func (o *facetOpt) setParams(params *pb.SearchParams) error {
	req := &pb.FacetRequest{Name: &o.name}
	params.IncludeFacet = append(params.IncludeFacet, req)
	if len(o.values) == 0 {
		return nil
	}
	vtype := reflect.TypeOf(o.values[0])
	reqParam := &pb.FacetRequestParam{}
	for _, v := range o.values {
		if reflect.TypeOf(v) != vtype {
			return errors.New("values must all be Atom, or must all be Range")
		}
		switch v := v.(type) {
		case Atom:
			reqParam.ValueConstraint = append(reqParam.ValueConstraint, string(v))
		case Range:
			rng, err := rangeToProto(v)
			if err != nil {
				return fmt.Errorf("invalid range: %v", err)
			}
			reqParam.Range = append(reqParam.Range, rng)
		default:
			return fmt.Errorf("unsupported value type %T", v)
		}
	}
	req.Params = reqParam
	return nil
}

// FacetDocumentDepth returns a FacetSearchOption which controls the number of
// documents to be evaluated with preparing facet results.
func FacetDocumentDepth(depth int) FacetSearchOption {
	return facetDepthOpt(depth)
}

type facetDepthOpt int

func (o facetDepthOpt) setParams(params *pb.SearchParams) error {
	params.FacetDepth = proto.Int32(int32(o))
	return nil
}

// FacetResult represents the number of times a particular facet and value
// appeared in the documents matching a search request.
type FacetResult struct {
	Facet

	// Count is the number of times this specific facet and value appeared in the
	// matching documents.
	Count int
}

// Range represents a numeric range with inclusive start and exclusive end.
// Start may be specified as math.Inf(-1) to indicate there is no minimum
// value, and End may similarly be specified as math.Inf(1); at least one of
// Start or End must be a finite number.
type Range struct {
	Start, End float64
}

var (
	negInf = math.Inf(-1)
	posInf = math.Inf(1)
)

// AtLeast returns a Range matching any value greater than, or equal to, min.
func AtLeast(min float64) Range {
	return Range{Start: min, End: posInf}
}

// LessThan returns a Range matching any value less than max.
func LessThan(max float64) Range {
	return Range{Start: negInf, End: max}
}

// SortOptions control the ordering and scoring of search results.
type SortOptions struct {
	// Expressions is a slice of expressions representing a multi-dimensional
	// sort.
	Expressions []SortExpression

	// Scorer, when specified, will cause the documents to be scored according to
	// search term frequency.
	Scorer Scorer

	// Limit is the maximum number of objects to score and/or sort. Limit cannot
	// be more than 10,000. The zero value indicates a default limit.
	Limit int
}

// SortExpression defines a single dimension for sorting a document.
type SortExpression struct {
	// Expr is evaluated to provide a sorting value for each document.
	// See https://cloud.google.com/appengine/docs/standard/go/search/options for
	// the supported expression syntax.
	Expr string

	// Reverse causes the documents to be sorted in ascending order.
	Reverse bool

	// The default value to use when no field is present or the expresion
	// cannot be calculated for a document. For text sorts, Default must
	// be of type string; for numeric sorts, float64.
	Default interface{}
}

// A Scorer defines how a document is scored.
type Scorer interface {
	toProto(*pb.ScorerSpec)
}

type enumScorer struct {
	enum pb.ScorerSpec_Scorer
}

func (e enumScorer) toProto(spec *pb.ScorerSpec) {
	spec.Scorer = e.enum.Enum()
}

var (
	// MatchScorer assigns a score based on term frequency in a document.
	MatchScorer Scorer = enumScorer{pb.ScorerSpec_MATCH_SCORER}

	// RescoringMatchScorer assigns a score based on the quality of the query
	// match. It is similar to a MatchScorer but uses a more complex scoring
	// algorithm based on match term frequency and other factors like field type.
	// Please be aware that this algorithm is continually refined and can change
	// over time without notice. This means that the ordering of search results
	// that use this scorer can also change without notice.
	RescoringMatchScorer Scorer = enumScorer{pb.ScorerSpec_RESCORING_MATCH_SCORER}
)

func sortToProto(sort *SortOptions, params *pb.SearchParams) error {
	for _, e := range sort.Expressions {
		spec := &pb.SortSpec{
			SortExpression: proto.String(e.Expr),
		}
		if e.Reverse {
			spec.SortDescending = proto.Bool(false)
		}
		if e.Default != nil {
			switch d := e.Default.(type) {
			case float64:
				spec.DefaultValueNumeric = &d
			case string:
				spec.DefaultValueText = &d
			default:
				return fmt.Errorf("search: invalid Default type %T for expression %q", d, e.Expr)
			}
		}
		params.SortSpec = append(params.SortSpec, spec)
	}

	spec := &pb.ScorerSpec{}
	if sort.Limit > 0 {
		spec.Limit = proto.Int32(int32(sort.Limit))
		params.ScorerSpec = spec
	}
	if sort.Scorer != nil {
		sort.Scorer.toProto(spec)
		params.ScorerSpec = spec
	}

	return nil
}

func refinementsToProto(refinements []Facet, params *pb.SearchParams) error {
	for _, r := range refinements {
		ref := &pb.FacetRefinement{
			Name: proto.String(r.Name),
		}
		switch v := r.Value.(type) {
		case Atom:
			ref.Value = proto.String(string(v))
		case Range:
			rng, err := rangeToProto(v)
			if err != nil {
				return fmt.Errorf("search: refinement for facet %q: %v", r.Name, err)
			}
			// Unfortunately there are two identical messages for identify Facet ranges.
			ref.Range = &pb.FacetRefinement_Range{Start: rng.Start, End: rng.End}
		default:
			return fmt.Errorf("search: unsupported refinement for facet %q of type %T", r.Name, v)
		}
		params.FacetRefinement = append(params.FacetRefinement, ref)
	}
	return nil
}

func rangeToProto(r Range) (*pb.FacetRange, error) {
	rng := &pb.FacetRange{}
	if r.Start != negInf {
		if !validFloat(r.Start) {
			return nil, errors.New("invalid value for Start")
		}
		rng.Start = proto.String(strconv.FormatFloat(r.Start, 'e', -1, 64))
	} else if r.End == posInf {
		return nil, errors.New("either Start or End must be finite")
	}
	if r.End != posInf {
		if !validFloat(r.End) {
			return nil, errors.New("invalid value for End")
		}
		rng.End = proto.String(strconv.FormatFloat(r.End, 'e', -1, 64))
	}
	return rng, nil
}

func protoToRange(rng *pb.FacetRefinement_Range) Range {
	r := Range{Start: negInf, End: posInf}
	if x, err := strconv.ParseFloat(rng.GetStart(), 64); err != nil {
		r.Start = x
	}
	if x, err := strconv.ParseFloat(rng.GetEnd(), 64); err != nil {
		r.End = x
	}
	return r
}

// Iterator is the result of searching an index for a query or listing an
// index.
type Iterator struct {
	c     context.Context
	index *Index
	err   error

	listRes       []*pb.Document
	listStartID   string
	listInclusive bool

	searchRes    []*pb.SearchResult
	facetRes     []*pb.FacetResult
	searchQuery  string
	searchCursor *string
	searchOffset int
	sort         *SortOptions

	fields      []string
	exprs       []FieldExpression
	refinements []Facet
	facetOpts   []FacetSearchOption

	more func(*Iterator) error

	count         int
	countAccuracy int
	limit         int // items left to return; 0 for unlimited.
	idsOnly       bool
}

// errIter returns an iterator that only returns the given error.
func errIter(err string) *Iterator {
	return &Iterator{
		err: errors.New(err),
	}
}

// Done is returned when a query iteration has completed.
var Done = errors.New("search: query has no more results")

// Count returns an approximation of the number of documents matched by the
// query. It is only valid to call for iterators returned by Search.
func (t *Iterator) Count() int { return t.count }

// fetchMore retrieves more results, if there are no errors or pending results.
func (t *Iterator) fetchMore() {
	if t.err == nil && len(t.listRes)+len(t.searchRes) == 0 && t.more != nil {
		t.err = t.more(t)
	}
}

// Next returns the ID of the next result. When there are no more results,
// Done is returned as the error.
//
// dst must be a non-nil struct pointer, implement the FieldLoadSaver
// interface, or be a nil interface value. If a non-nil dst is provided, it
// will be filled with the indexed fields. dst is ignored if this iterator was
// created with an IDsOnly option.
func (t *Iterator) Next(dst interface{}) (string, error) {
	t.fetchMore()
	if t.err != nil {
		return "", t.err
	}

	var doc *pb.Document
	var exprs []*pb.Field
	switch {
	case len(t.listRes) != 0:
		doc = t.listRes[0]
		t.listRes = t.listRes[1:]
	case len(t.searchRes) != 0:
		doc = t.searchRes[0].Document
		exprs = t.searchRes[0].Expression
		t.searchCursor = t.searchRes[0].Cursor
		t.searchRes = t.searchRes[1:]
	default:
		return "", Done
	}
	if doc == nil {
		return "", errors.New("search: internal error: no document returned")
	}
	if !t.idsOnly && dst != nil {
		if err := loadDoc(dst, doc, exprs); err != nil {
			return "", err
		}
	}
	return doc.GetId(), nil
}

// Cursor returns the cursor associated with the current document (that is,
// the document most recently returned by a call to Next).
//
// Passing this cursor in a future call to Search will cause those results
// to commence with the first document after the current document.
func (t *Iterator) Cursor() Cursor {
	if t.searchCursor == nil {
		return ""
	}
	return Cursor(*t.searchCursor)
}

// Facets returns the facets found within the search results, if any facets
// were requested in the SearchOptions.
func (t *Iterator) Facets() ([][]FacetResult, error) {
	t.fetchMore()
	if t.err != nil && t.err != Done {
		return nil, t.err
	}

	var facets [][]FacetResult
	for _, f := range t.facetRes {
		fres := make([]FacetResult, 0, len(f.Value))
		for _, v := range f.Value {
			ref := v.Refinement
			facet := FacetResult{
				Facet: Facet{Name: ref.GetName()},
				Count: int(v.GetCount()),
			}
			if ref.Value != nil {
				facet.Value = Atom(*ref.Value)
			} else {
				facet.Value = protoToRange(ref.Range)
			}
			fres = append(fres, facet)
		}
		facets = append(facets, fres)
	}
	return facets, nil
}

// saveDoc converts from a struct pointer or
// FieldLoadSaver/FieldMetadataLoadSaver to the Document protobuf.
func saveDoc(src interface{}) (*pb.Document, error) {
	var err error
	var fields []Field
	var meta *DocumentMetadata
	switch x := src.(type) {
	case FieldLoadSaver:
		fields, meta, err = x.Save()
	default:
		fields, meta, err = saveStructWithMeta(src)
	}
	if err != nil {
		return nil, err
	}

	fieldsProto, err := fieldsToProto(fields)
	if err != nil {
		return nil, err
	}
	d := &pb.Document{
		Field:         fieldsProto,
		OrderId:       proto.Int32(int32(time.Since(orderIDEpoch).Seconds())),
		OrderIdSource: pb.Document_DEFAULTED.Enum(),
	}
	if meta != nil {
		if meta.Rank != 0 {
			if !validDocRank(meta.Rank) {
				return nil, fmt.Errorf("search: invalid rank %d, must be [0, 2^31)", meta.Rank)
			}
			*d.OrderId = int32(meta.Rank)
			d.OrderIdSource = pb.Document_SUPPLIED.Enum()
		}
		if len(meta.Facets) > 0 {
			facets, err := facetsToProto(meta.Facets)
			if err != nil {
				return nil, err
			}
			d.Facet = facets
		}
	}
	return d, nil
}

func fieldsToProto(src []Field) ([]*pb.Field, error) {
	// Maps to catch duplicate time or numeric fields.
	timeFields, numericFields := make(map[string]bool), make(map[string]bool)
	dst := make([]*pb.Field, 0, len(src))
	for _, f := range src {
		if !validFieldName(f.Name) {
			return nil, fmt.Errorf("search: invalid field name %q", f.Name)
		}
		fieldValue := &pb.FieldValue{}
		switch x := f.Value.(type) {
		case string:
			fieldValue.Type = pb.FieldValue_TEXT.Enum()
			fieldValue.StringValue = proto.String(x)
		case Atom:
			fieldValue.Type = pb.FieldValue_ATOM.Enum()
			fieldValue.StringValue = proto.String(string(x))
		case HTML:
			fieldValue.Type = pb.FieldValue_HTML.Enum()
			fieldValue.StringValue = proto.String(string(x))
		case time.Time:
			if timeFields[f.Name] {
				return nil, fmt.Errorf("search: duplicate time field %q", f.Name)
			}
			timeFields[f.Name] = true
			fieldValue.Type = pb.FieldValue_DATE.Enum()
			fieldValue.StringValue = proto.String(strconv.FormatInt(x.UnixNano()/1e6, 10))
		case float64:
			if numericFields[f.Name] {
				return nil, fmt.Errorf("search: duplicate numeric field %q", f.Name)
			}
			if !validFloat(x) {
				return nil, fmt.Errorf("search: numeric field %q with invalid value %f", f.Name, x)
			}
			numericFields[f.Name] = true
			fieldValue.Type = pb.FieldValue_NUMBER.Enum()
			fieldValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64))
		case appengine.GeoPoint:
			if !x.Valid() {
				return nil, fmt.Errorf(
					"search: GeoPoint field %q with invalid value %v",
					f.Name, x)
			}
			fieldValue.Type = pb.FieldValue_GEO.Enum()
			fieldValue.Geo = &pb.FieldValue_Geo{
				Lat: proto.Float64(x.Lat),
				Lng: proto.Float64(x.Lng),
			}
		default:
			return nil, fmt.Errorf("search: unsupported field type: %v", reflect.TypeOf(f.Value))
		}
		if f.Language != "" {
			switch f.Value.(type) {
			case string, HTML:
				if !validLanguage(f.Language) {
					return nil, fmt.Errorf("search: invalid language for field %q: %q", f.Name, f.Language)
				}
				fieldValue.Language = proto.String(f.Language)
			default:
				return nil, fmt.Errorf("search: setting language not supported for field %q of type %T", f.Name, f.Value)
			}
		}
		if p := fieldValue.StringValue; p != nil && !utf8.ValidString(*p) {
			return nil, fmt.Errorf("search: %q field is invalid UTF-8: %q", f.Name, *p)
		}
		dst = append(dst, &pb.Field{
			Name:  proto.String(f.Name),
			Value: fieldValue,
		})
	}
	return dst, nil
}

func facetsToProto(src []Facet) ([]*pb.Facet, error) {
	dst := make([]*pb.Facet, 0, len(src))
	for _, f := range src {
		if !validFieldName(f.Name) {
			return nil, fmt.Errorf("search: invalid facet name %q", f.Name)
		}
		facetValue := &pb.FacetValue{}
		switch x := f.Value.(type) {
		case Atom:
			if !utf8.ValidString(string(x)) {
				return nil, fmt.Errorf("search: %q facet is invalid UTF-8: %q", f.Name, x)
			}
			facetValue.Type = pb.FacetValue_ATOM.Enum()
			facetValue.StringValue = proto.String(string(x))
		case float64:
			if !validFloat(x) {
				return nil, fmt.Errorf("search: numeric facet %q with invalid value %f", f.Name, x)
			}
			facetValue.Type = pb.FacetValue_NUMBER.Enum()
			facetValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64))
		default:
			return nil, fmt.Errorf("search: unsupported facet type: %v", reflect.TypeOf(f.Value))
		}
		dst = append(dst, &pb.Facet{
			Name:  proto.String(f.Name),
			Value: facetValue,
		})
	}
	return dst, nil
}

// loadDoc converts from protobufs to a struct pointer or
// FieldLoadSaver/FieldMetadataLoadSaver. The src param provides the document's
// stored fields and facets, and any document metadata.  An additional slice of
// fields, exprs, may optionally be provided to contain any derived expressions
// requested by the developer.
func loadDoc(dst interface{}, src *pb.Document, exprs []*pb.Field) (err error) {
	fields, err := protoToFields(src.Field)
	if err != nil {
		return err
	}
	facets, err := protoToFacets(src.Facet)
	if err != nil {
		return err
	}
	if len(exprs) > 0 {
		exprFields, err := protoToFields(exprs)
		if err != nil {
			return err
		}
		// Mark each field as derived.
		for i := range exprFields {
			exprFields[i].Derived = true
		}
		fields = append(fields, exprFields...)
	}
	meta := &DocumentMetadata{
		Rank:   int(src.GetOrderId()),
		Facets: facets,
	}
	switch x := dst.(type) {
	case FieldLoadSaver:
		return x.Load(fields, meta)
	default:
		return loadStructWithMeta(dst, fields, meta)
	}
}

func protoToFields(fields []*pb.Field) ([]Field, error) {
	dst := make([]Field, 0, len(fields))
	for _, field := range fields {
		fieldValue := field.GetValue()
		f := Field{
			Name: field.GetName(),
		}
		switch fieldValue.GetType() {
		case pb.FieldValue_TEXT:
			f.Value = fieldValue.GetStringValue()
			f.Language = fieldValue.GetLanguage()
		case pb.FieldValue_ATOM:
			f.Value = Atom(fieldValue.GetStringValue())
		case pb.FieldValue_HTML:
			f.Value = HTML(fieldValue.GetStringValue())
			f.Language = fieldValue.GetLanguage()
		case pb.FieldValue_DATE:
			sv := fieldValue.GetStringValue()
			millis, err := strconv.ParseInt(sv, 10, 64)
			if err != nil {
				return nil, fmt.Errorf("search: internal error: bad time.Time encoding %q: %v", sv, err)
			}
			f.Value = time.Unix(0, millis*1e6)
		case pb.FieldValue_NUMBER:
			sv := fieldValue.GetStringValue()
			x, err := strconv.ParseFloat(sv, 64)
			if err != nil {
				return nil, err
			}
			f.Value = x
		case pb.FieldValue_GEO:
			geoValue := fieldValue.GetGeo()
			geoPoint := appengine.GeoPoint{geoValue.GetLat(), geoValue.GetLng()}
			if !geoPoint.Valid() {
				return nil, fmt.Errorf("search: internal error: invalid GeoPoint encoding: %v", geoPoint)
			}
			f.Value = geoPoint
		default:
			return nil, fmt.Errorf("search: internal error: unknown data type %s", fieldValue.GetType())
		}
		dst = append(dst, f)
	}
	return dst, nil
}

func protoToFacets(facets []*pb.Facet) ([]Facet, error) {
	if len(facets) == 0 {
		return nil, nil
	}
	dst := make([]Facet, 0, len(facets))
	for _, facet := range facets {
		facetValue := facet.GetValue()
		f := Facet{
			Name: facet.GetName(),
		}
		switch facetValue.GetType() {
		case pb.FacetValue_ATOM:
			f.Value = Atom(facetValue.GetStringValue())
		case pb.FacetValue_NUMBER:
			sv := facetValue.GetStringValue()
			x, err := strconv.ParseFloat(sv, 64)
			if err != nil {
				return nil, err
			}
			f.Value = x
		default:
			return nil, fmt.Errorf("search: internal error: unknown data type %s", facetValue.GetType())
		}
		dst = append(dst, f)
	}
	return dst, nil
}

func namespaceMod(m proto.Message, namespace string) {
	set := func(s **string) {
		if *s == nil {
			*s = &namespace
		}
	}
	switch m := m.(type) {
	case *pb.IndexDocumentRequest:
		set(&m.Params.IndexSpec.Namespace)
	case *pb.ListDocumentsRequest:
		set(&m.Params.IndexSpec.Namespace)
	case *pb.DeleteDocumentRequest:
		set(&m.Params.IndexSpec.Namespace)
	case *pb.SearchRequest:
		set(&m.Params.IndexSpec.Namespace)
	}
}

func init() {
	internal.RegisterErrorCodeMap("search", pb.SearchServiceError_ErrorCode_name)
	internal.NamespaceMods["search"] = namespaceMod
}