1// Copyright 2015 The etcd Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package rafthttp 16 17import ( 18 "bytes" 19 "context" 20 "io" 21 "io/ioutil" 22 "net/http" 23 "time" 24 25 "github.com/coreos/etcd/pkg/httputil" 26 pioutil "github.com/coreos/etcd/pkg/ioutil" 27 "github.com/coreos/etcd/pkg/types" 28 "github.com/coreos/etcd/raft" 29 "github.com/coreos/etcd/snap" 30) 31 32var ( 33 // timeout for reading snapshot response body 34 snapResponseReadTimeout = 5 * time.Second 35) 36 37type snapshotSender struct { 38 from, to types.ID 39 cid types.ID 40 41 tr *Transport 42 picker *urlPicker 43 status *peerStatus 44 r Raft 45 errorc chan error 46 47 stopc chan struct{} 48} 49 50func newSnapshotSender(tr *Transport, picker *urlPicker, to types.ID, status *peerStatus) *snapshotSender { 51 return &snapshotSender{ 52 from: tr.ID, 53 to: to, 54 cid: tr.ClusterID, 55 tr: tr, 56 picker: picker, 57 status: status, 58 r: tr.Raft, 59 errorc: tr.ErrorC, 60 stopc: make(chan struct{}), 61 } 62} 63 64func (s *snapshotSender) stop() { close(s.stopc) } 65 66func (s *snapshotSender) send(merged snap.Message) { 67 start := time.Now() 68 69 m := merged.Message 70 to := types.ID(m.To).String() 71 72 body := createSnapBody(merged) 73 defer body.Close() 74 75 u := s.picker.pick() 76 req := createPostRequest(u, RaftSnapshotPrefix, body, "application/octet-stream", s.tr.URLs, s.from, s.cid) 77 78 plog.Infof("start to send database snapshot [index: %d, to %s]...", m.Snapshot.Metadata.Index, types.ID(m.To)) 79 snapshotSendInflights.WithLabelValues(to).Inc() 80 defer func() { 81 snapshotSendInflights.WithLabelValues(to).Dec() 82 }() 83 84 err := s.post(req) 85 defer merged.CloseWithError(err) 86 if err != nil { 87 plog.Warningf("database snapshot [index: %d, to: %s] failed to be sent out (%v)", m.Snapshot.Metadata.Index, types.ID(m.To), err) 88 89 // errMemberRemoved is a critical error since a removed member should 90 // always be stopped. So we use reportCriticalError to report it to errorc. 91 if err == errMemberRemoved { 92 reportCriticalError(err, s.errorc) 93 } 94 95 s.picker.unreachable(u) 96 s.status.deactivate(failureType{source: sendSnap, action: "post"}, err.Error()) 97 s.r.ReportUnreachable(m.To) 98 // report SnapshotFailure to raft state machine. After raft state 99 // machine knows about it, it would pause a while and retry sending 100 // new snapshot message. 101 s.r.ReportSnapshot(m.To, raft.SnapshotFailure) 102 sentFailures.WithLabelValues(to).Inc() 103 snapshotSendFailures.WithLabelValues(to).Inc() 104 return 105 } 106 s.status.activate() 107 s.r.ReportSnapshot(m.To, raft.SnapshotFinish) 108 plog.Infof("database snapshot [index: %d, to: %s] sent out successfully", m.Snapshot.Metadata.Index, types.ID(m.To)) 109 110 sentBytes.WithLabelValues(to).Add(float64(merged.TotalSize)) 111 112 snapshotSend.WithLabelValues(to).Inc() 113 snapshotSendSeconds.WithLabelValues(to).Observe(time.Since(start).Seconds()) 114} 115 116// post posts the given request. 117// It returns nil when request is sent out and processed successfully. 118func (s *snapshotSender) post(req *http.Request) (err error) { 119 ctx, cancel := context.WithCancel(context.Background()) 120 req = req.WithContext(ctx) 121 defer cancel() 122 123 type responseAndError struct { 124 resp *http.Response 125 body []byte 126 err error 127 } 128 result := make(chan responseAndError, 1) 129 130 go func() { 131 resp, err := s.tr.pipelineRt.RoundTrip(req) 132 if err != nil { 133 result <- responseAndError{resp, nil, err} 134 return 135 } 136 137 // close the response body when timeouts. 138 // prevents from reading the body forever when the other side dies right after 139 // successfully receives the request body. 140 time.AfterFunc(snapResponseReadTimeout, func() { httputil.GracefulClose(resp) }) 141 body, err := ioutil.ReadAll(resp.Body) 142 result <- responseAndError{resp, body, err} 143 }() 144 145 select { 146 case <-s.stopc: 147 return errStopped 148 case r := <-result: 149 if r.err != nil { 150 return r.err 151 } 152 return checkPostResponse(r.resp, r.body, req, s.to) 153 } 154} 155 156func createSnapBody(merged snap.Message) io.ReadCloser { 157 buf := new(bytes.Buffer) 158 enc := &messageEncoder{w: buf} 159 // encode raft message 160 if err := enc.encode(&merged.Message); err != nil { 161 plog.Panicf("encode message error (%v)", err) 162 } 163 164 return &pioutil.ReaderAndCloser{ 165 Reader: io.MultiReader(buf, merged.ReadCloser), 166 Closer: merged.ReadCloser, 167 } 168} 169