1// Copyright 2015 The etcd Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package etcdserver 16 17import ( 18 "context" 19 "encoding/json" 20 "fmt" 21 "io/ioutil" 22 "net/http" 23 "sort" 24 "strings" 25 "time" 26 27 "go.etcd.io/etcd/etcdserver/api/membership" 28 "go.etcd.io/etcd/pkg/types" 29 "go.etcd.io/etcd/version" 30 31 "github.com/coreos/go-semver/semver" 32 "go.uber.org/zap" 33) 34 35// isMemberBootstrapped tries to check if the given member has been bootstrapped 36// in the given cluster. 37func isMemberBootstrapped(lg *zap.Logger, cl *membership.RaftCluster, member string, rt http.RoundTripper, timeout time.Duration) bool { 38 rcl, err := getClusterFromRemotePeers(lg, getRemotePeerURLs(cl, member), timeout, false, rt) 39 if err != nil { 40 return false 41 } 42 id := cl.MemberByName(member).ID 43 m := rcl.Member(id) 44 if m == nil { 45 return false 46 } 47 if len(m.ClientURLs) > 0 { 48 return true 49 } 50 return false 51} 52 53// GetClusterFromRemotePeers takes a set of URLs representing etcd peers, and 54// attempts to construct a Cluster by accessing the members endpoint on one of 55// these URLs. The first URL to provide a response is used. If no URLs provide 56// a response, or a Cluster cannot be successfully created from a received 57// response, an error is returned. 58// Each request has a 10-second timeout. Because the upper limit of TTL is 5s, 59// 10 second is enough for building connection and finishing request. 60func GetClusterFromRemotePeers(lg *zap.Logger, urls []string, rt http.RoundTripper) (*membership.RaftCluster, error) { 61 return getClusterFromRemotePeers(lg, urls, 10*time.Second, true, rt) 62} 63 64// If logerr is true, it prints out more error messages. 65func getClusterFromRemotePeers(lg *zap.Logger, urls []string, timeout time.Duration, logerr bool, rt http.RoundTripper) (*membership.RaftCluster, error) { 66 if lg == nil { 67 lg = zap.NewNop() 68 } 69 cc := &http.Client{ 70 Transport: rt, 71 Timeout: timeout, 72 } 73 for _, u := range urls { 74 addr := u + "/members" 75 resp, err := cc.Get(addr) 76 if err != nil { 77 if logerr { 78 lg.Warn("failed to get cluster response", zap.String("address", addr), zap.Error(err)) 79 } 80 continue 81 } 82 b, err := ioutil.ReadAll(resp.Body) 83 resp.Body.Close() 84 if err != nil { 85 if logerr { 86 lg.Warn("failed to read body of cluster response", zap.String("address", addr), zap.Error(err)) 87 } 88 continue 89 } 90 var membs []*membership.Member 91 if err = json.Unmarshal(b, &membs); err != nil { 92 if logerr { 93 lg.Warn("failed to unmarshal cluster response", zap.String("address", addr), zap.Error(err)) 94 } 95 continue 96 } 97 id, err := types.IDFromString(resp.Header.Get("X-Etcd-Cluster-ID")) 98 if err != nil { 99 if logerr { 100 lg.Warn( 101 "failed to parse cluster ID", 102 zap.String("address", addr), 103 zap.String("header", resp.Header.Get("X-Etcd-Cluster-ID")), 104 zap.Error(err), 105 ) 106 } 107 continue 108 } 109 110 // check the length of membership members 111 // if the membership members are present then prepare and return raft cluster 112 // if membership members are not present then the raft cluster formed will be 113 // an invalid empty cluster hence return failed to get raft cluster member(s) from the given urls error 114 if len(membs) > 0 { 115 return membership.NewClusterFromMembers(lg, "", id, membs), nil 116 } 117 return nil, fmt.Errorf("failed to get raft cluster member(s) from the given URLs") 118 } 119 return nil, fmt.Errorf("could not retrieve cluster information from the given URLs") 120} 121 122// getRemotePeerURLs returns peer urls of remote members in the cluster. The 123// returned list is sorted in ascending lexicographical order. 124func getRemotePeerURLs(cl *membership.RaftCluster, local string) []string { 125 us := make([]string, 0) 126 for _, m := range cl.Members() { 127 if m.Name == local { 128 continue 129 } 130 us = append(us, m.PeerURLs...) 131 } 132 sort.Strings(us) 133 return us 134} 135 136// getVersions returns the versions of the members in the given cluster. 137// The key of the returned map is the member's ID. The value of the returned map 138// is the semver versions string, including server and cluster. 139// If it fails to get the version of a member, the key will be nil. 140func getVersions(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper) map[string]*version.Versions { 141 members := cl.Members() 142 vers := make(map[string]*version.Versions) 143 for _, m := range members { 144 if m.ID == local { 145 cv := "not_decided" 146 if cl.Version() != nil { 147 cv = cl.Version().String() 148 } 149 vers[m.ID.String()] = &version.Versions{Server: version.Version, Cluster: cv} 150 continue 151 } 152 ver, err := getVersion(lg, m, rt) 153 if err != nil { 154 lg.Warn("failed to get version", zap.String("remote-member-id", m.ID.String()), zap.Error(err)) 155 vers[m.ID.String()] = nil 156 } else { 157 vers[m.ID.String()] = ver 158 } 159 } 160 return vers 161} 162 163// decideClusterVersion decides the cluster version based on the versions map. 164// The returned version is the min server version in the map, or nil if the min 165// version in unknown. 166func decideClusterVersion(lg *zap.Logger, vers map[string]*version.Versions) *semver.Version { 167 var cv *semver.Version 168 lv := semver.Must(semver.NewVersion(version.Version)) 169 170 for mid, ver := range vers { 171 if ver == nil { 172 return nil 173 } 174 v, err := semver.NewVersion(ver.Server) 175 if err != nil { 176 lg.Warn( 177 "failed to parse server version of remote member", 178 zap.String("remote-member-id", mid), 179 zap.String("remote-member-version", ver.Server), 180 zap.Error(err), 181 ) 182 return nil 183 } 184 if lv.LessThan(*v) { 185 lg.Warn( 186 "leader found higher-versioned member", 187 zap.String("local-member-version", lv.String()), 188 zap.String("remote-member-id", mid), 189 zap.String("remote-member-version", ver.Server), 190 ) 191 } 192 if cv == nil { 193 cv = v 194 } else if v.LessThan(*cv) { 195 cv = v 196 } 197 } 198 return cv 199} 200 201// isCompatibleWithCluster return true if the local member has a compatible version with 202// the current running cluster. 203// The version is considered as compatible when at least one of the other members in the cluster has a 204// cluster version in the range of [MinClusterVersion, Version] and no known members has a cluster version 205// out of the range. 206// We set this rule since when the local member joins, another member might be offline. 207func isCompatibleWithCluster(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper) bool { 208 vers := getVersions(lg, cl, local, rt) 209 minV := semver.Must(semver.NewVersion(version.MinClusterVersion)) 210 maxV := semver.Must(semver.NewVersion(version.Version)) 211 maxV = &semver.Version{ 212 Major: maxV.Major, 213 Minor: maxV.Minor, 214 } 215 return isCompatibleWithVers(lg, vers, local, minV, maxV) 216} 217 218func isCompatibleWithVers(lg *zap.Logger, vers map[string]*version.Versions, local types.ID, minV, maxV *semver.Version) bool { 219 var ok bool 220 for id, v := range vers { 221 // ignore comparison with local version 222 if id == local.String() { 223 continue 224 } 225 if v == nil { 226 continue 227 } 228 clusterv, err := semver.NewVersion(v.Cluster) 229 if err != nil { 230 lg.Warn( 231 "failed to parse cluster version of remote member", 232 zap.String("remote-member-id", id), 233 zap.String("remote-member-cluster-version", v.Cluster), 234 zap.Error(err), 235 ) 236 continue 237 } 238 if clusterv.LessThan(*minV) { 239 lg.Warn( 240 "cluster version of remote member is not compatible; too low", 241 zap.String("remote-member-id", id), 242 zap.String("remote-member-cluster-version", clusterv.String()), 243 zap.String("minimum-cluster-version-supported", minV.String()), 244 ) 245 return false 246 } 247 if maxV.LessThan(*clusterv) { 248 lg.Warn( 249 "cluster version of remote member is not compatible; too high", 250 zap.String("remote-member-id", id), 251 zap.String("remote-member-cluster-version", clusterv.String()), 252 zap.String("minimum-cluster-version-supported", minV.String()), 253 ) 254 return false 255 } 256 ok = true 257 } 258 return ok 259} 260 261// getVersion returns the Versions of the given member via its 262// peerURLs. Returns the last error if it fails to get the version. 263func getVersion(lg *zap.Logger, m *membership.Member, rt http.RoundTripper) (*version.Versions, error) { 264 cc := &http.Client{ 265 Transport: rt, 266 } 267 var ( 268 err error 269 resp *http.Response 270 ) 271 272 for _, u := range m.PeerURLs { 273 addr := u + "/version" 274 resp, err = cc.Get(addr) 275 if err != nil { 276 lg.Warn( 277 "failed to reach the peer URL", 278 zap.String("address", addr), 279 zap.String("remote-member-id", m.ID.String()), 280 zap.Error(err), 281 ) 282 continue 283 } 284 var b []byte 285 b, err = ioutil.ReadAll(resp.Body) 286 resp.Body.Close() 287 if err != nil { 288 lg.Warn( 289 "failed to read body of response", 290 zap.String("address", addr), 291 zap.String("remote-member-id", m.ID.String()), 292 zap.Error(err), 293 ) 294 continue 295 } 296 var vers version.Versions 297 if err = json.Unmarshal(b, &vers); err != nil { 298 lg.Warn( 299 "failed to unmarshal response", 300 zap.String("address", addr), 301 zap.String("remote-member-id", m.ID.String()), 302 zap.Error(err), 303 ) 304 continue 305 } 306 return &vers, nil 307 } 308 return nil, err 309} 310 311func promoteMemberHTTP(ctx context.Context, url string, id uint64, peerRt http.RoundTripper) ([]*membership.Member, error) { 312 cc := &http.Client{Transport: peerRt} 313 // TODO: refactor member http handler code 314 // cannot import etcdhttp, so manually construct url 315 requestUrl := url + "/members/promote/" + fmt.Sprintf("%d", id) 316 req, err := http.NewRequest("POST", requestUrl, nil) 317 if err != nil { 318 return nil, err 319 } 320 req = req.WithContext(ctx) 321 resp, err := cc.Do(req) 322 if err != nil { 323 return nil, err 324 } 325 defer resp.Body.Close() 326 b, err := ioutil.ReadAll(resp.Body) 327 if err != nil { 328 return nil, err 329 } 330 331 if resp.StatusCode == http.StatusRequestTimeout { 332 return nil, ErrTimeout 333 } 334 if resp.StatusCode == http.StatusPreconditionFailed { 335 // both ErrMemberNotLearner and ErrLearnerNotReady have same http status code 336 if strings.Contains(string(b), ErrLearnerNotReady.Error()) { 337 return nil, ErrLearnerNotReady 338 } 339 if strings.Contains(string(b), membership.ErrMemberNotLearner.Error()) { 340 return nil, membership.ErrMemberNotLearner 341 } 342 return nil, fmt.Errorf("member promote: unknown error(%s)", string(b)) 343 } 344 if resp.StatusCode == http.StatusNotFound { 345 return nil, membership.ErrIDNotFound 346 } 347 348 if resp.StatusCode != http.StatusOK { // all other types of errors 349 return nil, fmt.Errorf("member promote: unknown error(%s)", string(b)) 350 } 351 352 var membs []*membership.Member 353 if err := json.Unmarshal(b, &membs); err != nil { 354 return nil, err 355 } 356 return membs, nil 357} 358