1// Copyright (c) 2015-2021 MinIO, Inc. 2// 3// This file is part of MinIO Object Storage stack 4// 5// This program is free software: you can redistribute it and/or modify 6// it under the terms of the GNU Affero General Public License as published by 7// the Free Software Foundation, either version 3 of the License, or 8// (at your option) any later version. 9// 10// This program is distributed in the hope that it will be useful 11// but WITHOUT ANY WARRANTY; without even the implied warranty of 12// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13// GNU Affero General Public License for more details. 14// 15// You should have received a copy of the GNU Affero General Public License 16// along with this program. If not, see <http://www.gnu.org/licenses/>. 17 18package cmd 19 20import ( 21 "fmt" 22 "net/url" 23 "path" 24 "time" 25 26 humanize "github.com/dustin/go-humanize" 27 "github.com/fatih/color" 28 "github.com/minio/cli" 29 json "github.com/minio/colorjson" 30 "github.com/minio/madmin-go" 31 "github.com/minio/mc/pkg/probe" 32 "github.com/minio/minio-go/v7/pkg/s3utils" 33 "github.com/minio/pkg/console" 34) 35 36var adminBucketRemoteAddFlags = []cli.Flag{ 37 cli.StringFlag{ 38 Name: "path", 39 Value: "auto", 40 Usage: "bucket path lookup supported by the server. Valid options are '[on,off,auto]'", 41 }, 42 cli.StringFlag{ 43 Name: "service", 44 Usage: "type of service. Valid options are '[replication]'", 45 }, 46 cli.StringFlag{ 47 Name: "region", 48 Usage: "region of the destination bucket (optional)", 49 }, 50 cli.StringFlag{ 51 Name: "bandwidth", 52 Usage: "Set bandwidth limit in bits per second (K,B,G,T for metric and Ki,Bi,Gi,Ti for IEC units)", 53 }, 54 cli.BoolFlag{ 55 Name: "sync", 56 Usage: "enable synchronous replication for this target. Default is async", 57 }, 58 cli.UintFlag{ 59 Name: "healthcheck-seconds", 60 Usage: "health check duration in seconds", 61 Value: 60, 62 }, 63 cli.BoolFlag{ 64 Name: "disable-proxy", 65 Usage: "disable proxying in active-active replication. If unset, default behavior is to proxy", 66 }, 67} 68var adminBucketRemoteAddCmd = cli.Command{ 69 Name: "add", 70 Usage: "add a new remote target", 71 Action: mainAdminBucketRemoteAdd, 72 OnUsageError: onUsageError, 73 Before: setGlobalsFromContext, 74 Flags: append(globalFlags, adminBucketRemoteAddFlags...), 75 CustomHelpTemplate: `NAME: 76 {{.HelpName}} - {{.Usage}} 77 78USAGE: 79 {{.HelpName}} TARGET http(s)://ACCESSKEY:SECRETKEY@DEST_URL/DEST_BUCKET [--path | --region | --bandwidth] --service 80 81TARGET: 82 Also called as alias/sourcebucketname 83 84DEST_BUCKET: 85 Also called as remote target bucket. 86 87DEST_URL: 88 Also called as remote endpoint. 89 90ACCESSKEY: 91 Also called as username. 92 93SECRETKEY: 94 Also called as password. 95 96FLAGS: 97 {{range .VisibleFlags}}{{.}} 98 {{end}} 99EXAMPLES: 100 1. Set a new remote replication target "targetbucket" in region "us-west-1" on https://minio.siteb.example.com for bucket 'sourcebucket'. 101 {{.Prompt}} {{.HelpName}} sitea/sourcebucket https://foobar:foo12345@minio.siteb.example.com/targetbucket \ 102 --service "replication" --region "us-west-1" 103 104 2. Set a new remote replication target 'targetbucket' in region "us-west-1" on https://minio.siteb.example.com for 105 bucket 'sourcebucket' with bandwidth set to 2 gigabits per second. Enable synchronous replication to the target 106 and perform health check of target every 100 seconds 107 {{.Prompt}} {{.HelpName}} sitea/sourcebucket https://foobar:foo12345@minio.siteb.example.com/targetbucket \ 108 --service "replication" --region "us-west-1 --bandwidth "2G" --sync 109`, 110} 111 112// checkAdminBucketRemoteAddSyntax - validate all the passed arguments 113func checkAdminBucketRemoteAddSyntax(ctx *cli.Context) { 114 argsNr := len(ctx.Args()) 115 if argsNr < 2 { 116 cli.ShowCommandHelpAndExit(ctx, ctx.Command.Name, 1) // last argument is exit code 117 } 118 if argsNr > 2 { 119 fatalIf(errInvalidArgument().Trace(ctx.Args().Tail()...), 120 "Incorrect number of arguments for remote add command.") 121 } 122} 123 124// RemoteMessage container for content message structure 125type RemoteMessage struct { 126 op string 127 Status string `json:"status"` 128 AccessKey string `json:"accessKey,omitempty"` 129 SecretKey string `json:"secretKey,omitempty"` 130 SourceBucket string `json:"sourceBucket"` 131 TargetURL string `json:"TargetURL,omitempty"` 132 TargetBucket string `json:"TargetBucket,omitempty"` 133 RemoteARN string `json:"RemoteARN,omitempty"` 134 Path string `json:"path,omitempty"` 135 Region string `json:"region,omitempty"` 136 ServiceType string `json:"service"` 137 Bandwidth int64 `json:"bandwidth"` 138 ReplicationSync bool `json:"replicationSync"` 139 Proxy bool `json:"proxy"` 140 HealthCheckDuration time.Duration `json:"healthcheckDuration"` 141 ResetID string `json:"resetID"` 142 ResetBefore time.Time `json:"resetBeforeDate"` 143} 144 145func (r RemoteMessage) String() string { 146 switch r.op { 147 case "ls": 148 message := console.Colorize("TargetURL", fmt.Sprintf("%s ", r.TargetURL)) 149 message += console.Colorize("SourceBucket", r.SourceBucket) 150 message += console.Colorize("Arrow", "->") 151 message += console.Colorize("TargetBucket", r.TargetBucket) 152 message += " " 153 message += console.Colorize("ARN", r.RemoteARN) 154 syncStr := " " 155 if r.ReplicationSync && r.ServiceType == string(madmin.ReplicationService) { 156 syncStr = "sync" 157 } 158 message += " " + console.Colorize("SyncLabel", syncStr) 159 proxyStr := " " 160 if r.Proxy && r.ServiceType == string(madmin.ReplicationService) { 161 proxyStr = "proxy" 162 } 163 message += " " 164 message += console.Colorize("ProxyLabel", proxyStr) 165 return message 166 case "rm": 167 return console.Colorize("RemoteMessage", "Removed remote target for `"+r.SourceBucket+"` bucket successfully.") 168 case "add": 169 return console.Colorize("RemoteMessage", "Remote ARN = `"+r.RemoteARN+"`.") 170 case "edit": 171 return console.Colorize("RemoteMessage", "Remote target updated successfully for target with ARN:`"+r.RemoteARN+"`.") 172 } 173 return "" 174} 175 176// JSON returns jsonified message 177func (r RemoteMessage) JSON() string { 178 r.Status = "success" 179 jsonMessageBytes, e := json.MarshalIndent(r, "", " ") 180 fatalIf(probe.NewError(e), "Unable to marshal into JSON.") 181 182 return string(jsonMessageBytes) 183} 184 185func extractCredentialURL(argURL string) (accessKey, secretKey string, u *url.URL) { 186 var parsedURL string 187 if hostKeyTokens.MatchString(argURL) { 188 fatalIf(errInvalidArgument().Trace(argURL), "temporary tokens are not allowed for remote targets") 189 } 190 if hostKeys.MatchString(argURL) { 191 parts := hostKeys.FindStringSubmatch(argURL) 192 if len(parts) != 5 { 193 fatalIf(errInvalidArgument().Trace(argURL), "Unsupported remote target format, please check --help") 194 } 195 accessKey = parts[2] 196 secretKey = parts[3] 197 parsedURL = fmt.Sprintf("%s%s", parts[1], parts[4]) 198 } 199 var e error 200 if parsedURL == "" { 201 fatalIf(errInvalidArgument().Trace(argURL), "No valid credentials were detected") 202 } 203 u, e = url.Parse(parsedURL) 204 if e != nil { 205 fatalIf(errInvalidArgument().Trace(parsedURL), "Unsupported URL format %v", e) 206 } 207 208 return accessKey, secretKey, u 209} 210 211// fetchRemoteTarget - returns the dest bucket, dest endpoint, access and secret key 212func fetchRemoteTarget(cli *cli.Context) (sourceBucket string, bktTarget *madmin.BucketTarget) { 213 args := cli.Args() 214 argCount := len(args) 215 if argCount < 2 { 216 fatalIf(probe.NewError(fmt.Errorf("Missing Remote target configuration")), "Unable to parse remote target") 217 } 218 _, sourceBucket = url2Alias(args[0]) 219 p := cli.String("path") 220 if !isValidPath(p) { 221 fatalIf(errInvalidArgument().Trace(p), 222 "Unrecognized bucket path style. Valid options are `[on,off, auto]`.") 223 } 224 225 tgtURL := args[1] 226 accessKey, secretKey, u := extractCredentialURL(tgtURL) 227 var tgtBucket string 228 if u.Path != "" { 229 tgtBucket = path.Clean(u.Path[1:]) 230 } 231 if e := s3utils.CheckValidBucketName(tgtBucket); e != nil { 232 fatalIf(probe.NewError(e).Trace(tgtURL), "Invalid target bucket specified") 233 } 234 235 serviceType := cli.String("service") 236 if !madmin.ServiceType(serviceType).IsValid() { 237 fatalIf(errInvalidArgument().Trace(serviceType), "Invalid service type. Valid option is `[replication]`.") 238 } 239 if cli.IsSet("sync") && serviceType != string(madmin.ReplicationService) { 240 fatalIf(errInvalidArgument(), "Invalid usage. --sync flag applies only to replication service") 241 } 242 bandwidthStr := cli.String("bandwidth") 243 bandwidth, err := getBandwidthInBytes(bandwidthStr) 244 if err != nil { 245 fatalIf(errInvalidArgument().Trace(bandwidthStr), "Invalid bandwidth number") 246 } 247 console.SetColor(cred, color.New(color.FgYellow, color.Italic)) 248 creds := &madmin.Credentials{AccessKey: accessKey, SecretKey: secretKey} 249 disableproxy := cli.Bool("disable-proxy") 250 bktTarget = &madmin.BucketTarget{ 251 TargetBucket: tgtBucket, 252 Secure: u.Scheme == "https", 253 Credentials: creds, 254 Endpoint: u.Host, 255 Path: p, 256 API: "s3v4", 257 Type: madmin.ServiceType(serviceType), 258 Region: cli.String("region"), 259 BandwidthLimit: int64(bandwidth), 260 ReplicationSync: cli.Bool("sync"), 261 DisableProxy: disableproxy, 262 HealthCheckDuration: time.Duration(cli.Uint("healthcheck-seconds")) * time.Second, 263 } 264 return sourceBucket, bktTarget 265} 266 267func getBandwidthInBytes(bandwidthStr string) (bandwidth uint64, err error) { 268 if bandwidthStr != "" { 269 bandwidth, err = humanize.ParseBytes(bandwidthStr) 270 if err != nil { 271 return 272 } 273 } 274 bandwidth = bandwidth / 8 275 return 276} 277 278// mainAdminBucketRemoteAdd is the handle for "mc admin bucket remote set" command. 279func mainAdminBucketRemoteAdd(ctx *cli.Context) error { 280 checkAdminBucketRemoteAddSyntax(ctx) 281 console.SetColor("RemoteMessage", color.New(color.FgGreen)) 282 283 // Get the alias parameter from cli 284 args := ctx.Args() 285 aliasedURL := args.Get(0) 286 // Create a new MinIO Admin Client 287 client, cerr := newAdminClient(aliasedURL) 288 fatalIf(cerr, "Unable to initialize admin connection.") 289 290 sourceBucket, bktTarget := fetchRemoteTarget(ctx) 291 arn, e := client.SetRemoteTarget(globalContext, sourceBucket, bktTarget) 292 if e != nil { 293 fatalIf(probe.NewError(e).Trace(args...), "Unable to configure remote target") 294 } 295 296 printMsg(RemoteMessage{ 297 op: ctx.Command.Name, 298 TargetURL: bktTarget.URL().String(), 299 TargetBucket: bktTarget.TargetBucket, 300 AccessKey: bktTarget.Credentials.AccessKey, 301 SourceBucket: sourceBucket, 302 RemoteARN: arn, 303 ReplicationSync: bktTarget.ReplicationSync, 304 Proxy: !bktTarget.DisableProxy, 305 }) 306 307 return nil 308} 309