1package docker 2 3import ( 4 "context" 5 "errors" 6 "fmt" 7 "io" 8 "net" 9 "net/http" 10 "strconv" 11 "strings" 12 "text/template" 13 "time" 14 15 "github.com/cenkalti/backoff/v4" 16 "github.com/docker/cli/cli/connhelper" 17 dockertypes "github.com/docker/docker/api/types" 18 dockercontainertypes "github.com/docker/docker/api/types/container" 19 eventtypes "github.com/docker/docker/api/types/events" 20 "github.com/docker/docker/api/types/filters" 21 swarmtypes "github.com/docker/docker/api/types/swarm" 22 "github.com/docker/docker/api/types/versions" 23 "github.com/docker/docker/client" 24 "github.com/docker/go-connections/nat" 25 "github.com/docker/go-connections/sockets" 26 ptypes "github.com/traefik/paerser/types" 27 "github.com/traefik/traefik/v2/pkg/config/dynamic" 28 "github.com/traefik/traefik/v2/pkg/job" 29 "github.com/traefik/traefik/v2/pkg/log" 30 "github.com/traefik/traefik/v2/pkg/provider" 31 "github.com/traefik/traefik/v2/pkg/safe" 32 "github.com/traefik/traefik/v2/pkg/types" 33 "github.com/traefik/traefik/v2/pkg/version" 34) 35 36const ( 37 // DockerAPIVersion is a constant holding the version of the Provider API traefik will use. 38 DockerAPIVersion = "1.24" 39 40 // SwarmAPIVersion is a constant holding the version of the Provider API traefik will use. 41 SwarmAPIVersion = "1.24" 42) 43 44// DefaultTemplateRule The default template for the default rule. 45const DefaultTemplateRule = "Host(`{{ normalize .Name }}`)" 46 47var _ provider.Provider = (*Provider)(nil) 48 49// Provider holds configurations of the provider. 50type Provider struct { 51 Constraints string `description:"Constraints is an expression that Traefik matches against the container's labels to determine whether to create any route for that container." json:"constraints,omitempty" toml:"constraints,omitempty" yaml:"constraints,omitempty" export:"true"` 52 Watch bool `description:"Watch Docker Swarm events." json:"watch,omitempty" toml:"watch,omitempty" yaml:"watch,omitempty" export:"true"` 53 Endpoint string `description:"Docker server endpoint. Can be a tcp or a unix socket endpoint." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"` 54 DefaultRule string `description:"Default rule." json:"defaultRule,omitempty" toml:"defaultRule,omitempty" yaml:"defaultRule,omitempty"` 55 TLS *types.ClientTLS `description:"Enable Docker TLS support." json:"tls,omitempty" toml:"tls,omitempty" yaml:"tls,omitempty" export:"true"` 56 ExposedByDefault bool `description:"Expose containers by default." json:"exposedByDefault,omitempty" toml:"exposedByDefault,omitempty" yaml:"exposedByDefault,omitempty" export:"true"` 57 UseBindPortIP bool `description:"Use the ip address from the bound port, rather than from the inner network." json:"useBindPortIP,omitempty" toml:"useBindPortIP,omitempty" yaml:"useBindPortIP,omitempty" export:"true"` 58 SwarmMode bool `description:"Use Docker on Swarm Mode." json:"swarmMode,omitempty" toml:"swarmMode,omitempty" yaml:"swarmMode,omitempty" export:"true"` 59 Network string `description:"Default Docker network used." json:"network,omitempty" toml:"network,omitempty" yaml:"network,omitempty" export:"true"` 60 SwarmModeRefreshSeconds ptypes.Duration `description:"Polling interval for swarm mode." json:"swarmModeRefreshSeconds,omitempty" toml:"swarmModeRefreshSeconds,omitempty" yaml:"swarmModeRefreshSeconds,omitempty" export:"true"` 61 HTTPClientTimeout ptypes.Duration `description:"Client timeout for HTTP connections." json:"httpClientTimeout,omitempty" toml:"httpClientTimeout,omitempty" yaml:"httpClientTimeout,omitempty" export:"true"` 62 defaultRuleTpl *template.Template 63} 64 65// SetDefaults sets the default values. 66func (p *Provider) SetDefaults() { 67 p.Watch = true 68 p.ExposedByDefault = true 69 p.Endpoint = "unix:///var/run/docker.sock" 70 p.SwarmMode = false 71 p.SwarmModeRefreshSeconds = ptypes.Duration(15 * time.Second) 72 p.DefaultRule = DefaultTemplateRule 73} 74 75// Init the provider. 76func (p *Provider) Init() error { 77 defaultRuleTpl, err := provider.MakeDefaultRuleTemplate(p.DefaultRule, nil) 78 if err != nil { 79 return fmt.Errorf("error while parsing default rule: %w", err) 80 } 81 82 p.defaultRuleTpl = defaultRuleTpl 83 return nil 84} 85 86// dockerData holds the need data to the provider. 87type dockerData struct { 88 ID string 89 ServiceName string 90 Name string 91 Labels map[string]string // List of labels set to container or service 92 NetworkSettings networkSettings 93 Health string 94 Node *dockertypes.ContainerNode 95 ExtraConf configuration 96} 97 98// NetworkSettings holds the networks data to the provider. 99type networkSettings struct { 100 NetworkMode dockercontainertypes.NetworkMode 101 Ports nat.PortMap 102 Networks map[string]*networkData 103} 104 105// Network holds the network data to the provider. 106type networkData struct { 107 Name string 108 Addr string 109 Port int 110 Protocol string 111 ID string 112} 113 114func (p *Provider) createClient() (client.APIClient, error) { 115 opts, err := p.getClientOpts() 116 if err != nil { 117 return nil, err 118 } 119 120 httpHeaders := map[string]string{ 121 "User-Agent": "Traefik " + version.Version, 122 } 123 opts = append(opts, client.WithHTTPHeaders(httpHeaders)) 124 125 apiVersion := DockerAPIVersion 126 if p.SwarmMode { 127 apiVersion = SwarmAPIVersion 128 } 129 opts = append(opts, client.WithVersion(apiVersion)) 130 131 return client.NewClientWithOpts(opts...) 132} 133 134func (p *Provider) getClientOpts() ([]client.Opt, error) { 135 helper, err := connhelper.GetConnectionHelper(p.Endpoint) 136 if err != nil { 137 return nil, err 138 } 139 140 // SSH 141 if helper != nil { 142 // https://github.com/docker/cli/blob/ebca1413117a3fcb81c89d6be226dcec74e5289f/cli/context/docker/load.go#L112-L123 143 144 httpClient := &http.Client{ 145 Transport: &http.Transport{ 146 DialContext: helper.Dialer, 147 }, 148 } 149 150 return []client.Opt{ 151 client.WithHTTPClient(httpClient), 152 client.WithTimeout(time.Duration(p.HTTPClientTimeout)), 153 client.WithHost(helper.Host), // To avoid 400 Bad Request: malformed Host header daemon error 154 client.WithDialContext(helper.Dialer), 155 }, nil 156 } 157 158 opts := []client.Opt{ 159 client.WithHost(p.Endpoint), 160 client.WithTimeout(time.Duration(p.HTTPClientTimeout)), 161 } 162 163 if p.TLS != nil { 164 ctx := log.With(context.Background(), log.Str(log.ProviderName, "docker")) 165 166 conf, err := p.TLS.CreateTLSConfig(ctx) 167 if err != nil { 168 return nil, fmt.Errorf("unable to create client TLS configuration: %w", err) 169 } 170 171 hostURL, err := client.ParseHostURL(p.Endpoint) 172 if err != nil { 173 return nil, err 174 } 175 176 tr := &http.Transport{ 177 TLSClientConfig: conf, 178 } 179 180 if err := sockets.ConfigureTransport(tr, hostURL.Scheme, hostURL.Host); err != nil { 181 return nil, err 182 } 183 184 opts = append(opts, client.WithHTTPClient(&http.Client{Transport: tr, Timeout: time.Duration(p.HTTPClientTimeout)})) 185 } 186 187 return opts, nil 188} 189 190// Provide allows the docker provider to provide configurations to traefik using the given configuration channel. 191func (p *Provider) Provide(configurationChan chan<- dynamic.Message, pool *safe.Pool) error { 192 pool.GoCtx(func(routineCtx context.Context) { 193 ctxLog := log.With(routineCtx, log.Str(log.ProviderName, "docker")) 194 logger := log.FromContext(ctxLog) 195 196 operation := func() error { 197 var err error 198 ctx, cancel := context.WithCancel(ctxLog) 199 defer cancel() 200 201 ctx = log.With(ctx, log.Str(log.ProviderName, "docker")) 202 203 dockerClient, err := p.createClient() 204 if err != nil { 205 logger.Errorf("Failed to create a client for docker, error: %s", err) 206 return err 207 } 208 209 serverVersion, err := dockerClient.ServerVersion(ctx) 210 if err != nil { 211 logger.Errorf("Failed to retrieve information of the docker client and server host: %s", err) 212 return err 213 } 214 logger.Debugf("Provider connection established with docker %s (API %s)", serverVersion.Version, serverVersion.APIVersion) 215 var dockerDataList []dockerData 216 if p.SwarmMode { 217 dockerDataList, err = p.listServices(ctx, dockerClient) 218 if err != nil { 219 logger.Errorf("Failed to list services for docker swarm mode, error %s", err) 220 return err 221 } 222 } else { 223 dockerDataList, err = p.listContainers(ctx, dockerClient) 224 if err != nil { 225 logger.Errorf("Failed to list containers for docker, error %s", err) 226 return err 227 } 228 } 229 230 configuration := p.buildConfiguration(ctxLog, dockerDataList) 231 configurationChan <- dynamic.Message{ 232 ProviderName: "docker", 233 Configuration: configuration, 234 } 235 if p.Watch { 236 if p.SwarmMode { 237 errChan := make(chan error) 238 239 // TODO: This need to be change. Linked to Swarm events docker/docker#23827 240 ticker := time.NewTicker(time.Duration(p.SwarmModeRefreshSeconds)) 241 242 pool.GoCtx(func(ctx context.Context) { 243 ctx = log.With(ctx, log.Str(log.ProviderName, "docker")) 244 logger := log.FromContext(ctx) 245 246 defer close(errChan) 247 for { 248 select { 249 case <-ticker.C: 250 services, err := p.listServices(ctx, dockerClient) 251 if err != nil { 252 logger.Errorf("Failed to list services for docker, error %s", err) 253 errChan <- err 254 return 255 } 256 257 configuration := p.buildConfiguration(ctx, services) 258 if configuration != nil { 259 configurationChan <- dynamic.Message{ 260 ProviderName: "docker", 261 Configuration: configuration, 262 } 263 } 264 265 case <-ctx.Done(): 266 ticker.Stop() 267 return 268 } 269 } 270 }) 271 if err, ok := <-errChan; ok { 272 return err 273 } 274 // channel closed 275 } else { 276 f := filters.NewArgs() 277 f.Add("type", "container") 278 options := dockertypes.EventsOptions{ 279 Filters: f, 280 } 281 282 startStopHandle := func(m eventtypes.Message) { 283 logger.Debugf("Provider event received %+v", m) 284 containers, err := p.listContainers(ctx, dockerClient) 285 if err != nil { 286 logger.Errorf("Failed to list containers for docker, error %s", err) 287 // Call cancel to get out of the monitor 288 return 289 } 290 291 configuration := p.buildConfiguration(ctx, containers) 292 if configuration != nil { 293 message := dynamic.Message{ 294 ProviderName: "docker", 295 Configuration: configuration, 296 } 297 select { 298 case configurationChan <- message: 299 case <-ctx.Done(): 300 } 301 } 302 } 303 304 eventsc, errc := dockerClient.Events(ctx, options) 305 for { 306 select { 307 case event := <-eventsc: 308 if event.Action == "start" || 309 event.Action == "die" || 310 strings.HasPrefix(event.Action, "health_status") { 311 startStopHandle(event) 312 } 313 case err := <-errc: 314 if errors.Is(err, io.EOF) { 315 logger.Debug("Provider event stream closed") 316 } 317 return err 318 case <-ctx.Done(): 319 return nil 320 } 321 } 322 } 323 } 324 return nil 325 } 326 327 notify := func(err error, time time.Duration) { 328 logger.Errorf("Provider connection error %+v, retrying in %s", err, time) 329 } 330 err := backoff.RetryNotify(safe.OperationWithRecover(operation), backoff.WithContext(job.NewBackOff(backoff.NewExponentialBackOff()), ctxLog), notify) 331 if err != nil { 332 logger.Errorf("Cannot connect to docker server %+v", err) 333 } 334 }) 335 336 return nil 337} 338 339func (p *Provider) listContainers(ctx context.Context, dockerClient client.ContainerAPIClient) ([]dockerData, error) { 340 containerList, err := dockerClient.ContainerList(ctx, dockertypes.ContainerListOptions{}) 341 if err != nil { 342 return nil, err 343 } 344 345 var inspectedContainers []dockerData 346 // get inspect containers 347 for _, container := range containerList { 348 dData := inspectContainers(ctx, dockerClient, container.ID) 349 if len(dData.Name) == 0 { 350 continue 351 } 352 353 extraConf, err := p.getConfiguration(dData) 354 if err != nil { 355 log.FromContext(ctx).Errorf("Skip container %s: %v", getServiceName(dData), err) 356 continue 357 } 358 dData.ExtraConf = extraConf 359 360 inspectedContainers = append(inspectedContainers, dData) 361 } 362 return inspectedContainers, nil 363} 364 365func inspectContainers(ctx context.Context, dockerClient client.ContainerAPIClient, containerID string) dockerData { 366 containerInspected, err := dockerClient.ContainerInspect(ctx, containerID) 367 if err != nil { 368 log.FromContext(ctx).Warnf("Failed to inspect container %s, error: %s", containerID, err) 369 return dockerData{} 370 } 371 372 // This condition is here to avoid to have empty IP https://github.com/traefik/traefik/issues/2459 373 // We register only container which are running 374 if containerInspected.ContainerJSONBase != nil && containerInspected.ContainerJSONBase.State != nil && containerInspected.ContainerJSONBase.State.Running { 375 return parseContainer(containerInspected) 376 } 377 378 return dockerData{} 379} 380 381func parseContainer(container dockertypes.ContainerJSON) dockerData { 382 dData := dockerData{ 383 NetworkSettings: networkSettings{}, 384 } 385 386 if container.ContainerJSONBase != nil { 387 dData.ID = container.ContainerJSONBase.ID 388 dData.Name = container.ContainerJSONBase.Name 389 dData.ServiceName = dData.Name // Default ServiceName to be the container's Name. 390 dData.Node = container.ContainerJSONBase.Node 391 392 if container.ContainerJSONBase.HostConfig != nil { 393 dData.NetworkSettings.NetworkMode = container.ContainerJSONBase.HostConfig.NetworkMode 394 } 395 396 if container.State != nil && container.State.Health != nil { 397 dData.Health = container.State.Health.Status 398 } 399 } 400 401 if container.Config != nil && container.Config.Labels != nil { 402 dData.Labels = container.Config.Labels 403 } 404 405 if container.NetworkSettings != nil { 406 if container.NetworkSettings.Ports != nil { 407 dData.NetworkSettings.Ports = container.NetworkSettings.Ports 408 } 409 if container.NetworkSettings.Networks != nil { 410 dData.NetworkSettings.Networks = make(map[string]*networkData) 411 for name, containerNetwork := range container.NetworkSettings.Networks { 412 dData.NetworkSettings.Networks[name] = &networkData{ 413 ID: containerNetwork.NetworkID, 414 Name: name, 415 Addr: containerNetwork.IPAddress, 416 } 417 } 418 } 419 } 420 return dData 421} 422 423func (p *Provider) listServices(ctx context.Context, dockerClient client.APIClient) ([]dockerData, error) { 424 logger := log.FromContext(ctx) 425 426 serviceList, err := dockerClient.ServiceList(ctx, dockertypes.ServiceListOptions{}) 427 if err != nil { 428 return nil, err 429 } 430 431 serverVersion, err := dockerClient.ServerVersion(ctx) 432 if err != nil { 433 return nil, err 434 } 435 436 networkListArgs := filters.NewArgs() 437 // https://docs.docker.com/engine/api/v1.29/#tag/Network (Docker 17.06) 438 if versions.GreaterThanOrEqualTo(serverVersion.APIVersion, "1.29") { 439 networkListArgs.Add("scope", "swarm") 440 } else { 441 networkListArgs.Add("driver", "overlay") 442 } 443 444 networkList, err := dockerClient.NetworkList(ctx, dockertypes.NetworkListOptions{Filters: networkListArgs}) 445 if err != nil { 446 logger.Debugf("Failed to network inspect on client for docker, error: %s", err) 447 return nil, err 448 } 449 450 networkMap := make(map[string]*dockertypes.NetworkResource) 451 for _, network := range networkList { 452 networkToAdd := network 453 networkMap[network.ID] = &networkToAdd 454 } 455 456 var dockerDataList []dockerData 457 var dockerDataListTasks []dockerData 458 459 for _, service := range serviceList { 460 dData, err := p.parseService(ctx, service, networkMap) 461 if err != nil { 462 logger.Errorf("Skip container %s: %v", getServiceName(dData), err) 463 continue 464 } 465 466 if dData.ExtraConf.Docker.LBSwarm { 467 if len(dData.NetworkSettings.Networks) > 0 { 468 dockerDataList = append(dockerDataList, dData) 469 } 470 } else { 471 isGlobalSvc := service.Spec.Mode.Global != nil 472 dockerDataListTasks, err = listTasks(ctx, dockerClient, service.ID, dData, networkMap, isGlobalSvc) 473 if err != nil { 474 logger.Warn(err) 475 } else { 476 dockerDataList = append(dockerDataList, dockerDataListTasks...) 477 } 478 } 479 } 480 return dockerDataList, err 481} 482 483func (p *Provider) parseService(ctx context.Context, service swarmtypes.Service, networkMap map[string]*dockertypes.NetworkResource) (dockerData, error) { 484 logger := log.FromContext(ctx) 485 486 dData := dockerData{ 487 ID: service.ID, 488 ServiceName: service.Spec.Annotations.Name, 489 Name: service.Spec.Annotations.Name, 490 Labels: service.Spec.Annotations.Labels, 491 NetworkSettings: networkSettings{}, 492 } 493 494 extraConf, err := p.getConfiguration(dData) 495 if err != nil { 496 return dockerData{}, err 497 } 498 dData.ExtraConf = extraConf 499 500 if service.Spec.EndpointSpec != nil { 501 if service.Spec.EndpointSpec.Mode == swarmtypes.ResolutionModeDNSRR { 502 if dData.ExtraConf.Docker.LBSwarm { 503 logger.Warnf("Ignored %s endpoint-mode not supported, service name: %s. Fallback to Traefik load balancing", swarmtypes.ResolutionModeDNSRR, service.Spec.Annotations.Name) 504 } 505 } else if service.Spec.EndpointSpec.Mode == swarmtypes.ResolutionModeVIP { 506 dData.NetworkSettings.Networks = make(map[string]*networkData) 507 for _, virtualIP := range service.Endpoint.VirtualIPs { 508 networkService := networkMap[virtualIP.NetworkID] 509 if networkService != nil { 510 if len(virtualIP.Addr) > 0 { 511 ip, _, _ := net.ParseCIDR(virtualIP.Addr) 512 network := &networkData{ 513 Name: networkService.Name, 514 ID: virtualIP.NetworkID, 515 Addr: ip.String(), 516 } 517 dData.NetworkSettings.Networks[network.Name] = network 518 } else { 519 logger.Debugf("No virtual IPs found in network %s", virtualIP.NetworkID) 520 } 521 } else { 522 logger.Debugf("Network not found, id: %s", virtualIP.NetworkID) 523 } 524 } 525 } 526 } 527 return dData, nil 528} 529 530func listTasks(ctx context.Context, dockerClient client.APIClient, serviceID string, 531 serviceDockerData dockerData, networkMap map[string]*dockertypes.NetworkResource, isGlobalSvc bool) ([]dockerData, error) { 532 serviceIDFilter := filters.NewArgs() 533 serviceIDFilter.Add("service", serviceID) 534 serviceIDFilter.Add("desired-state", "running") 535 536 taskList, err := dockerClient.TaskList(ctx, dockertypes.TaskListOptions{Filters: serviceIDFilter}) 537 if err != nil { 538 return nil, err 539 } 540 541 var dockerDataList []dockerData 542 for _, task := range taskList { 543 if task.Status.State != swarmtypes.TaskStateRunning { 544 continue 545 } 546 dData := parseTasks(ctx, task, serviceDockerData, networkMap, isGlobalSvc) 547 if len(dData.NetworkSettings.Networks) > 0 { 548 dockerDataList = append(dockerDataList, dData) 549 } 550 } 551 return dockerDataList, err 552} 553 554func parseTasks(ctx context.Context, task swarmtypes.Task, serviceDockerData dockerData, 555 networkMap map[string]*dockertypes.NetworkResource, isGlobalSvc bool) dockerData { 556 dData := dockerData{ 557 ID: task.ID, 558 ServiceName: serviceDockerData.Name, 559 Name: serviceDockerData.Name + "." + strconv.Itoa(task.Slot), 560 Labels: serviceDockerData.Labels, 561 ExtraConf: serviceDockerData.ExtraConf, 562 NetworkSettings: networkSettings{}, 563 } 564 565 if isGlobalSvc { 566 dData.Name = serviceDockerData.Name + "." + task.ID 567 } 568 569 if task.NetworksAttachments != nil { 570 dData.NetworkSettings.Networks = make(map[string]*networkData) 571 for _, virtualIP := range task.NetworksAttachments { 572 if networkService, present := networkMap[virtualIP.Network.ID]; present { 573 if len(virtualIP.Addresses) > 0 { 574 // Not sure about this next loop - when would a task have multiple IP's for the same network? 575 for _, addr := range virtualIP.Addresses { 576 ip, _, _ := net.ParseCIDR(addr) 577 network := &networkData{ 578 ID: virtualIP.Network.ID, 579 Name: networkService.Name, 580 Addr: ip.String(), 581 } 582 dData.NetworkSettings.Networks[network.Name] = network 583 } 584 } else { 585 log.FromContext(ctx).Debugf("No IP addresses found for network %s", virtualIP.Network.ID) 586 } 587 } 588 } 589 } 590 return dData 591} 592