1"""Functions for accessing docker via the docker cli.""" 2from __future__ import (absolute_import, division, print_function) 3__metaclass__ = type 4 5import json 6import os 7import time 8 9from .io import ( 10 open_binary_file, 11 read_text_file, 12) 13 14from .util import ( 15 ApplicationError, 16 common_environment, 17 display, 18 find_executable, 19 SubprocessError, 20) 21 22from .http import ( 23 urlparse, 24) 25 26from .util_common import ( 27 run_command, 28) 29 30from .config import ( 31 EnvironmentConfig, 32) 33 34BUFFER_SIZE = 256 * 256 35 36 37def docker_available(): 38 """ 39 :rtype: bool 40 """ 41 return find_executable('docker', required=False) 42 43 44def get_docker_hostname(): # type: () -> str 45 """Return the hostname of the Docker service.""" 46 try: 47 return get_docker_hostname.hostname 48 except AttributeError: 49 pass 50 51 docker_host = os.environ.get('DOCKER_HOST') 52 53 if docker_host and docker_host.startswith('tcp://'): 54 try: 55 hostname = urlparse(docker_host)[1].split(':')[0] 56 display.info('Detected Docker host: %s' % hostname, verbosity=1) 57 except ValueError: 58 hostname = 'localhost' 59 display.warning('Could not parse DOCKER_HOST environment variable "%s", falling back to localhost.' % docker_host) 60 else: 61 hostname = 'localhost' 62 display.info('Assuming Docker is available on localhost.', verbosity=1) 63 64 get_docker_hostname.hostname = hostname 65 66 return hostname 67 68 69def get_docker_container_id(): 70 """ 71 :rtype: str | None 72 """ 73 try: 74 return get_docker_container_id.container_id 75 except AttributeError: 76 pass 77 78 path = '/proc/self/cpuset' 79 container_id = None 80 81 if os.path.exists(path): 82 # File content varies based on the environment: 83 # No Container: / 84 # Docker: /docker/c86f3732b5ba3d28bb83b6e14af767ab96abbc52de31313dcb1176a62d91a507 85 # Azure Pipelines (Docker): /azpl_job/0f2edfed602dd6ec9f2e42c867f4d5ee640ebf4c058e6d3196d4393bb8fd0891 86 # Podman: /../../../../../.. 87 contents = read_text_file(path) 88 89 cgroup_path, cgroup_name = os.path.split(contents.strip()) 90 91 if cgroup_path in ('/docker', '/azpl_job'): 92 container_id = cgroup_name 93 94 get_docker_container_id.container_id = container_id 95 96 if container_id: 97 display.info('Detected execution in Docker container: %s' % container_id, verbosity=1) 98 99 return container_id 100 101 102def get_docker_container_ip(args, container_id): 103 """ 104 :type args: EnvironmentConfig 105 :type container_id: str 106 :rtype: str 107 """ 108 results = docker_inspect(args, container_id) 109 network_settings = results[0]['NetworkSettings'] 110 networks = network_settings.get('Networks') 111 112 if networks: 113 network_name = get_docker_preferred_network_name(args) 114 115 if not network_name: 116 # Sort networks and use the first available. 117 # This assumes all containers will have access to the same networks. 118 network_name = sorted(networks.keys()).pop(0) 119 120 ipaddress = networks[network_name]['IPAddress'] 121 else: 122 # podman doesn't provide Networks, fall back to using IPAddress 123 ipaddress = network_settings['IPAddress'] 124 125 if not ipaddress: 126 raise ApplicationError('Cannot retrieve IP address for container: %s' % container_id) 127 128 return ipaddress 129 130 131def get_docker_network_name(args, container_id): # type: (EnvironmentConfig, str) -> str 132 """ 133 Return the network name of the specified container. 134 Raises an exception if zero or more than one network is found. 135 """ 136 networks = get_docker_networks(args, container_id) 137 138 if not networks: 139 raise ApplicationError('No network found for Docker container: %s.' % container_id) 140 141 if len(networks) > 1: 142 raise ApplicationError('Found multiple networks for Docker container %s instead of only one: %s' % (container_id, ', '.join(networks))) 143 144 return networks[0] 145 146 147def get_docker_preferred_network_name(args): # type: (EnvironmentConfig) -> str 148 """ 149 Return the preferred network name for use with Docker. The selection logic is: 150 - the network selected by the user with `--docker-network` 151 - the network of the currently running docker container (if any) 152 - the default docker network (returns None) 153 """ 154 network = None 155 156 if args.docker_network: 157 network = args.docker_network 158 else: 159 current_container_id = get_docker_container_id() 160 161 if current_container_id: 162 # Make sure any additional containers we launch use the same network as the current container we're running in. 163 # This is needed when ansible-test is running in a container that is not connected to Docker's default network. 164 network = get_docker_network_name(args, current_container_id) 165 166 return network 167 168 169def is_docker_user_defined_network(network): # type: (str) -> bool 170 """Return True if the network being used is a user-defined network.""" 171 return network and network != 'bridge' 172 173 174def get_docker_networks(args, container_id): 175 """ 176 :param args: EnvironmentConfig 177 :param container_id: str 178 :rtype: list[str] 179 """ 180 results = docker_inspect(args, container_id) 181 # podman doesn't return Networks- just silently return None if it's missing... 182 networks = results[0]['NetworkSettings'].get('Networks') 183 if networks is None: 184 return None 185 return sorted(networks) 186 187 188def docker_pull(args, image): 189 """ 190 :type args: EnvironmentConfig 191 :type image: str 192 """ 193 if ('@' in image or ':' in image) and docker_images(args, image): 194 display.info('Skipping docker pull of existing image with tag or digest: %s' % image, verbosity=2) 195 return 196 197 if not args.docker_pull: 198 display.warning('Skipping docker pull for "%s". Image may be out-of-date.' % image) 199 return 200 201 for _iteration in range(1, 10): 202 try: 203 docker_command(args, ['pull', image]) 204 return 205 except SubprocessError: 206 display.warning('Failed to pull docker image "%s". Waiting a few seconds before trying again.' % image) 207 time.sleep(3) 208 209 raise ApplicationError('Failed to pull docker image "%s".' % image) 210 211 212def docker_put(args, container_id, src, dst): 213 """ 214 :type args: EnvironmentConfig 215 :type container_id: str 216 :type src: str 217 :type dst: str 218 """ 219 # avoid 'docker cp' due to a bug which causes 'docker rm' to fail 220 with open_binary_file(src) as src_fd: 221 docker_exec(args, container_id, ['dd', 'of=%s' % dst, 'bs=%s' % BUFFER_SIZE], 222 options=['-i'], stdin=src_fd, capture=True) 223 224 225def docker_get(args, container_id, src, dst): 226 """ 227 :type args: EnvironmentConfig 228 :type container_id: str 229 :type src: str 230 :type dst: str 231 """ 232 # avoid 'docker cp' due to a bug which causes 'docker rm' to fail 233 with open_binary_file(dst, 'wb') as dst_fd: 234 docker_exec(args, container_id, ['dd', 'if=%s' % src, 'bs=%s' % BUFFER_SIZE], 235 options=['-i'], stdout=dst_fd, capture=True) 236 237 238def docker_run(args, image, options, cmd=None): 239 """ 240 :type args: EnvironmentConfig 241 :type image: str 242 :type options: list[str] | None 243 :type cmd: list[str] | None 244 :rtype: str | None, str | None 245 """ 246 if not options: 247 options = [] 248 249 if not cmd: 250 cmd = [] 251 252 network = get_docker_preferred_network_name(args) 253 254 if is_docker_user_defined_network(network): 255 # Only when the network is not the default bridge network. 256 # Using this with the default bridge network results in an error when using --link: links are only supported for user-defined networks 257 options.extend(['--network', network]) 258 259 for _iteration in range(1, 3): 260 try: 261 return docker_command(args, ['run'] + options + [image] + cmd, capture=True) 262 except SubprocessError as ex: 263 display.error(ex) 264 display.warning('Failed to run docker image "%s". Waiting a few seconds before trying again.' % image) 265 time.sleep(3) 266 267 raise ApplicationError('Failed to run docker image "%s".' % image) 268 269 270def docker_images(args, image): 271 """ 272 :param args: CommonConfig 273 :param image: str 274 :rtype: list[dict[str, any]] 275 """ 276 try: 277 stdout, _dummy = docker_command(args, ['images', image, '--format', '{{json .}}'], capture=True, always=True) 278 except SubprocessError as ex: 279 if 'no such image' in ex.stderr: 280 return [] # podman does not handle this gracefully, exits 125 281 282 if 'function "json" not defined' in ex.stderr: 283 # podman > 2 && < 2.2.0 breaks with --format {{json .}}, and requires --format json 284 # So we try this as a fallback. If it fails again, we just raise the exception and bail. 285 stdout, _dummy = docker_command(args, ['images', image, '--format', 'json'], capture=True, always=True) 286 else: 287 raise ex 288 289 if stdout.startswith('['): 290 # modern podman outputs a pretty-printed json list. Just load the whole thing. 291 return json.loads(stdout) 292 293 # docker outputs one json object per line (jsonl) 294 return [json.loads(line) for line in stdout.splitlines()] 295 296 297def docker_rm(args, container_id): 298 """ 299 :type args: EnvironmentConfig 300 :type container_id: str 301 """ 302 try: 303 docker_command(args, ['rm', '-f', container_id], capture=True) 304 except SubprocessError as ex: 305 if 'no such container' in ex.stderr: 306 pass # podman does not handle this gracefully, exits 1 307 else: 308 raise ex 309 310 311def docker_inspect(args, container_id): 312 """ 313 :type args: EnvironmentConfig 314 :type container_id: str 315 :rtype: list[dict] 316 """ 317 if args.explain: 318 return [] 319 320 try: 321 stdout = docker_command(args, ['inspect', container_id], capture=True)[0] 322 return json.loads(stdout) 323 except SubprocessError as ex: 324 if 'no such image' in ex.stderr: 325 return [] # podman does not handle this gracefully, exits 125 326 try: 327 return json.loads(ex.stdout) 328 except Exception: 329 raise ex 330 331 332def docker_network_disconnect(args, container_id, network): 333 """ 334 :param args: EnvironmentConfig 335 :param container_id: str 336 :param network: str 337 """ 338 docker_command(args, ['network', 'disconnect', network, container_id], capture=True) 339 340 341def docker_network_inspect(args, network): 342 """ 343 :type args: EnvironmentConfig 344 :type network: str 345 :rtype: list[dict] 346 """ 347 if args.explain: 348 return [] 349 350 try: 351 stdout = docker_command(args, ['network', 'inspect', network], capture=True)[0] 352 return json.loads(stdout) 353 except SubprocessError as ex: 354 try: 355 return json.loads(ex.stdout) 356 except Exception: 357 raise ex 358 359 360def docker_exec(args, container_id, cmd, options=None, capture=False, stdin=None, stdout=None): 361 """ 362 :type args: EnvironmentConfig 363 :type container_id: str 364 :type cmd: list[str] 365 :type options: list[str] | None 366 :type capture: bool 367 :type stdin: BinaryIO | None 368 :type stdout: BinaryIO | None 369 :rtype: str | None, str | None 370 """ 371 if not options: 372 options = [] 373 374 return docker_command(args, ['exec'] + options + [container_id] + cmd, capture=capture, stdin=stdin, stdout=stdout) 375 376 377def docker_info(args): 378 """ 379 :param args: CommonConfig 380 :rtype: dict[str, any] 381 """ 382 stdout, _dummy = docker_command(args, ['info', '--format', '{{json .}}'], capture=True, always=True) 383 return json.loads(stdout) 384 385 386def docker_version(args): 387 """ 388 :param args: CommonConfig 389 :rtype: dict[str, any] 390 """ 391 stdout, _dummy = docker_command(args, ['version', '--format', '{{json .}}'], capture=True, always=True) 392 return json.loads(stdout) 393 394 395def docker_command(args, cmd, capture=False, stdin=None, stdout=None, always=False): 396 """ 397 :type args: CommonConfig 398 :type cmd: list[str] 399 :type capture: bool 400 :type stdin: file | None 401 :type stdout: file | None 402 :type always: bool 403 :rtype: str | None, str | None 404 """ 405 env = docker_environment() 406 return run_command(args, ['docker'] + cmd, env=env, capture=capture, stdin=stdin, stdout=stdout, always=always) 407 408 409def docker_environment(): 410 """ 411 :rtype: dict[str, str] 412 """ 413 env = common_environment() 414 env.update(dict((key, os.environ[key]) for key in os.environ if key.startswith('DOCKER_'))) 415 return env 416