1# -*- coding: utf-8 -*- 2# Copyright 2011 Google Inc. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Implementation of acl command for cloud storage providers.""" 16 17from __future__ import absolute_import 18from __future__ import print_function 19from __future__ import division 20from __future__ import unicode_literals 21 22from apitools.base.py import encoding 23from gslib import metrics 24from gslib.cloud_api import AccessDeniedException 25from gslib.cloud_api import BadRequestException 26from gslib.cloud_api import PreconditionException 27from gslib.cloud_api import Preconditions 28from gslib.cloud_api import ServiceException 29from gslib.command import Command 30from gslib.command import SetAclExceptionHandler 31from gslib.command import SetAclFuncWrapper 32from gslib.command_argument import CommandArgument 33from gslib.cs_api_map import ApiSelector 34from gslib.exception import CommandException 35from gslib.help_provider import CreateHelpText 36from gslib.storage_url import StorageUrlFromString 37from gslib.storage_url import UrlsAreForSingleProvider 38from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages 39from gslib.utils import acl_helper 40from gslib.utils.constants import NO_MAX 41from gslib.utils.retry_util import Retry 42 43_SET_SYNOPSIS = """ 44 gsutil acl set [-f] [-r] [-a] <file-or-canned_acl_name> url... 45""" 46 47_GET_SYNOPSIS = """ 48 gsutil acl get url 49""" 50 51_CH_SYNOPSIS = """ 52 gsutil acl ch [-f] [-r] <grant>... url... 53 54 where each <grant> is one of the following forms: 55 56 -u <id>|<email>:<perm> 57 -g <id>|<email>|<domain>|All|AllAuth:<perm> 58 -p (viewers|editors|owners)-<project number>:<perm> 59 -d <id>|<email>|<domain>|All|AllAuth|(viewers|editors|owners)-<project number> 60""" 61 62_GET_DESCRIPTION = """ 63<B>GET</B> 64 The "acl get" command gets the ACL text for a bucket or object, which you can 65 save and edit for the acl set command. 66""" 67 68_SET_DESCRIPTION = """ 69<B>SET</B> 70 The "acl set" command allows you to set an Access Control List on one or 71 more buckets and objects. The file-or-canned_acl_name parameter names either 72 a canned ACL or the path to a file that contains ACL text. The simplest way 73 to use the "acl set" command is to specify one of the canned ACLs, e.g.,: 74 75 gsutil acl set private gs://bucket 76 77 If you want to make an object or bucket publicly readable or writable, it is 78 recommended to use "acl ch", to avoid accidentally removing OWNER permissions. 79 See the "acl ch" section for details. 80 81 See `Predefined ACLs 82 <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ 83 for a list of canned ACLs. 84 85 If you want to define more fine-grained control over your data, you can 86 retrieve an ACL using the "acl get" command, save the output to a file, edit 87 the file, and then use the "acl set" command to set that ACL on the buckets 88 and/or objects. For example: 89 90 gsutil acl get gs://bucket/file.txt > acl.txt 91 92 Make changes to acl.txt such as adding an additional grant, then: 93 94 gsutil acl set acl.txt gs://cats/file.txt 95 96 Note that you can set an ACL on multiple buckets or objects at once. For 97 example, to set ACLs on all .jpg files found in a bucket: 98 99 gsutil acl set acl.txt gs://bucket/**.jpg 100 101 If you have a large number of ACLs to update you might want to use the 102 gsutil -m option, to perform a parallel (multi-threaded/multi-processing) 103 update: 104 105 gsutil -m acl set acl.txt gs://bucket/**.jpg 106 107 Note that multi-threading/multi-processing is only done when the named URLs 108 refer to objects, which happens either if you name specific objects or 109 if you enumerate objects by using an object wildcard or specifying 110 the acl -r flag. 111 112 113<B>SET OPTIONS</B> 114 The "set" sub-command has the following options 115 116 -R, -r Performs "acl set" request recursively, to all objects under 117 the specified URL. 118 119 -a Performs "acl set" request on all object versions. 120 121 -f Normally gsutil stops at the first error. The -f option causes 122 it to continue when it encounters errors. If some of the ACLs 123 couldn't be set, gsutil's exit status will be non-zero even if 124 this flag is set. This option is implicitly set when running 125 "gsutil -m acl...". 126""" 127 128_CH_DESCRIPTION = """ 129<B>CH</B> 130 The "acl ch" (or "acl change") command updates access control lists, similar 131 in spirit to the Linux chmod command. You can specify multiple access grant 132 additions and deletions in a single command run; all changes will be made 133 atomically to each object in turn. For example, if the command requests 134 deleting one grant and adding a different grant, the ACLs being updated will 135 never be left in an intermediate state where one grant has been deleted but 136 the second grant not yet added. Each change specifies a user or group grant 137 to add or delete, and for grant additions, one of R, W, O (for the 138 permission to be granted). A more formal description is provided in a later 139 section; below we provide examples. 140 141<B>CH EXAMPLES</B> 142 Examples for "ch" sub-command: 143 144 Grant anyone on the internet READ access to the object example-object: 145 146 gsutil acl ch -u AllUsers:R gs://example-bucket/example-object 147 148 NOTE: By default, publicly readable objects are served with a Cache-Control 149 header allowing such objects to be cached for 3600 seconds. If you need to 150 ensure that updates become visible immediately, you should set a 151 Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on 152 such objects. For help doing this, see "gsutil help setmeta". 153 154 Grant anyone on the internet WRITE access to the bucket example-bucket: 155 156 WARNING: this is not recommended as you will be responsible for the content 157 158 gsutil acl ch -u AllUsers:W gs://example-bucket 159 160 Grant the user john.doe@example.com WRITE access to the bucket 161 example-bucket: 162 163 gsutil acl ch -u john.doe@example.com:WRITE gs://example-bucket 164 165 Grant the group admins@example.com OWNER access to all jpg files in 166 example-bucket: 167 168 gsutil acl ch -g admins@example.com:O gs://example-bucket/**.jpg 169 170 Grant the owners of project example-project WRITE access to the bucket 171 example-bucket: 172 173 gsutil acl ch -p owners-example-project:W gs://example-bucket 174 175 NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access 176 to a project's viewers/editors respectively. 177 178 Remove access to the bucket example-bucket for the viewers of project number 179 12345: 180 181 gsutil acl ch -d viewers-12345 gs://example-bucket 182 183 NOTE: You cannot remove the project owners group from ACLs of gs:// buckets in 184 the given project. Attempts to do so will appear to succeed, but the service 185 will add the project owners group into the new set of ACLs before applying it. 186 187 Note that removing a project requires you to reference the project by 188 its number (which you can see with the acl get command) as opposed to its 189 project ID string. 190 191 Grant the user with the specified canonical ID READ access to all objects 192 in example-bucket that begin with folder/: 193 194 gsutil acl ch -r \\ 195 -u 84fac329bceSAMPLE777d5d22b8SAMPLE785ac2SAMPLE2dfcf7c4adf34da46:R \\ 196 gs://example-bucket/folder/ 197 198 Grant the service account foo@developer.gserviceaccount.com WRITE access to 199 the bucket example-bucket: 200 201 gsutil acl ch -u foo@developer.gserviceaccount.com:W gs://example-bucket 202 203 Grant all users from the `G Suite 204 <https://www.google.com/work/apps/business/>`_ domain my-domain.org READ 205 access to the bucket gcs.my-domain.org: 206 207 gsutil acl ch -g my-domain.org:R gs://gcs.my-domain.org 208 209 Remove any current access by john.doe@example.com from the bucket 210 example-bucket: 211 212 gsutil acl ch -d john.doe@example.com gs://example-bucket 213 214 If you have a large number of objects to update, enabling multi-threading 215 with the gsutil -m flag can significantly improve performance. The 216 following command adds OWNER for admin@example.org using 217 multi-threading: 218 219 gsutil -m acl ch -r -u admin@example.org:O gs://example-bucket 220 221 Grant READ access to everyone from my-domain.org and to all authenticated 222 users, and grant OWNER to admin@mydomain.org, for the buckets 223 my-bucket and my-other-bucket, with multi-threading enabled: 224 225 gsutil -m acl ch -r -g my-domain.org:R -g AllAuth:R \\ 226 -u admin@mydomain.org:O gs://my-bucket/ gs://my-other-bucket 227 228<B>CH ROLES</B> 229 You may specify the following roles with either their shorthand or 230 their full name: 231 232 R: READ 233 W: WRITE 234 O: OWNER 235 236 For more information on these roles and the access they grant, see the 237 permissions section of the `Access Control Lists page 238 <https://cloud.google.com/storage/docs/access-control/lists#permissions>`_. 239 240<B>CH ENTITIES</B> 241 There are four different entity types: Users, Groups, All Authenticated Users, 242 and All Users. 243 244 Users are added with -u and a plain ID or email address, as in 245 "-u john-doe@gmail.com:r". Note: Service Accounts are considered to be users. 246 247 Groups are like users, but specified with the -g flag, as in 248 "-g power-users@example.com:fc". Groups may also be specified as a full 249 domain, as in "-g my-company.com:r". 250 251 AllAuthenticatedUsers and AllUsers are specified directly, as 252 in "-g AllUsers:R" or "-g AllAuthenticatedUsers:O". These are case 253 insensitive, and may be shortened to "all" and "allauth", respectively. 254 255 Removing roles is specified with the -d flag and an ID, email 256 address, domain, or one of AllUsers or AllAuthenticatedUsers. 257 258 Many entities' roles can be specified on the same command line, allowing 259 bundled changes to be executed in a single run. This will reduce the number of 260 requests made to the server. 261 262<B>CH OPTIONS</B> 263 The "ch" sub-command has the following options 264 265 -d Remove all roles associated with the matching entity. 266 267 -f Normally gsutil stops at the first error. The -f option causes 268 it to continue when it encounters errors. With this option the 269 gsutil exit status will be 0 even if some ACLs couldn't be 270 changed. 271 272 -g Add or modify a group entity's role. 273 274 -p Add or modify a project viewers/editors/owners role. 275 276 -R, -r Performs acl ch request recursively, to all objects under the 277 specified URL. 278 279 -u Add or modify a user entity's role. 280""" 281 282_SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + 283 _CH_SYNOPSIS.lstrip('\n') + '\n\n') 284 285_DESCRIPTION = (""" 286 The acl command has three sub-commands: 287""" + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION, _CH_DESCRIPTION])) 288 289_DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) 290 291_get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) 292_set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) 293_ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) 294 295 296def _ApplyExceptionHandler(cls, exception): 297 cls.logger.error('Encountered a problem: %s', exception) 298 cls.everything_set_okay = False 299 300 301def _ApplyAclChangesWrapper(cls, url_or_expansion_result, thread_state=None): 302 cls.ApplyAclChanges(url_or_expansion_result, thread_state=thread_state) 303 304 305class AclCommand(Command): 306 """Implementation of gsutil acl command.""" 307 308 # Command specification. See base class for documentation. 309 command_spec = Command.CreateCommandSpec( 310 'acl', 311 command_name_aliases=['getacl', 'setacl', 'chacl'], 312 usage_synopsis=_SYNOPSIS, 313 min_args=2, 314 max_args=NO_MAX, 315 supported_sub_args='afRrg:u:d:p:', 316 file_url_ok=False, 317 provider_url_ok=False, 318 urls_start_arg=1, 319 gs_api_support=[ApiSelector.XML, ApiSelector.JSON], 320 gs_default_api=ApiSelector.JSON, 321 argparse_arguments={ 322 'set': [ 323 CommandArgument.MakeFileURLOrCannedACLArgument(), 324 CommandArgument.MakeZeroOrMoreCloudURLsArgument() 325 ], 326 'get': [CommandArgument.MakeNCloudURLsArgument(1)], 327 'ch': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 328 }) 329 # Help specification. See help_provider.py for documentation. 330 help_spec = Command.HelpSpec( 331 help_name='acl', 332 help_name_aliases=['getacl', 'setacl', 'chmod', 'chacl'], 333 help_type='command_help', 334 help_one_line_summary='Get, set, or change bucket and/or object ACLs', 335 help_text=_DETAILED_HELP_TEXT, 336 subcommand_help_text={ 337 'get': _get_help_text, 338 'set': _set_help_text, 339 'ch': _ch_help_text 340 }, 341 ) 342 343 def _CalculateUrlsStartArg(self): 344 if not self.args: 345 self.RaiseWrongNumberOfArgumentsException() 346 if (self.args[0].lower() == 'set') or (self.command_alias_used == 'setacl'): 347 return 1 348 else: 349 return 0 350 351 def _SetAcl(self): 352 """Parses options and sets ACLs on the specified buckets/objects.""" 353 self.continue_on_error = False 354 if self.sub_opts: 355 for o, unused_a in self.sub_opts: 356 if o == '-a': 357 self.all_versions = True 358 elif o == '-f': 359 self.continue_on_error = True 360 elif o == '-r' or o == '-R': 361 self.recursion_requested = True 362 else: 363 self.RaiseInvalidArgumentException() 364 try: 365 self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler) 366 except AccessDeniedException as unused_e: 367 self._WarnServiceAccounts() 368 raise 369 if not self.everything_set_okay: 370 raise CommandException('ACLs for some objects could not be set.') 371 372 def _ChAcl(self): 373 """Parses options and changes ACLs on the specified buckets/objects.""" 374 self.parse_versions = True 375 self.changes = [] 376 self.continue_on_error = False 377 378 if self.sub_opts: 379 for o, a in self.sub_opts: 380 if o == '-f': 381 self.continue_on_error = True 382 elif o == '-g': 383 if 'gserviceaccount.com' in a: 384 raise CommandException( 385 'Service accounts are considered users, not groups; please use ' 386 '"gsutil acl ch -u" instead of "gsutil acl ch -g"') 387 self.changes.append( 388 acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.GROUP)) 389 elif o == '-p': 390 self.changes.append( 391 acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.PROJECT)) 392 elif o == '-u': 393 self.changes.append( 394 acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.USER)) 395 elif o == '-d': 396 self.changes.append(acl_helper.AclDel(a)) 397 elif o == '-r' or o == '-R': 398 self.recursion_requested = True 399 else: 400 self.RaiseInvalidArgumentException() 401 402 if not self.changes: 403 raise CommandException('Please specify at least one access change ' 404 'with the -g, -u, or -d flags') 405 406 if (not UrlsAreForSingleProvider(self.args) or 407 StorageUrlFromString(self.args[0]).scheme != 'gs'): 408 raise CommandException( 409 'The "{0}" command can only be used with gs:// URLs'.format( 410 self.command_name)) 411 412 self.everything_set_okay = True 413 self.ApplyAclFunc(_ApplyAclChangesWrapper, 414 _ApplyExceptionHandler, 415 self.args, 416 object_fields=['acl', 'generation', 'metageneration']) 417 if not self.everything_set_okay: 418 raise CommandException('ACLs for some objects could not be set.') 419 420 def _RaiseForAccessDenied(self, url): 421 self._WarnServiceAccounts() 422 raise CommandException('Failed to set acl for %s. Please ensure you have ' 423 'OWNER-role access to this resource.' % url) 424 425 @Retry(ServiceException, tries=3, timeout_secs=1) 426 def ApplyAclChanges(self, name_expansion_result, thread_state=None): 427 """Applies the changes in self.changes to the provided URL. 428 429 Args: 430 name_expansion_result: NameExpansionResult describing the target object. 431 thread_state: If present, gsutil Cloud API instance to apply the changes. 432 """ 433 if thread_state: 434 gsutil_api = thread_state 435 else: 436 gsutil_api = self.gsutil_api 437 438 url = name_expansion_result.expanded_storage_url 439 if url.IsBucket(): 440 bucket = gsutil_api.GetBucket(url.bucket_name, 441 provider=url.scheme, 442 fields=['acl', 'metageneration']) 443 current_acl = bucket.acl 444 elif url.IsObject(): 445 gcs_object = encoding.JsonToMessage(apitools_messages.Object, 446 name_expansion_result.expanded_result) 447 current_acl = gcs_object.acl 448 449 if not current_acl: 450 self._RaiseForAccessDenied(url) 451 if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: 452 self.logger.info('No changes to %s', url) 453 return 454 455 try: 456 if url.IsBucket(): 457 preconditions = Preconditions(meta_gen_match=bucket.metageneration) 458 bucket_metadata = apitools_messages.Bucket(acl=current_acl) 459 gsutil_api.PatchBucket(url.bucket_name, 460 bucket_metadata, 461 preconditions=preconditions, 462 provider=url.scheme, 463 fields=['id']) 464 else: # Object 465 preconditions = Preconditions(gen_match=gcs_object.generation, 466 meta_gen_match=gcs_object.metageneration) 467 object_metadata = apitools_messages.Object(acl=current_acl) 468 try: 469 gsutil_api.PatchObjectMetadata(url.bucket_name, 470 url.object_name, 471 object_metadata, 472 preconditions=preconditions, 473 provider=url.scheme, 474 generation=url.generation, 475 fields=['id']) 476 except PreconditionException as e: 477 # Special retry case where we want to do an additional step, the read 478 # of the read-modify-write cycle, to fetch the correct object 479 # metadata before reattempting ACL changes. 480 self._RefetchObjectMetadataAndApplyAclChanges(url, gsutil_api) 481 482 self.logger.info('Updated ACL on %s', url) 483 except BadRequestException as e: 484 # Don't retry on bad requests, e.g. invalid email address. 485 raise CommandException('Received bad request from server: %s' % str(e)) 486 except AccessDeniedException: 487 self._RaiseForAccessDenied(url) 488 except PreconditionException as e: 489 # For objects, retry attempts should have already been handled. 490 if url.IsObject(): 491 raise CommandException(str(e)) 492 # For buckets, raise PreconditionException and continue to next retry. 493 raise e 494 495 @Retry(PreconditionException, tries=3, timeout_secs=1) 496 def _RefetchObjectMetadataAndApplyAclChanges(self, url, gsutil_api): 497 """Reattempts object ACL changes after a PreconditionException.""" 498 gcs_object = gsutil_api.GetObjectMetadata( 499 url.bucket_name, 500 url.object_name, 501 provider=url.scheme, 502 fields=['acl', 'generation', 'metageneration']) 503 current_acl = gcs_object.acl 504 505 if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: 506 self.logger.info('No changes to %s', url) 507 return 508 509 object_metadata = apitools_messages.Object(acl=current_acl) 510 preconditions = Preconditions(gen_match=gcs_object.generation, 511 meta_gen_match=gcs_object.metageneration) 512 gsutil_api.PatchObjectMetadata(url.bucket_name, 513 url.object_name, 514 object_metadata, 515 preconditions=preconditions, 516 provider=url.scheme, 517 generation=gcs_object.generation, 518 fields=['id']) 519 520 def _ApplyAclChangesAndReturnChangeCount(self, storage_url, acl_message): 521 modification_count = 0 522 for change in self.changes: 523 modification_count += change.Execute(storage_url, acl_message, 'acl', 524 self.logger) 525 return modification_count 526 527 def RunCommand(self): 528 """Command entry point for the acl command.""" 529 action_subcommand = self.args.pop(0) 530 self.ParseSubOpts(check_args=True) 531 532 # Commands with both suboptions and subcommands need to reparse for 533 # suboptions, so we log again. 534 metrics.LogCommandParams(sub_opts=self.sub_opts) 535 self.def_acl = False 536 if action_subcommand == 'get': 537 metrics.LogCommandParams(subcommands=[action_subcommand]) 538 self.GetAndPrintAcl(self.args[0]) 539 elif action_subcommand == 'set': 540 metrics.LogCommandParams(subcommands=[action_subcommand]) 541 self._SetAcl() 542 elif action_subcommand in ('ch', 'change'): 543 metrics.LogCommandParams(subcommands=[action_subcommand]) 544 self._ChAcl() 545 else: 546 raise CommandException( 547 ('Invalid subcommand "%s" for the %s command.\n' 548 'See "gsutil help acl".') % (action_subcommand, self.command_name)) 549 550 return 0 551