1# -*- coding: utf-8 -*- 2# Copyright 2011 Google Inc. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Implementation of acl command for cloud storage providers.""" 16 17from __future__ import absolute_import 18from __future__ import print_function 19from __future__ import division 20from __future__ import unicode_literals 21 22from apitools.base.py import encoding 23from gslib import metrics 24from gslib.cloud_api import AccessDeniedException 25from gslib.cloud_api import BadRequestException 26from gslib.cloud_api import PreconditionException 27from gslib.cloud_api import Preconditions 28from gslib.cloud_api import ServiceException 29from gslib.command import Command 30from gslib.command import SetAclExceptionHandler 31from gslib.command import SetAclFuncWrapper 32from gslib.command_argument import CommandArgument 33from gslib.cs_api_map import ApiSelector 34from gslib.exception import CommandException 35from gslib.help_provider import CreateHelpText 36from gslib.storage_url import StorageUrlFromString 37from gslib.storage_url import UrlsAreForSingleProvider 38from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages 39from gslib.utils import acl_helper 40from gslib.utils.constants import NO_MAX 41from gslib.utils.retry_util import Retry 42 43_SET_SYNOPSIS = """ 44 gsutil acl set [-f] [-r] [-a] file-or-canned_acl_name url... 45""" 46 47_GET_SYNOPSIS = """ 48 gsutil acl get url 49""" 50 51_CH_SYNOPSIS = """ 52 gsutil acl ch [-f] [-r] <grant>... url... 53 54 where each <grant> is one of the following forms: 55 56 -u <id|email>:<perm> 57 -g <id|email|domain|All|AllAuth>:<perm> 58 -p <viewers|editors|owners>-<project number>:<perm> 59 -d <id|email|domain|All|AllAuth|<viewers|editors|owners>-<project number>>:<perm> 60""" 61 62_GET_DESCRIPTION = """ 63<B>GET</B> 64 The "acl get" command gets the ACL text for a bucket or object, which you can 65 save and edit for the acl set command. 66""" 67 68_SET_DESCRIPTION = """ 69<B>SET</B> 70 The "acl set" command allows you to set an Access Control List on one or 71 more buckets and objects. The simplest way to use it is to specify one of 72 the canned ACLs, e.g.,: 73 74 gsutil acl set private gs://bucket 75 76 If you want to make an object or bucket publicly readable or writable, it is 77 recommended to use "acl ch", to avoid accidentally removing OWNER permissions. 78 See the "acl ch" section for details. 79 80 See `Predefined ACLs 81 <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ 82 for a list of canned ACLs. 83 84 If you want to define more fine-grained control over your data, you can 85 retrieve an ACL using the "acl get" command, save the output to a file, edit 86 the file, and then use the "acl set" command to set that ACL on the buckets 87 and/or objects. For example: 88 89 gsutil acl get gs://bucket/file.txt > acl.txt 90 91 Make changes to acl.txt such as adding an additional grant, then: 92 93 gsutil acl set acl.txt gs://cats/file.txt 94 95 Note that you can set an ACL on multiple buckets or objects at once, 96 for example: 97 98 gsutil acl set acl.txt gs://bucket/*.jpg 99 100 If you have a large number of ACLs to update you might want to use the 101 gsutil -m option, to perform a parallel (multi-threaded/multi-processing) 102 update: 103 104 gsutil -m acl set acl.txt gs://bucket/*.jpg 105 106 Note that multi-threading/multi-processing is only done when the named URLs 107 refer to objects, which happens either if you name specific objects or 108 if you enumerate objects by using an object wildcard or specifying 109 the acl -r flag. 110 111 112<B>SET OPTIONS</B> 113 The "set" sub-command has the following options 114 115 -R, -r Performs "acl set" request recursively, to all objects under 116 the specified URL. 117 118 -a Performs "acl set" request on all object versions. 119 120 -f Normally gsutil stops at the first error. The -f option causes 121 it to continue when it encounters errors. If some of the ACLs 122 couldn't be set, gsutil's exit status will be non-zero even if 123 this flag is set. This option is implicitly set when running 124 "gsutil -m acl...". 125""" 126 127_CH_DESCRIPTION = """ 128<B>CH</B> 129 The "acl ch" (or "acl change") command updates access control lists, similar 130 in spirit to the Linux chmod command. You can specify multiple access grant 131 additions and deletions in a single command run; all changes will be made 132 atomically to each object in turn. For example, if the command requests 133 deleting one grant and adding a different grant, the ACLs being updated will 134 never be left in an intermediate state where one grant has been deleted but 135 the second grant not yet added. Each change specifies a user or group grant 136 to add or delete, and for grant additions, one of R, W, O (for the 137 permission to be granted). A more formal description is provided in a later 138 section; below we provide examples. 139 140<B>CH EXAMPLES</B> 141 Examples for "ch" sub-command: 142 143 Grant anyone on the internet READ access to the object example-object: 144 145 gsutil acl ch -u AllUsers:R gs://example-bucket/example-object 146 147 NOTE: By default, publicly readable objects are served with a Cache-Control 148 header allowing such objects to be cached for 3600 seconds. If you need to 149 ensure that updates become visible immediately, you should set a 150 Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on 151 such objects. For help doing this, see "gsutil help setmeta". 152 153 Grant anyone on the internet WRITE access to the bucket example-bucket 154 (WARNING: this is not recommended as you will be responsible for the content): 155 156 gsutil acl ch -u AllUsers:W gs://example-bucket 157 158 Grant the user john.doe@example.com WRITE access to the bucket 159 example-bucket: 160 161 gsutil acl ch -u john.doe@example.com:WRITE gs://example-bucket 162 163 Grant the group admins@example.com OWNER access to all jpg files in 164 the top level of example-bucket: 165 166 gsutil acl ch -g admins@example.com:O gs://example-bucket/*.jpg 167 168 Grant the owners of project example-project WRITE access to the bucket 169 example-bucket: 170 171 gsutil acl ch -p owners-example-project:W gs://example-bucket 172 173 NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access 174 to a project's viewers/editors respectively. 175 176 Remove access to the bucket example-bucket for the viewers of project number 177 12345: 178 179 gsutil acl ch -d viewers-12345 gs://example-bucket 180 181 NOTE: You cannot remove the project owners group from ACLs of gs:// buckets in 182 the given project. Attempts to do so will appear to succeed, but the service 183 will add the project owners group into the new set of ACLs before applying it. 184 185 Note that removing a project requires you to reference the project by 186 its number (which you can see with the acl get command) as opposed to its 187 project ID string. 188 189 Grant the user with the specified canonical ID READ access to all objects 190 in example-bucket that begin with folder/: 191 192 gsutil acl ch -r \\ 193 -u 84fac329bceSAMPLE777d5d22b8SAMPLE785ac2SAMPLE2dfcf7c4adf34da46:R \\ 194 gs://example-bucket/folder/ 195 196 Grant the service account foo@developer.gserviceaccount.com WRITE access to 197 the bucket example-bucket: 198 199 gsutil acl ch -u foo@developer.gserviceaccount.com:W gs://example-bucket 200 201 Grant all users from the `G Suite 202 <https://www.google.com/work/apps/business/>`_ domain my-domain.org READ 203 access to the bucket gcs.my-domain.org: 204 205 gsutil acl ch -g my-domain.org:R gs://gcs.my-domain.org 206 207 Remove any current access by john.doe@example.com from the bucket 208 example-bucket: 209 210 gsutil acl ch -d john.doe@example.com gs://example-bucket 211 212 If you have a large number of objects to update, enabling multi-threading 213 with the gsutil -m flag can significantly improve performance. The 214 following command adds OWNER for admin@example.org using 215 multi-threading: 216 217 gsutil -m acl ch -r -u admin@example.org:O gs://example-bucket 218 219 Grant READ access to everyone from my-domain.org and to all authenticated 220 users, and grant OWNER to admin@mydomain.org, for the buckets 221 my-bucket and my-other-bucket, with multi-threading enabled: 222 223 gsutil -m acl ch -r -g my-domain.org:R -g AllAuth:R \\ 224 -u admin@mydomain.org:O gs://my-bucket/ gs://my-other-bucket 225 226<B>CH ROLES</B> 227 You may specify the following roles with either their shorthand or 228 their full name: 229 230 R: READ 231 W: WRITE 232 O: OWNER 233 234 For more information on these roles and the access they grant, see the 235 permissions section of the `Access Control Lists page 236 <https://cloud.google.com/storage/docs/access-control/lists#permissions>`_. 237 238<B>CH ENTITIES</B> 239 There are four different entity types: Users, Groups, All Authenticated Users, 240 and All Users. 241 242 Users are added with -u and a plain ID or email address, as in 243 "-u john-doe@gmail.com:r". Note: Service Accounts are considered to be users. 244 245 Groups are like users, but specified with the -g flag, as in 246 "-g power-users@example.com:fc". Groups may also be specified as a full 247 domain, as in "-g my-company.com:r". 248 249 AllAuthenticatedUsers and AllUsers are specified directly, as 250 in "-g AllUsers:R" or "-g AllAuthenticatedUsers:O". These are case 251 insensitive, and may be shortened to "all" and "allauth", respectively. 252 253 Removing roles is specified with the -d flag and an ID, email 254 address, domain, or one of AllUsers or AllAuthenticatedUsers. 255 256 Many entities' roles can be specified on the same command line, allowing 257 bundled changes to be executed in a single run. This will reduce the number of 258 requests made to the server. 259 260<B>CH OPTIONS</B> 261 The "ch" sub-command has the following options 262 263 -d Remove all roles associated with the matching entity. 264 265 -f Normally gsutil stops at the first error. The -f option causes 266 it to continue when it encounters errors. With this option the 267 gsutil exit status will be 0 even if some ACLs couldn't be 268 changed. 269 270 -g Add or modify a group entity's role. 271 272 -p Add or modify a project viewers/editors/owners role. 273 274 -R, -r Performs acl ch request recursively, to all objects under the 275 specified URL. 276 277 -u Add or modify a user entity's role. 278""" 279 280_SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + 281 _CH_SYNOPSIS.lstrip('\n') + '\n\n') 282 283_DESCRIPTION = (""" 284 The acl command has three sub-commands: 285""" + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION, _CH_DESCRIPTION])) 286 287_DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) 288 289_get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) 290_set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) 291_ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) 292 293 294def _ApplyExceptionHandler(cls, exception): 295 cls.logger.error('Encountered a problem: %s', exception) 296 cls.everything_set_okay = False 297 298 299def _ApplyAclChangesWrapper(cls, url_or_expansion_result, thread_state=None): 300 cls.ApplyAclChanges(url_or_expansion_result, thread_state=thread_state) 301 302 303class AclCommand(Command): 304 """Implementation of gsutil acl command.""" 305 306 # Command specification. See base class for documentation. 307 command_spec = Command.CreateCommandSpec( 308 'acl', 309 command_name_aliases=['getacl', 'setacl', 'chacl'], 310 usage_synopsis=_SYNOPSIS, 311 min_args=2, 312 max_args=NO_MAX, 313 supported_sub_args='afRrg:u:d:p:', 314 file_url_ok=False, 315 provider_url_ok=False, 316 urls_start_arg=1, 317 gs_api_support=[ApiSelector.XML, ApiSelector.JSON], 318 gs_default_api=ApiSelector.JSON, 319 argparse_arguments={ 320 'set': [ 321 CommandArgument.MakeFileURLOrCannedACLArgument(), 322 CommandArgument.MakeZeroOrMoreCloudURLsArgument() 323 ], 324 'get': [CommandArgument.MakeNCloudURLsArgument(1)], 325 'ch': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 326 }) 327 # Help specification. See help_provider.py for documentation. 328 help_spec = Command.HelpSpec( 329 help_name='acl', 330 help_name_aliases=['getacl', 'setacl', 'chmod', 'chacl'], 331 help_type='command_help', 332 help_one_line_summary='Get, set, or change bucket and/or object ACLs', 333 help_text=_DETAILED_HELP_TEXT, 334 subcommand_help_text={ 335 'get': _get_help_text, 336 'set': _set_help_text, 337 'ch': _ch_help_text 338 }, 339 ) 340 341 def _CalculateUrlsStartArg(self): 342 if not self.args: 343 self.RaiseWrongNumberOfArgumentsException() 344 if (self.args[0].lower() == 'set') or (self.command_alias_used == 'setacl'): 345 return 1 346 else: 347 return 0 348 349 def _SetAcl(self): 350 """Parses options and sets ACLs on the specified buckets/objects.""" 351 self.continue_on_error = False 352 if self.sub_opts: 353 for o, unused_a in self.sub_opts: 354 if o == '-a': 355 self.all_versions = True 356 elif o == '-f': 357 self.continue_on_error = True 358 elif o == '-r' or o == '-R': 359 self.recursion_requested = True 360 else: 361 self.RaiseInvalidArgumentException() 362 try: 363 self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler) 364 except AccessDeniedException as unused_e: 365 self._WarnServiceAccounts() 366 raise 367 if not self.everything_set_okay: 368 raise CommandException('ACLs for some objects could not be set.') 369 370 def _ChAcl(self): 371 """Parses options and changes ACLs on the specified buckets/objects.""" 372 self.parse_versions = True 373 self.changes = [] 374 self.continue_on_error = False 375 376 if self.sub_opts: 377 for o, a in self.sub_opts: 378 if o == '-f': 379 self.continue_on_error = True 380 elif o == '-g': 381 if 'gserviceaccount.com' in a: 382 raise CommandException( 383 'Service accounts are considered users, not groups; please use ' 384 '"gsutil acl ch -u" instead of "gsutil acl ch -g"') 385 self.changes.append( 386 acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.GROUP)) 387 elif o == '-p': 388 self.changes.append( 389 acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.PROJECT)) 390 elif o == '-u': 391 self.changes.append( 392 acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.USER)) 393 elif o == '-d': 394 self.changes.append(acl_helper.AclDel(a)) 395 elif o == '-r' or o == '-R': 396 self.recursion_requested = True 397 else: 398 self.RaiseInvalidArgumentException() 399 400 if not self.changes: 401 raise CommandException('Please specify at least one access change ' 402 'with the -g, -u, or -d flags') 403 404 if (not UrlsAreForSingleProvider(self.args) or 405 StorageUrlFromString(self.args[0]).scheme != 'gs'): 406 raise CommandException( 407 'The "{0}" command can only be used with gs:// URLs'.format( 408 self.command_name)) 409 410 self.everything_set_okay = True 411 self.ApplyAclFunc(_ApplyAclChangesWrapper, 412 _ApplyExceptionHandler, 413 self.args, 414 object_fields=['acl', 'generation', 'metageneration']) 415 if not self.everything_set_okay: 416 raise CommandException('ACLs for some objects could not be set.') 417 418 def _RaiseForAccessDenied(self, url): 419 self._WarnServiceAccounts() 420 raise CommandException('Failed to set acl for %s. Please ensure you have ' 421 'OWNER-role access to this resource.' % url) 422 423 @Retry(ServiceException, tries=3, timeout_secs=1) 424 def ApplyAclChanges(self, name_expansion_result, thread_state=None): 425 """Applies the changes in self.changes to the provided URL. 426 427 Args: 428 name_expansion_result: NameExpansionResult describing the target object. 429 thread_state: If present, gsutil Cloud API instance to apply the changes. 430 """ 431 if thread_state: 432 gsutil_api = thread_state 433 else: 434 gsutil_api = self.gsutil_api 435 436 url = name_expansion_result.expanded_storage_url 437 if url.IsBucket(): 438 bucket = gsutil_api.GetBucket(url.bucket_name, 439 provider=url.scheme, 440 fields=['acl', 'metageneration']) 441 current_acl = bucket.acl 442 elif url.IsObject(): 443 gcs_object = encoding.JsonToMessage(apitools_messages.Object, 444 name_expansion_result.expanded_result) 445 current_acl = gcs_object.acl 446 447 if not current_acl: 448 self._RaiseForAccessDenied(url) 449 if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: 450 self.logger.info('No changes to %s', url) 451 return 452 453 try: 454 if url.IsBucket(): 455 preconditions = Preconditions(meta_gen_match=bucket.metageneration) 456 bucket_metadata = apitools_messages.Bucket(acl=current_acl) 457 gsutil_api.PatchBucket(url.bucket_name, 458 bucket_metadata, 459 preconditions=preconditions, 460 provider=url.scheme, 461 fields=['id']) 462 else: # Object 463 preconditions = Preconditions(gen_match=gcs_object.generation, 464 meta_gen_match=gcs_object.metageneration) 465 object_metadata = apitools_messages.Object(acl=current_acl) 466 try: 467 gsutil_api.PatchObjectMetadata(url.bucket_name, 468 url.object_name, 469 object_metadata, 470 preconditions=preconditions, 471 provider=url.scheme, 472 generation=url.generation, 473 fields=['id']) 474 except PreconditionException as e: 475 # Special retry case where we want to do an additional step, the read 476 # of the read-modify-write cycle, to fetch the correct object 477 # metadata before reattempting ACL changes. 478 self._RefetchObjectMetadataAndApplyAclChanges(url, gsutil_api) 479 480 self.logger.info('Updated ACL on %s', url) 481 except BadRequestException as e: 482 # Don't retry on bad requests, e.g. invalid email address. 483 raise CommandException('Received bad request from server: %s' % str(e)) 484 except AccessDeniedException: 485 self._RaiseForAccessDenied(url) 486 except PreconditionException as e: 487 # For objects, retry attempts should have already been handled. 488 if url.IsObject(): 489 raise CommandException(str(e)) 490 # For buckets, raise PreconditionException and continue to next retry. 491 raise e 492 493 @Retry(PreconditionException, tries=3, timeout_secs=1) 494 def _RefetchObjectMetadataAndApplyAclChanges(self, url, gsutil_api): 495 """Reattempts object ACL changes after a PreconditionException.""" 496 gcs_object = gsutil_api.GetObjectMetadata( 497 url.bucket_name, 498 url.object_name, 499 provider=url.scheme, 500 fields=['acl', 'generation', 'metageneration']) 501 current_acl = gcs_object.acl 502 503 if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: 504 self.logger.info('No changes to %s', url) 505 return 506 507 object_metadata = apitools_messages.Object(acl=current_acl) 508 preconditions = Preconditions(gen_match=gcs_object.generation, 509 meta_gen_match=gcs_object.metageneration) 510 gsutil_api.PatchObjectMetadata(url.bucket_name, 511 url.object_name, 512 object_metadata, 513 preconditions=preconditions, 514 provider=url.scheme, 515 generation=gcs_object.generation, 516 fields=['id']) 517 518 def _ApplyAclChangesAndReturnChangeCount(self, storage_url, acl_message): 519 modification_count = 0 520 for change in self.changes: 521 modification_count += change.Execute(storage_url, acl_message, 'acl', 522 self.logger) 523 return modification_count 524 525 def RunCommand(self): 526 """Command entry point for the acl command.""" 527 action_subcommand = self.args.pop(0) 528 self.ParseSubOpts(check_args=True) 529 530 # Commands with both suboptions and subcommands need to reparse for 531 # suboptions, so we log again. 532 metrics.LogCommandParams(sub_opts=self.sub_opts) 533 self.def_acl = False 534 if action_subcommand == 'get': 535 metrics.LogCommandParams(subcommands=[action_subcommand]) 536 self.GetAndPrintAcl(self.args[0]) 537 elif action_subcommand == 'set': 538 metrics.LogCommandParams(subcommands=[action_subcommand]) 539 self._SetAcl() 540 elif action_subcommand in ('ch', 'change'): 541 metrics.LogCommandParams(subcommands=[action_subcommand]) 542 self._ChAcl() 543 else: 544 raise CommandException( 545 ('Invalid subcommand "%s" for the %s command.\n' 546 'See "gsutil help acl".') % (action_subcommand, self.command_name)) 547 548 return 0 549