1# -*- coding: utf-8 -*- 2# Copyright 2011 Google Inc. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Implementation of logging configuration command for buckets.""" 16 17from __future__ import absolute_import 18from __future__ import print_function 19from __future__ import division 20from __future__ import unicode_literals 21 22import sys 23 24from apitools.base.py import encoding 25 26from gslib import metrics 27from gslib.command import Command 28from gslib.command_argument import CommandArgument 29from gslib.cs_api_map import ApiSelector 30from gslib.exception import CommandException 31from gslib.exception import NO_URLS_MATCHED_TARGET 32from gslib.help_provider import CreateHelpText 33from gslib.storage_url import StorageUrlFromString 34from gslib.storage_url import UrlsAreForSingleProvider 35from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages 36from gslib.utils.constants import NO_MAX 37from gslib.utils import text_util 38 39_SET_SYNOPSIS = """ 40 gsutil logging set on -b <logging_bucket_name> [-o <log_object_prefix>] gs://<bucket_name>... 41 gsutil logging set off gs://<bucket_name>... 42""" 43 44_GET_SYNOPSIS = """ 45 gsutil logging get gs://<bucket_name> 46""" 47 48_SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + '\n' 49 50_SET_DESCRIPTION = """ 51<B>SET</B> 52 The set sub-command has two sub-commands: 53 54<B>ON</B> 55 The "gsutil logging set on" command will enable usage logging of the 56 buckets named by the specified URLs, outputting log files in the specified 57 logging_bucket. logging_bucket must already exist, and all URLs must name 58 buckets (e.g., gs://bucket). The required bucket parameter specifies the 59 bucket to which the logs are written, and the optional log_object_prefix 60 parameter specifies the prefix for log object names. The default prefix 61 is the bucket name. For example, the command: 62 63 gsutil logging set on -b gs://my_logging_bucket -o UsageLog \\ 64 gs://my_bucket1 gs://my_bucket2 65 66 will cause all read and write activity to objects in gs://mybucket1 and 67 gs://mybucket2 to be logged to objects prefixed with the name "UsageLog", 68 with those log objects written to the bucket gs://my_logging_bucket. 69 70 In addition to enabling logging on your bucket(s), you will also need to grant 71 cloud-storage-analytics@google.com write access to the log bucket, using this 72 command: 73 74 gsutil acl ch -g cloud-storage-analytics@google.com:W gs://my_logging_bucket 75 76 Note that log data may contain sensitive information, so you should make 77 sure to set an appropriate default bucket ACL to protect that data. (See 78 "gsutil help defacl".) 79 80<B>OFF</B> 81 This command will disable usage logging of the buckets named by the 82 specified URLs. All URLs must name buckets (e.g., gs://bucket). 83 84 No logging data is removed from the log buckets when you disable logging, 85 but Google Cloud Storage will stop delivering new logs once you have 86 run this command. 87 88""" 89 90_GET_DESCRIPTION = """ 91<B>GET</B> 92 If logging is enabled for the specified bucket url, the server responds 93 with a JSON document that looks something like this: 94 95 { 96 "logBucket": "my_logging_bucket", 97 "logObjectPrefix": "UsageLog" 98 } 99 100 You can download log data from your log bucket using the gsutil cp command. 101 102""" 103 104_DESCRIPTION = """ 105 Google Cloud Storage offers usage logs and storage data in the form of 106 CSV files that you can download and view. Usage logs provide information 107 for all of the requests made on a specified bucket in the last 24 hours, 108 while the storage logs provide information about the storage consumption of 109 that bucket for the last 24 hour period. The logs and storage data files 110 are automatically created as new objects in a bucket that you specify, in 111 24 hour intervals. 112 113 The logging command has two sub-commands: 114""" + _SET_DESCRIPTION + _GET_DESCRIPTION + """ 115 116<B>USAGE LOG AND STORAGE DATA FIELDS</B> 117 For a complete list of usage log fields and storage data fields, see: 118 https://cloud.google.com/storage/docs/access-logs#format 119""" 120 121_DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) 122 123_get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) 124_set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) 125 126 127class LoggingCommand(Command): 128 """Implementation of gsutil logging command.""" 129 130 # Command specification. See base class for documentation. 131 command_spec = Command.CreateCommandSpec( 132 'logging', 133 command_name_aliases=['disablelogging', 'enablelogging', 'getlogging'], 134 usage_synopsis=_SYNOPSIS, 135 min_args=2, 136 max_args=NO_MAX, 137 supported_sub_args='b:o:', 138 file_url_ok=False, 139 provider_url_ok=False, 140 urls_start_arg=0, 141 gs_api_support=[ApiSelector.XML, ApiSelector.JSON], 142 gs_default_api=ApiSelector.JSON, 143 argparse_arguments=[ 144 CommandArgument('mode', choices=['on', 'off']), 145 CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), 146 ], 147 ) 148 # Help specification. See help_provider.py for documentation. 149 help_spec = Command.HelpSpec( 150 help_name='logging', 151 help_name_aliases=[ 152 'loggingconfig', 153 'logs', 154 'log', 155 'getlogging', 156 'enablelogging', 157 'disablelogging', 158 ], 159 help_type='command_help', 160 help_one_line_summary='Configure or retrieve logging on buckets', 161 help_text=_DETAILED_HELP_TEXT, 162 subcommand_help_text={ 163 'get': _get_help_text, 164 'set': _set_help_text, 165 }, 166 ) 167 168 def _Get(self): 169 """Gets logging configuration for a bucket.""" 170 bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( 171 self.args[0], bucket_fields=['logging']) 172 173 if bucket_url.scheme == 's3': 174 text_util.print_to_fd(self.gsutil_api.XmlPassThroughGetLogging( 175 bucket_url, provider=bucket_url.scheme), 176 end='') 177 else: 178 if (bucket_metadata.logging and bucket_metadata.logging.logBucket and 179 bucket_metadata.logging.logObjectPrefix): 180 text_util.print_to_fd( 181 str(encoding.MessageToJson(bucket_metadata.logging))) 182 else: 183 text_util.print_to_fd('%s has no logging configuration.' % bucket_url) 184 return 0 185 186 def _Enable(self): 187 """Enables logging configuration for a bucket.""" 188 # Disallow multi-provider 'logging set on' calls, because the schemas 189 # differ. 190 if not UrlsAreForSingleProvider(self.args): 191 raise CommandException('"logging set on" command spanning providers not ' 192 'allowed.') 193 target_bucket_url = None 194 target_prefix = None 195 for opt, opt_arg in self.sub_opts: 196 if opt == '-b': 197 target_bucket_url = StorageUrlFromString(opt_arg) 198 if opt == '-o': 199 target_prefix = opt_arg 200 201 if not target_bucket_url: 202 raise CommandException('"logging set on" requires \'-b <log_bucket>\' ' 203 'option') 204 if not target_bucket_url.IsBucket(): 205 raise CommandException('-b option must specify a bucket URL.') 206 207 # Iterate over URLs, expanding wildcards and setting logging on each. 208 some_matched = False 209 for url_str in self.args: 210 bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) 211 for blr in bucket_iter: 212 url = blr.storage_url 213 some_matched = True 214 self.logger.info('Enabling logging on %s...', blr) 215 logging = apitools_messages.Bucket.LoggingValue( 216 logBucket=target_bucket_url.bucket_name, 217 logObjectPrefix=target_prefix or url.bucket_name) 218 219 bucket_metadata = apitools_messages.Bucket(logging=logging) 220 self.gsutil_api.PatchBucket(url.bucket_name, 221 bucket_metadata, 222 provider=url.scheme, 223 fields=['id']) 224 if not some_matched: 225 raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args)) 226 return 0 227 228 def _Disable(self): 229 """Disables logging configuration for a bucket.""" 230 # Iterate over URLs, expanding wildcards, and disabling logging on each. 231 some_matched = False 232 for url_str in self.args: 233 bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) 234 for blr in bucket_iter: 235 url = blr.storage_url 236 some_matched = True 237 self.logger.info('Disabling logging on %s...', blr) 238 logging = apitools_messages.Bucket.LoggingValue() 239 240 bucket_metadata = apitools_messages.Bucket(logging=logging) 241 self.gsutil_api.PatchBucket(url.bucket_name, 242 bucket_metadata, 243 provider=url.scheme, 244 fields=['id']) 245 if not some_matched: 246 raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args)) 247 return 0 248 249 def RunCommand(self): 250 """Command entry point for the logging command.""" 251 # Parse the subcommand and alias for the new logging command. 252 action_subcommand = self.args.pop(0) 253 if action_subcommand == 'get': 254 func = self._Get 255 metrics.LogCommandParams(subcommands=[action_subcommand]) 256 elif action_subcommand == 'set': 257 state_subcommand = self.args.pop(0) 258 if not self.args: 259 self.RaiseWrongNumberOfArgumentsException() 260 if state_subcommand == 'on': 261 func = self._Enable 262 metrics.LogCommandParams( 263 subcommands=[action_subcommand, state_subcommand]) 264 elif state_subcommand == 'off': 265 func = self._Disable 266 metrics.LogCommandParams( 267 subcommands=[action_subcommand, state_subcommand]) 268 else: 269 raise CommandException( 270 ('Invalid subcommand "%s" for the "%s %s" command.\n' 271 'See "gsutil help logging".') % 272 (state_subcommand, self.command_name, action_subcommand)) 273 else: 274 raise CommandException(('Invalid subcommand "%s" for the %s command.\n' 275 'See "gsutil help logging".') % 276 (action_subcommand, self.command_name)) 277 self.ParseSubOpts(check_args=True) 278 # Commands with both suboptions and subcommands need to reparse for 279 # suboptions, so we log again. 280 metrics.LogCommandParams(sub_opts=self.sub_opts) 281 func() 282 return 0 283