1# -------------------------------------------------------------------------------------------- 2# Copyright (c) Microsoft Corporation. All rights reserved. 3# Licensed under the MIT License. See License.txt in the project root for license information. 4# -------------------------------------------------------------------------------------------- 5 6import copy 7import json 8import os 9import time 10import zipfile 11import requests 12import urllib3 13 14try: 15 # Try importing Python 3 urllib.parse 16 from urllib.parse import quote 17except ImportError: 18 # If urllib.parse was not imported, use Python 2 module urlparse 19 from urllib import quote # pylint: disable=import-error 20 21from knack.util import CLIError 22from azure.cli.command_modules.botservice.http_response_validator import HttpResponseValidator 23from azure.cli.command_modules.botservice.web_app_operations import WebAppOperations 24from azure.cli.core.commands.client_factory import get_subscription_id 25 26 27class KuduClient: # pylint:disable=too-many-instance-attributes 28 def __init__(self, cmd, resource_group_name, name, bot, logger): 29 self.__cmd = cmd 30 self.__resource_group_name = resource_group_name 31 self.__name = name 32 self.__bot = bot 33 self.__logger = logger 34 35 # Properties set after self.__initialize() is called. 36 self.__initialized = False 37 self.__password = None 38 self.__scm_url = None 39 self.__auth_headers = None 40 self.bot_site_name = WebAppOperations.get_bot_site_name(self.__bot.properties.endpoint) 41 42 def download_bot_zip(self, file_save_path, folder_path): 43 """Download bot's source code from Kudu. 44 45 This method looks for the zipped source code in the site/clirepo/ folder on Kudu. If the code is not there, the 46 contents of site/wwwroot are zipped and then downloaded. 47 48 :param file_save_path: string 49 :param folder_path: string 50 :return: None 51 """ 52 if not self.__initialized: 53 self.__initialize() 54 55 headers = self.__get_application_octet_stream_headers() 56 # Download source code in zip format from Kudu 57 response = requests.get(self.__scm_url + '/api/zip/site/clirepo/', headers=headers) 58 # If the status_code is not 200, the source code was not successfully retrieved. 59 # Run the prepareSrc.cmd to zip up the code and prepare it for download. 60 if response.status_code != 200: 61 # try getting the bot from wwwroot instead 62 payload = { 63 'command': 'PostDeployScripts\\prepareSrc.cmd {0}'.format(self.__password), 64 'dir': r'site\wwwroot' 65 } 66 prepareSrc_response = requests.post(self.__scm_url + '/api/command', 67 data=json.dumps(payload), 68 headers=self.__get_application_json_headers()) 69 HttpResponseValidator.check_response_status(prepareSrc_response) 70 71 # Overwrite previous "response" with bot-src.zip. 72 response = requests.get(self.__scm_url + '/api/vfs/site/bot-src.zip', 73 headers=headers) 74 HttpResponseValidator.check_response_status(response) 75 76 download_path = os.path.join(file_save_path, 'download.zip') 77 with open(os.path.join(file_save_path, 'download.zip'), 'wb') as f: 78 f.write(response.content) 79 zip_ref = zipfile.ZipFile(download_path) 80 zip_ref.extractall(folder_path) 81 zip_ref.close() 82 os.remove(download_path) 83 84 def get_bot_file(self, bot_file): 85 """Retrieve the .bot file from Kudu. 86 87 :param bot_file: 88 :return: 89 """ 90 if not self.__initialized: 91 self.__initialize() 92 93 if bot_file.startswith('./') or bot_file.startswith('.\\'): 94 bot_file = bot_file[2:] 95 # Format backslashes to forward slashes and URL escape 96 bot_file = quote(bot_file.replace('\\', '/')) 97 request_url = self.__scm_url + '/api/vfs/site/wwwroot/' + bot_file 98 self.__logger.info('Attempting to retrieve .bot file content from %s' % request_url) 99 response = requests.get(request_url, headers=self.__get_application_octet_stream_headers()) 100 HttpResponseValidator.check_response_status(response) 101 self.__logger.info('Bot file successfully retrieved from Kudu.') 102 return json.loads(response.text) 103 104 def install_node_dependencies(self): 105 """Installs Node.js dependencies at `site/wwwroot/` for Node.js bots. 106 107 This method is only called when the detected bot is a Node.js bot. 108 109 :return: Dictionary with results of the HTTP Kudu request 110 """ 111 if not self.__initialized: 112 self.__initialize() 113 114 payload = { 115 'command': 'npm install', 116 'dir': r'site\wwwroot' 117 } 118 # npm install can take a very long time to complete. By default, Azure's load balancer will terminate 119 # connections after 230 seconds of no inbound or outbound packets. This timeout is not configurable. 120 try: 121 response = requests.post(self.__scm_url + '/api/command', data=json.dumps(payload), 122 headers=self.__get_application_json_headers()) 123 HttpResponseValidator.check_response_status(response) 124 except CLIError as e: 125 if response.status_code == 500 and 'The request timed out.' in response.text: 126 self.__logger.warning('npm install is taking longer than expected and did not finish within the ' 127 'Azure-specified timeout of 230 seconds.') 128 self.__logger.warning('The installation is likely still in progress. This is a known issue, please wait' 129 ' a short while before messaging your bot. You can also visit Kudu to manually ' 130 'install the npm dependencies. (https://github.com/projectkudu/kudu/wiki)') 131 self.__logger.warning('Your Kudu website for this bot is: %s' % self.__scm_url) 132 self.__logger.warning('\nYou can also use `--keep-node-modules` in your `az bot publish` command to ' 133 'not `npm install` the dependencies for the bot on Kudu.') 134 subscription_id = get_subscription_id(self.__cmd.cli_ctx) 135 self.__logger.warning('Alternatively, you can configure your Application Settings for the App Service ' 136 'to build during zipdeploy by using the following command:\n az webapp config ' 137 'appsettings set -n %s -g %s --subscription %s --settings ' 138 'SCM_DO_BUILD_DURING_DEPLOYMENT=true' % 139 (self.bot_site_name, self.__resource_group_name, subscription_id)) 140 141 else: 142 raise e 143 144 return response.json() 145 146 def publish(self, zip_file_path, timeout, keep_node_modules, detected_language): 147 """Publishes zipped bot source code to Kudu. 148 149 Performs the following steps: 150 1. Empties the `site/clirepo/` folder on Kudu 151 2. Pushes the code to `site/clirepo/` 152 3. Deploys the code via the zipdeploy API. (https://github.com/projectkudu/kudu/wiki/REST-API#zip-deployment) 153 4. Gets the results of the latest Kudu deployment 154 155 :param zip_file_path: 156 :param timeout: 157 :param keep_node_modules: 158 :param detected_language: 159 :return: Dictionary with results of the latest deployment 160 """ 161 if not self.__initialized: 162 self.__initialize() 163 self.__empty_source_folder() 164 165 headers = self.__get_application_octet_stream_headers() 166 with open(zip_file_path, 'rb') as fs: 167 zip_content = fs.read() 168 response = requests.put(self.__scm_url + '/api/zip/site/clirepo', 169 headers=headers, 170 data=zip_content) 171 HttpResponseValidator.check_response_status(response) 172 173 return self.__enable_zip_deploy(zip_file_path, timeout, keep_node_modules, detected_language) 174 175 def __check_zip_deployment_status(self, timeout=None): 176 deployment_status_url = self.__scm_url + '/api/deployments/latest' 177 total_trials = (int(timeout) // 2) if timeout else 450 178 num_trials = 0 179 while num_trials < total_trials: 180 time.sleep(2) 181 response = requests.get(deployment_status_url, headers=self.__auth_headers) 182 res_dict = response.json() 183 num_trials = num_trials + 1 184 if res_dict.get('status', 0) == 3: 185 raise CLIError('Zip deployment failed.') 186 if res_dict.get('status', 0) == 4: 187 break 188 if 'progress' in res_dict: 189 self.__logger.debug(res_dict['progress']) 190 # if the deployment is taking longer than expected 191 if res_dict.get('status', 0) != 4: 192 raise CLIError("""Deployment is taking longer than expected. Please verify 193 status at '{}' beforing launching the app""".format(deployment_status_url)) 194 return res_dict 195 196 def __empty_source_folder(self): 197 """Remove the `clirepo/` folder from Kudu. 198 199 This method is called from KuduClient.publish() in preparation for uploading the user's local source code. 200 After removing the folder from Kudu, the method performs another request to recreate the `clirepo/` folder. 201 :return: 202 """ 203 # The `clirepo/` folder contains the zipped up source code 204 payload = { 205 'command': 'rm -rf clirepo && mkdir clirepo', 206 'dir': r'site' 207 } 208 headers = self.__get_application_json_headers() 209 response = requests.post(self.__scm_url + '/api/command', data=json.dumps(payload), headers=headers) 210 HttpResponseValidator.check_response_status(response) 211 212 def __empty_wwwroot_folder_except_for_node_modules(self): 213 """Empty site/wwwroot/ folder but retain node_modules folder. 214 :return: 215 """ 216 self.__logger.info('Removing all files and folders from "site/wwwroot/" except for node_modules.') 217 payload = { 218 'command': '(for /D %i in (.\\*) do if not %~nxi == node_modules rmdir /s/q %i) && (for %i in (.\\*) ' 219 'del %i)', 220 'dir': r'site\wwwroot' 221 } 222 headers = self.__get_application_json_headers() 223 response = requests.post(self.__scm_url + '/api/command', data=json.dumps(payload), headers=headers) 224 HttpResponseValidator.check_response_status(response) 225 self.__logger.info('All files and folders successfully removed from "site/wwwroot/" except for node_modules.') 226 227 def __empty_wwwroot_folder(self): 228 """Empty the site/wwwroot/ folder from Kudu. 229 230 Empties the site/wwwroot/ folder by removing the entire directory, and then recreating it. Called when 231 publishing a bot to Kudu. 232 """ 233 self.__logger.info('Emptying the "site/wwwroot/" folder on Kudu in preparation for publishing.') 234 payload = { 235 'command': 'rm -rf wwwroot && mkdir wwwroot', 236 'dir': r'site' 237 } 238 headers = self.__get_application_json_headers() 239 response = requests.post(self.__scm_url + '/api/command', data=json.dumps(payload), headers=headers) 240 HttpResponseValidator.check_response_status(response) 241 self.__logger.info('"site/wwwroot/" successfully emptied.') 242 243 def __enable_zip_deploy(self, zip_file_path, timeout, keep_node_modules, detected_language): 244 """Pushes local bot's source code in zip format to Kudu for deployment. 245 246 This method deploys the zipped bot source code via Kudu's zipdeploy API. This API does not run any build 247 processes such as `npm install`, `dotnet restore`, `dotnet publish`, etc. 248 249 :param zip_file_path: string 250 :return: Dictionary with results of the latest deployment 251 """ 252 253 zip_url = self.__scm_url + '/api/zipdeploy?isAsync=true' 254 headers = self.__get_application_octet_stream_headers() 255 if not keep_node_modules or detected_language == 'Csharp': 256 self.__empty_source_folder() 257 else: 258 self.__empty_wwwroot_folder_except_for_node_modules() 259 260 with open(os.path.realpath(os.path.expanduser(zip_file_path)), 'rb') as fs: 261 print(zip_file_path) 262 zip_content = fs.read() 263 self.__logger.info('Source code read, uploading to Kudu.') 264 r = requests.post(zip_url, data=zip_content, headers=headers) 265 if r.status_code != 202: 266 raise CLIError("Zip deployment {} failed with status code '{}' and reason '{}'".format( 267 zip_url, r.status_code, r.text)) 268 269 self.__logger.info('Retrieving current deployment info.') 270 # On successful deployment navigate to the app, display the latest deployment JSON response. 271 return self.__check_zip_deployment_status(timeout) 272 273 def __get_application_json_headers(self): 274 headers = copy.deepcopy(self.__auth_headers) 275 headers['content-type'] = 'application/json' 276 return headers 277 278 def __get_application_octet_stream_headers(self): 279 headers = copy.deepcopy(self.__auth_headers) 280 headers['content-type'] = 'application/octet-stream' 281 return headers 282 283 def __initialize(self): 284 """Generates necessary data for performing calls to Kudu based off of data passed in on initialization. 285 286 :return: None 287 """ 288 user_name, password = WebAppOperations.get_site_credential(self.__cmd.cli_ctx, 289 self.__resource_group_name, 290 self.bot_site_name, 291 None) 292 293 # Store the password for download_bot_zip: 294 self.__password = password 295 self.__scm_url = WebAppOperations.get_scm_url(self.__cmd, 296 self.__resource_group_name, 297 self.bot_site_name, 298 None) 299 300 self.__auth_headers = urllib3.util.make_headers(basic_auth='{0}:{1}'.format(user_name, password)) 301 self.__initialized = True 302