clearlinux/WALinuxAgent
2
0
Fork 0
mirror of https://github.com/clearlinux/WALinuxAgent.git synced 2026-09-05 21:31:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

[#823] -- Show config options in use

Browse Source 
[#784] -- Adjust http retry and logging

Signed-off-by: Brendan Dixon <brendandixon@me.com>
This commit is contained in:
Brendan Dixon
2017-08-11 09:37:33 -07:00
parent 53ca96622b
commit d0b583cc6f
23 changed files with 1124 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
azurelinuxagent/agent.py
+9
View File
@@ -128,6 +128,11 @@ class Agent(object):
update_handler = get_update_handler()
update_handler.run()
def show_configuration(self):
configuration = conf.get_configuration()
for k in sorted(configuration.keys()):
print("{0} = {1}".format(k, configuration[k]))
def main(args=[]):
"""
Parse command line arguments, exit with usage() on error.
@@ -157,6 +162,8 @@ def main(args=[]):
agent.daemon()
elif command == "run-exthandlers":
agent.run_exthandlers()
elif command == "show-configuration":
agent.show_configuration()
except Exception:
logger.error(u"Failed to run '{0}': {1}",
command,
@@ -198,6 +205,8 @@ def parse_args(sys_args):
verbose = True
elif re.match("^([-/]*)force", a):
force = True
elif re.match("^([-/]*show-configuration", a):
cmd = "show-configuration"
elif re.match("^([-/]*)(help|usage|\\?)", a):
cmd = "help"
else:
azurelinuxagent/common/conf.py
+67 -1
View File
@@ -85,6 +85,69 @@ def load_conf_from_file(conf_file_path, conf=__conf__):
raise AgentConfigError(("Failed to load conf file:{0}, {1}"
"").format(conf_file_path, err))
__SWITCH_OPTIONS__ = {
"OS.EnableRDMA" : False,
"OS.UpdateRdmaDriver" : False,
"OS.CheckRdmaDriver" : False,
"Logs.Verbose" : False,
"OS.EnableFIPS" : False,
"Provisioning.Enabled" : True,
"Provisioning.UseCloudInit" : False,
"Provisioning.AllowResetSysUser" : False,
"Provisioning.RegenerateSshHostKeyPair" : False,
"Provisioning.DeleteRootPassword" : False,
"Provisioning.DecodeCustomData" : False,
"Provisioning.ExecuteCustomData" : False,
"Provisioning.MonitorHostName" : False,
"DetectScvmmEnv" : False,
"ResourceDisk.Format" : False,
"DetectScvmmEnv" : False,
"ResourceDisk.Format" : False,
"ResourceDisk.EnableSwap" : False,
"AutoUpdate.Enabled" : True,
"EnableOverProvisioning" : False,
"OS.AllowHTTP" : False
}
__STRING_OPTIONS__ = {
"Lib.Dir" : "/var/lib/waagent",
"DVD.MountPoint" : "/mnt/cdrom/secure",
"Pid.File" : "/var/run/waagent.pid",
"Extension.LogDir" : "/var/log/azure",
"OS.OpensslPath" : "/usr/bin/openssl",
"OS.SshDir" : "/etc/ssh",
"OS.HomeDir" : "/home",
"OS.PasswordPath" : "/etc/shadow",
"OS.SudoersDir" : "/etc/sudoers.d",
"OS.RootDeviceScsiTimeout" : None,
"Provisioning.SshHostKeyPairType" : "rsa",
"Provisioning.PasswordCryptId" : "6",
"HttpProxy.Host" : None,
"ResourceDisk.MountPoint" : "/mnt/resource",
"ResourceDisk.MountOptions" : None,
"ResourceDisk.Filesystem" : "ext3",
"AutoUpdate.GAFamily" : "Prod"
}
__INTEGER_OPTIONS__ = {
"Provisioning.PasswordCryptSaltLength" : 10,
"HttpProxy.Port" : None,
"ResourceDisk.SwapSizeMB" : 0,
"Autoupdate.Frequency" : 3600
}
def get_configuration(conf=__conf__):
options = {}
for option in __SWITCH_OPTIONS__:
options[option] = conf.get_switch(option, __SWITCH_OPTIONS__[option])
for option in __STRING_OPTIONS__:
options[option] = conf.get(option, __STRING_OPTIONS__[option])
for option in __INTEGER_OPTIONS__:
options[option] = conf.get_int(option, __INTEGER_OPTIONS__[option])
return options
def enable_rdma(conf=__conf__):
return conf.get_switch("OS.EnableRDMA", False) or \
@@ -251,4 +314,7 @@ def get_autoupdate_frequency(conf=__conf__):
return conf.get_int("Autoupdate.Frequency", 3600)
def get_enable_overprovisioning(conf=__conf__):
return conf.get_switch("EnableOverProvisioning", False)
return conf.get_switch("EnableOverProvisioning", False)
def get_allow_http(conf=__conf__):
return conf.get_switch("OS.AllowHTTP", False)
azurelinuxagent/common/exception.py
+9 -1
View File
@@ -86,7 +86,6 @@ class DhcpError(AgentError):
def __init__(self, msg=None, inner=None):
super(DhcpError, self).__init__('000006', msg, inner)
class OSUtilError(AgentError):
"""
Failed to perform operation to OS configuration
@@ -148,3 +147,12 @@ class UpdateError(AgentError):
def __init__(self, msg=None, inner=None):
super(UpdateError, self).__init__('000012', msg, inner)
class BadRequestError(HttpError):
"""
The server rejected the request (i.e., status code 400)
"""
def __init__(self, msg=None, inner=None):
super(BadRequestError, self).__init__(msg, inner)
azurelinuxagent/common/protocol/hostplugin.py
+18 -77
View File
@@ -22,7 +22,8 @@ import json
import traceback
from azurelinuxagent.common import logger
from azurelinuxagent.common.exception import ProtocolError, HttpError
from azurelinuxagent.common.exception import BadRequestError, \
HttpError, ProtocolError
from azurelinuxagent.common.future import ustr, httpclient
from azurelinuxagent.common.utils import restutil
from azurelinuxagent.common.utils import textutil
@@ -85,10 +86,10 @@ class HostPluginProtocol(object):
try:
headers = {HEADER_CONTAINER_ID: self.container_id}
response = restutil.http_get(url, headers)
if response.status != httpclient.OK:
if restutil.request_failed(response):
logger.error(
"HostGAPlugin: Failed Get API versions: {0}".format(
self.read_response_error(response)))
restutil.read_response_error(response)))
else:
return_val = ustr(remove_bom(response.read()), encoding='utf-8')
@@ -117,42 +118,7 @@ class HostPluginProtocol(object):
return url, headers
def put_vm_log(self, content):
"""
Try to upload the given content to the host plugin
:param deployment_id: the deployment id, which is obtained from the
goal state (tenant name)
:param container_id: the container id, which is obtained from the
goal state
:param content: the binary content of the zip file to upload
:return:
"""
if not self.ensure_initialized():
raise ProtocolError("HostGAPlugin: Host plugin channel is not available")
if content is None \
or self.container_id is None \
or self.deployment_id is None:
logger.error(
"HostGAPlugin: Invalid arguments passed: "
"[{0}], [{1}], [{2}]".format(
content,
self.container_id,
self.deployment_id))
return
url = URI_FORMAT_PUT_LOG.format(self.endpoint, HOST_PLUGIN_PORT)
headers = {"x-ms-vmagentlog-deploymentid": self.deployment_id,
"x-ms-vmagentlog-containerid": self.container_id}
logger.periodic(
logger.EVERY_FIFTEEN_MINUTES,
"HostGAPlugin: Put VM log to [{0}]".format(url))
try:
response = restutil.http_put(url, content, headers)
if response.status != httpclient.OK:
logger.error("HostGAPlugin: Put log failed: Code {0}".format(
response.status))
except HttpError as e:
logger.error("HostGAPlugin: Put log exception: {0}".format(e))
raise NotImplementedError("Unimplemented")
def put_vm_status(self, status_blob, sas_url, config_blob_type=None):
"""
@@ -169,6 +135,7 @@ class HostPluginProtocol(object):
logger.verbose("HostGAPlugin: Posting VM status")
try:
blob_type = status_blob.type if status_blob.type else config_blob_type
if blob_type == "BlockBlob":
@@ -176,17 +143,14 @@ class HostPluginProtocol(object):
else:
self._put_page_blob_status(sas_url, status_blob)
if not HostPluginProtocol.is_default_channel():
except Exception as e:
# If the HostPlugin rejects the request,
# let the error continue, but set to use the HostPlugin
if isinstance(e, BadRequestError):
logger.verbose("HostGAPlugin: Setting host plugin as default channel")
HostPluginProtocol.set_default_channel(True)
except Exception as e:
message = "HostGAPlugin: Exception Put VM status: {0}, {1}".format(e, traceback.format_exc())
from azurelinuxagent.common.event import WALAEventOperation, report_event
report_event(op=WALAEventOperation.ReportStatus,
is_success=False,
message=message)
logger.warn("HostGAPlugin: resetting default channel")
HostPluginProtocol.set_default_channel(False)
raise
def _put_block_blob_status(self, sas_url, status_blob):
url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
@@ -198,9 +162,9 @@ class HostPluginProtocol(object):
bytearray(status_blob.data, encoding='utf-8')),
headers=self._build_status_headers())
if response.status != httpclient.OK:
if restutil.request_failed(response):
raise HttpError("HostGAPlugin: Put BlockBlob failed: {0}".format(
self.read_response_error(response)))
restutil.read_response_error(response)))
else:
logger.verbose("HostGAPlugin: Put BlockBlob status succeeded")
@@ -219,10 +183,10 @@ class HostPluginProtocol(object):
status_blob.get_page_blob_create_headers(status_size)),
headers=self._build_status_headers())
if response.status != httpclient.OK:
if restutil.request_failed(response):
raise HttpError(
"HostGAPlugin: Failed PageBlob clean-up: {0}".format(
self.read_response_error(response)))
restutil.read_response_error(response)))
else:
logger.verbose("HostGAPlugin: PageBlob clean-up succeeded")
@@ -249,11 +213,11 @@ class HostPluginProtocol(object):
buf),
headers=self._build_status_headers())
if response.status != httpclient.OK:
if restutil.request_failed(response):
raise HttpError(
"HostGAPlugin Error: Put PageBlob bytes [{0},{1}]: " \
"{2}".format(
start, end, self.read_response_error(response)))
start, end, restutil.read_response_error(response)))
# Advance to the next page (if any)
start = end
@@ -287,26 +251,3 @@ class HostPluginProtocol(object):
if PY_VERSION_MAJOR > 2:
return s.decode('utf-8')
return s
@staticmethod
def read_response_error(response):
result = ''
if response is not None:
try:
body = remove_bom(response.read())
result = "[{0}: {1}] {2}".format(response.status,
response.reason,
body)
# this result string is passed upstream to several methods
# which do a raise HttpError() or a format() of some kind;
# as a result it cannot have any unicode characters
if PY_VERSION_MAJOR < 3:
result = ustr(result, encoding='ascii', errors='ignore')
else:
result = result\
.encode(encoding='ascii', errors='ignore')\
.decode(encoding='ascii', errors='ignore')
except Exception:
logger.warn(traceback.format_exc())
return result
azurelinuxagent/common/protocol/metadata.py
+2 -2
View File
@@ -88,7 +88,7 @@ class MetadataProtocol(Protocol):
except HttpError as e:
raise ProtocolError(ustr(e))
if resp.status != httpclient.OK:
if restutil.request_failed(resp):
raise ProtocolError("{0} - GET: {1}".format(resp.status, url))
data = resp.read()
@@ -103,7 +103,7 @@ class MetadataProtocol(Protocol):
resp = restutil.http_put(url, json.dumps(data), headers=headers)
except HttpError as e:
raise ProtocolError(ustr(e))
if resp.status != httpclient.OK:
if restutil.request_failed(resp):
raise ProtocolError("{0} - PUT: {1}".format(resp.status, url))
def _post_data(self, url, data, headers=None):
azurelinuxagent/common/protocol/restapi.py
+1 -1
View File
@@ -318,7 +318,7 @@ class Protocol(DataContract):
def download_ext_handler_pkg(self, uri, headers=None):
try:
resp = restutil.http_get(uri, chk_proxy=True, headers=headers)
if resp.status == restutil.httpclient.OK:
if restutil.request_succeeded(resp):
return resp.read()
except Exception as e:
logger.warn("Failed to download from: {0}".format(uri), e)
azurelinuxagent/common/protocol/wire.py
+196 -139
View File
@@ -26,7 +26,8 @@ import azurelinuxagent.common.conf as conf
import azurelinuxagent.common.utils.fileutil as fileutil
import azurelinuxagent.common.utils.textutil as textutil
from azurelinuxagent.common.exception import ProtocolNotFoundError
from azurelinuxagent.common.exception import BadRequestError, \
ProtocolNotFoundError
from azurelinuxagent.common.future import httpclient, bytebuffer
from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol
from azurelinuxagent.common.protocol.restapi import *
@@ -96,7 +97,10 @@ class WireProtocol(Protocol):
cryptutil = CryptUtil(conf.get_openssl_cmd())
cryptutil.gen_transport_cert(trans_prv_file, trans_cert_file)
self.client.update_goal_state(forced=True)
self.update_goal_state(forced=True)
def update_goal_state(self, forced=False, max_retry=3):
self.client.update_goal_state(forced=forced, max_retry=max_retry)
def get_vminfo(self):
goal_state = self.client.get_goal_state()
@@ -117,7 +121,7 @@ class WireProtocol(Protocol):
def get_vmagent_manifests(self):
# Update goal state to get latest extensions config
self.client.update_goal_state()
self.update_goal_state()
goal_state = self.client.get_goal_state()
ext_conf = self.client.get_ext_conf()
return ext_conf.vmagent_manifests, goal_state.incarnation
@@ -130,7 +134,7 @@ class WireProtocol(Protocol):
def get_ext_handlers(self):
logger.verbose("Get extension handler config")
# Update goal state to get latest extensions config
self.client.update_goal_state()
self.update_goal_state()
goal_state = self.client.get_goal_state()
ext_conf = self.client.get_ext_conf()
# In wire protocol, incarnation is equivalent to ETag
@@ -533,29 +537,27 @@ class WireClient(object):
self.req_count = 0
def call_wireserver(self, http_req, *args, **kwargs):
"""
Call wire server; handle throttling (403), resource gone (410) and
service unavailable (503).
"""
self.prevent_throttling()
for retry in range(0, 3):
try:
# Never use the HTTP proxy for wireserver
kwargs['chk_proxy'] = False
resp = http_req(*args, **kwargs)
if resp.status == httpclient.FORBIDDEN:
logger.warn("Sending too many requests to wire server. ")
logger.info("Sleeping {0}s to avoid throttling.",
LONG_WAITING_INTERVAL)
time.sleep(LONG_WAITING_INTERVAL)
elif resp.status == httpclient.SERVICE_UNAVAILABLE:
logger.warn("Service temporarily unavailable, sleeping {0}s "
"before retrying.", LONG_WAITING_INTERVAL)
time.sleep(LONG_WAITING_INTERVAL)
elif resp.status == httpclient.GONE:
msg = args[0] if len(args) > 0 else ""
raise WireProtocolResourceGone(msg)
else:
return resp
raise ProtocolError(("Calling wire server failed: "
"{0}").format(resp.status))
except Exception as e:
raise ProtocolError("[Wireserver Exception] {0}".format(
ustr(e)))
if resp is not None and resp.status == httpclient.GONE:
msg = args[0] if len(args) > 0 else ""
raise WireProtocolResourceGone(msg)
elif restutil.request_failed(resp):
msg = "[Wireserver Failed] URI {0} ".format(args[0])
if resp is not None:
msg += " [HTTP Failed] Status Code {0}".format(resp.status)
raise ProtocolError(msg)
return resp
def decode_config(self, data):
if data is None:
@@ -565,16 +567,9 @@ class WireClient(object):
return xml_text
def fetch_config(self, uri, headers):
try:
resp = self.call_wireserver(restutil.http_get,
uri,
headers=headers)
except HttpError as e:
raise ProtocolError(ustr(e))
if resp.status != httpclient.OK:
raise ProtocolError("{0} - {1}".format(resp.status, uri))
resp = self.call_wireserver(restutil.http_get,
uri,
headers=headers)
return self.decode_config(resp.read())
def fetch_cache(self, local_file):
@@ -595,25 +590,11 @@ class WireClient(object):
@staticmethod
def call_storage_service(http_req, *args, **kwargs):
"""
Call storage service, handle SERVICE_UNAVAILABLE(503)
"""
# Default to use the configured HTTP proxy
if not 'chk_proxy' in kwargs or kwargs['chk_proxy'] is None:
kwargs['chk_proxy'] = True
for retry in range(0, 3):
resp = http_req(*args, **kwargs)
if resp.status == httpclient.SERVICE_UNAVAILABLE:
logger.warn("Storage service is temporarily unavailable. ")
logger.info("Will retry in {0} seconds. ",
LONG_WAITING_INTERVAL)
time.sleep(LONG_WAITING_INTERVAL)
else:
return resp
raise ProtocolError(("Calling storage endpoint failed: "
"{0}").format(resp.status))
return http_req(*args, **kwargs)
def fetch_manifest(self, version_uris):
logger.verbose("Fetch manifest")
@@ -621,47 +602,61 @@ class WireClient(object):
response = None
if not HostPluginProtocol.is_default_channel():
response = self.fetch(version.uri)
if not response:
if HostPluginProtocol.is_default_channel():
logger.verbose("Using host plugin as default channel")
else:
logger.verbose("Manifest could not be downloaded, falling back to host plugin")
host = self.get_host_plugin()
uri, headers = host.get_artifact_request(version.uri)
response = self.fetch(uri, headers, chk_proxy=False)
if not response:
host = self.get_host_plugin(force_update=True)
logger.info("Retry fetch in {0} seconds",
SHORT_WAITING_INTERVAL)
time.sleep(SHORT_WAITING_INTERVAL)
else:
host.manifest_uri = version.uri
logger.verbose("Manifest downloaded successfully from host plugin")
if not HostPluginProtocol.is_default_channel():
logger.info("Setting host plugin as default channel")
HostPluginProtocol.set_default_channel(True)
logger.verbose("Failed to download manifest, "
"switching to host plugin")
try:
host = self.get_host_plugin()
uri, headers = host.get_artifact_request(version.uri)
response = self.fetch(uri, headers, chk_proxy=False)
# If the HostPlugin rejects the request,
# let the error continue, but set to use the HostPlugin
except BadRequestError:
HostPluginProtocol.set_default_channel(True)
raise
host.manifest_uri = version.uri
logger.verbose("Manifest downloaded successfully from host plugin")
if not HostPluginProtocol.is_default_channel():
logger.info("Setting host plugin as default channel")
HostPluginProtocol.set_default_channel(True)
if response:
return response
raise ProtocolError("Failed to fetch manifest from all sources")
def fetch(self, uri, headers=None, chk_proxy=None):
logger.verbose("Fetch [{0}] with headers [{1}]", uri, headers)
return_value = None
try:
resp = self.call_storage_service(
restutil.http_get,
uri,
headers,
chk_proxy=chk_proxy)
if resp.status == httpclient.OK:
return_value = self.decode_config(resp.read())
else:
logger.warn("Could not fetch {0} [{1}]",
uri,
HostPluginProtocol.read_response_error(resp))
restutil.http_get,
uri,
headers=headers,
chk_proxy=chk_proxy)
if restutil.request_failed(resp):
msg = "[Storage Failed] URI {0} ".format(uri)
if resp is not None:
msg += restutil.read_response_error(resp)
logger.warn(msg)
raise ProtocolError(msg)
return self.decode_config(resp.read())
except (HttpError, ProtocolError) as e:
logger.verbose("Fetch failed from [{0}]: {1}", uri, e)
return return_value
if isinstance(e, BadRequestError):
raise
return None
def update_hosting_env(self, goal_state):
if goal_state.hosting_env_uri is None:
@@ -793,20 +788,45 @@ class WireClient(object):
return self.ext_conf
def get_ext_manifest(self, ext_handler, goal_state):
local_file = MANIFEST_FILE_NAME.format(ext_handler.name,
goal_state.incarnation)
local_file = os.path.join(conf.get_lib_dir(), local_file)
xml_text = self.fetch_manifest(ext_handler.versionUris)
self.save_cache(local_file, xml_text)
return ExtensionManifest(xml_text)
for update_goal_state in [False, True]:
try:
if update_goal_state:
self.update_goal_state(forced=True)
goal_state = self.get_goal_state()
local_file = MANIFEST_FILE_NAME.format(
ext_handler.name,
goal_state.incarnation)
local_file = os.path.join(conf.get_lib_dir(), local_file)
xml_text = self.fetch_manifest(ext_handler.versionUris)
self.save_cache(local_file, xml_text)
return ExtensionManifest(xml_text)
except BadRequestError:
continue
raise ProtocolError("Failed to retrieve extension manifest")
def get_gafamily_manifest(self, vmagent_manifest, goal_state):
local_file = MANIFEST_FILE_NAME.format(vmagent_manifest.family,
goal_state.incarnation)
local_file = os.path.join(conf.get_lib_dir(), local_file)
xml_text = self.fetch_manifest(vmagent_manifest.versionsManifestUris)
fileutil.write_file(local_file, xml_text)
return ExtensionManifest(xml_text)
for update_goal_state in [False, True]:
try:
if update_goal_state:
self.update_goal_state(forced=True)
goal_state = self.get_goal_state()
local_file = MANIFEST_FILE_NAME.format(
vmagent_manifest.family,
goal_state.incarnation)
local_file = os.path.join(conf.get_lib_dir(), local_file)
xml_text = self.fetch_manifest(
vmagent_manifest.versionsManifestUris)
fileutil.write_file(local_file, xml_text)
return ExtensionManifest(xml_text)
except BadRequestError:
continue
raise ProtocolError("Failed to retrieve GAFamily manifest")
def check_wire_protocol_version(self):
uri = VERSION_INFO_URI.format(self.endpoint)
@@ -825,39 +845,55 @@ class WireClient(object):
raise ProtocolNotFoundError(error)
def upload_status_blob(self):
ext_conf = self.get_ext_conf()
blob_uri = ext_conf.status_upload_blob
blob_type = ext_conf.status_upload_blob_type
if blob_uri is not None:
if not blob_type in ["BlockBlob", "PageBlob"]:
blob_type = "BlockBlob"
logger.verbose("Status Blob type is unspecified "
"-- assuming it is a BlockBlob")
for update_goal_state in [False, True]:
try:
self.status_blob.prepare(blob_type)
if update_goal_state:
self.update_goal_state(forced=True)
ext_conf = self.get_ext_conf()
blob_uri = ext_conf.status_upload_blob
blob_type = ext_conf.status_upload_blob_type
if blob_uri is not None:
if not blob_type in ["BlockBlob", "PageBlob"]:
blob_type = "BlockBlob"
logger.verbose("Status Blob type is unspecified "
"-- assuming it is a BlockBlob")
try:
self.status_blob.prepare(blob_type)
except Exception as e:
self.report_status_event(
"Exception creating status blob: {0}", ustr(e))
return
if not HostPluginProtocol.is_default_channel():
try:
if self.status_blob.upload(blob_uri):
return
except HttpError as e:
pass
host = self.get_host_plugin()
host.put_vm_status(self.status_blob,
ext_conf.status_upload_blob,
ext_conf.status_upload_blob_type)
HostPluginProtocol.set_default_channel(True)
return
except Exception as e:
# If the HostPlugin rejects the request,
# let the error continue, but set to use the HostPlugin
if isinstance(e, BadRequestError):
HostPluginProtocol.set_default_channel(True)
continue
self.report_status_event(
"Exception creating status blob: {0}",
e)
"Exception uploading status blob: {0}", ustr(e))
return
uploaded = False
if not HostPluginProtocol.is_default_channel():
try:
uploaded = self.status_blob.upload(blob_uri)
except HttpError as e:
pass
if not uploaded:
host = self.get_host_plugin()
host.put_vm_status(self.status_blob,
ext_conf.status_upload_blob,
ext_conf.status_upload_blob_type)
def report_role_prop(self, thumbprint):
goal_state = self.get_goal_state()
role_prop = _build_role_properties(goal_state.container_id,
@@ -898,11 +934,12 @@ class WireClient(object):
health_report_uri,
health_report,
headers=headers,
max_retry=30)
max_retry=30,
retry_delay=15)
except HttpError as e:
raise ProtocolError((u"Failed to send provision status: "
u"{0}").format(e))
if resp.status != httpclient.OK:
if restutil.request_failed(resp):
raise ProtocolError((u"Failed to send provision status: "
u",{0}: {1}").format(resp.status,
resp.read()))
@@ -921,7 +958,7 @@ class WireClient(object):
except HttpError as e:
raise ProtocolError("Failed to send events:{0}".format(e))
if resp.status != httpclient.OK:
if restutil.request_failed(resp):
logger.verbose(resp.read())
raise ProtocolError(
"Failed to send events:{0}".format(resp.status))
@@ -981,12 +1018,8 @@ class WireClient(object):
"x-ms-guest-agent-public-x509-cert": cert
}
def get_host_plugin(self, force_update=False):
if self.host_plugin is None or force_update:
if force_update:
logger.warn("Forcing update of goal state")
self.goal_state = None
self.update_goal_state(forced=True)
def get_host_plugin(self):
if self.host_plugin is None:
goal_state = self.get_goal_state()
self.host_plugin = HostPluginProtocol(self.endpoint,
goal_state.container_id,
@@ -999,23 +1032,47 @@ class WireClient(object):
def get_artifacts_profile(self):
artifacts_profile = None
if self.has_artifacts_profile_blob():
blob = self.ext_conf.artifacts_profile_blob
logger.verbose("Getting the artifacts profile")
profile = self.fetch(blob)
for update_goal_state in [False, True]:
try:
if update_goal_state:
self.update_goal_state(forced=True)
if profile is None:
logger.warn("Download failed, falling back to host plugin")
host = self.get_host_plugin()
uri, headers = host.get_artifact_request(blob)
profile = self.decode_config(self.fetch(uri, headers, chk_proxy=False))
if self.has_artifacts_profile_blob():
blob = self.ext_conf.artifacts_profile_blob
if not textutil.is_str_none_or_whitespace(profile):
logger.verbose("Artifacts profile downloaded successfully")
artifacts_profile = InVMArtifactsProfile(profile)
profile = None
if not HostPluginProtocol.is_default_channel():
logger.verbose("Retrieving the artifacts profile")
profile = self.fetch(blob)
return artifacts_profile
if profile is None:
if HostPluginProtocol.is_default_channel():
logger.verbose("Using host plugin as default channel")
else:
logger.verbose("Failed to download artifacts profile, "
"switching to host plugin")
host = self.get_host_plugin()
uri, headers = host.get_artifact_request(blob)
config = self.fetch(uri, headers, chk_proxy=False)
profile = self.decode_config(config)
if not textutil.is_str_none_or_whitespace(profile):
logger.verbose("Artifacts profile downloaded")
artifacts_profile = InVMArtifactsProfile(profile)
return artifacts_profile
except BadRequestError:
HostPluginProtocol.set_default_channel(True)
continue
except Exception as e:
logger.warn(
"Exception retrieving artifacts profile: {0}".format(
ustr(e)))
return None
class VersionInfo(object):
def __init__(self, xml_text):
azurelinuxagent/common/utils/restutil.py
+230 -80
View File
@@ -18,19 +18,83 @@
#
import time
import traceback
import azurelinuxagent.common.conf as conf
import azurelinuxagent.common.logger as logger
from azurelinuxagent.common.exception import HttpError
from azurelinuxagent.common.future import httpclient, urlparse
import azurelinuxagent.common.utils.textutil as textutil
"""
REST api util functions
"""
from azurelinuxagent.common.exception import BadRequestError, HttpError
from azurelinuxagent.common.future import httpclient, urlparse, ustr
from azurelinuxagent.common.version import PY_VERSION_MAJOR
RETRY_WAITING_INTERVAL = 3
secure_warning = True
SECURE_WARNING_EMITTED = False
DEFAULT_RETRIES = 3
SHORT_DELAY_IN_SECONDS = 5
LONG_DELAY_IN_SECONDS = 15
RETRY_CODES = [
httpclient.RESET_CONTENT,
httpclient.PARTIAL_CONTENT,
httpclient.FORBIDDEN,
httpclient.INTERNAL_SERVER_ERROR,
httpclient.NOT_IMPLEMENTED,
httpclient.SERVICE_UNAVAILABLE,
httpclient.GATEWAY_TIMEOUT,
httpclient.INSUFFICIENT_STORAGE
]
OK_CODES = [
httpclient.OK,
httpclient.CREATED,
httpclient.ACCEPTED
]
THROTTLE_CODES = [
httpclient.FORBIDDEN,
httpclient.SERVICE_UNAVAILABLE
]
RETRY_EXCEPTIONS = [
httpclient.NotConnected,
httpclient.IncompleteRead,
httpclient.ImproperConnectionState,
httpclient.BadStatusLine
]
# Note:
# - The Python library does not define constants for all possible
# errno values; these come from the standard C/C++ header
RETRY_IOERRORS = [
64, # ENONET -- Machine is not on the network
67, # ENOLINK -- Link has been severed
70, # ECOMM -- Communication error on send
78, # EREMCHG -- Remote address changed
85, # ERESTART -- Interrupted system call should be restarted
100, # ENETDOWN -- Network is down
101, # ENETUNREACH -- Network is unreachable
102, # ENETRESET -- Network dropped connection because of reset
103, # ECONNABORTED -- Software caused connection abort
104, # ECONNRESET -- Connection reset by peer
111, # ECONNREFUSED -- Connection refused
112 # EHOSTDOWN -- Host is down
]
def _is_retry_status(status, retry_codes=RETRY_CODES):
return status in retry_codes
def _is_retry_errno(errno):
return errno in RETRY_IOERRORS
def _is_retry_exception(e):
return len([x for x in RETRY_EXCEPTIONS if isinstance(e, x)]) > 0
def _is_throttle_status(status):
return status in THROTTLE_CODES
def _parse_url(url):
o = urlparse(url)
@@ -45,11 +109,7 @@ def _parse_url(url):
return o.hostname, o.port, secure, rel_uri
def get_http_proxy():
"""
Get http_proxy and https_proxy from environment variables.
Username and password is not supported now.
"""
def _get_http_proxy():
host = conf.get_httpproxy_host()
port = conf.get_httpproxy_port()
return host, port
@@ -98,43 +158,63 @@ def _http_request(method, host, rel_uri, port=None, data=None, secure=False,
return resp
def http_request(method, url, data, headers=None, max_retry=3,
chk_proxy=False):
"""
Sending http request to server
On error, sleep 10 and retry max_retry times.
"""
def http_request(method,
url, data, headers=None,
chk_proxy=False,
max_retry=DEFAULT_RETRIES,
retry_codes=RETRY_CODES,
retry_delay=SHORT_DELAY_IN_SECONDS):
global SECURE_WARNING_EMITTED
host, port, secure, rel_uri = _parse_url(url)
global secure_warning
# Check proxy
proxy_host, proxy_port = (None, None)
if chk_proxy:
proxy_host, proxy_port = get_http_proxy()
proxy_host, proxy_port = _get_http_proxy()
# If httplib module is not built with ssl support. Fallback to http
# If httplib module is not built with ssl support,
# fallback to HTTP if allowed
if secure and not hasattr(httpclient, "HTTPSConnection"):
secure = False
if secure_warning:
logger.warn("httplib is not built with ssl support")
secure_warning = False
if not conf.get_allow_http():
raise HttpError("HTTPS is unavailable and required")
# If httplib module doesn't support https tunnelling. Fallback to http
secure = False
if not SECURE_WARNING_EMITTED:
logger.warn("Python does not include SSL support")
SECURE_WARNING_EMITTED = True
# If httplib module doesn't support HTTPS tunnelling,
# fallback to HTTP if allowed
if secure and proxy_host is not None and proxy_port is not None \
and not hasattr(httpclient.HTTPSConnection, "set_tunnel"):
if not conf.get_allow_http():
raise HttpError("HTTPS tunnelling is unavailable and required")
secure = False
if secure_warning:
logger.warn("httplib does not support https tunnelling "
"(new in python 2.7)")
secure_warning = False
if not SECURE_WARNING_EMITTED:
logger.warn("Python does not support HTTPS tunnelling")
SECURE_WARNING_EMITTED = True
if proxy_host or proxy_port:
logger.verbose("HTTP proxy: [{0}:{1}]", proxy_host, proxy_port)
retry_msg = ''
log_msg = "HTTP {0}".format(method)
for retry in range(0, max_retry):
retry_interval = RETRY_WAITING_INTERVAL
msg = ''
attempt = 0
delay = retry_delay
while attempt < max_retry:
if attempt > 0:
logger.info("[HTTP Retry] Attempt {0} of {1}: {2}",
attempt+1,
max_retry,
msg)
time.sleep(delay)
attempt += 1
delay = retry_delay
try:
resp = _http_request(method,
host,
@@ -145,55 +225,125 @@ def http_request(method, url, data, headers=None, max_retry=3,
headers=headers,
proxy_host=proxy_host,
proxy_port=proxy_port)
logger.verbose("HTTP response status: [{0}]", resp.status)
logger.verbose("[HTTP Response] Status Code {0}", resp.status)
if request_failed(resp):
if _is_retry_status(resp.status, retry_codes=retry_codes):
msg = '[HTTP Retry] HTTP {0} Status Code {1}'.format(
method, resp.status)
if _is_throttle_status(resp.statue):
delay = LONG_DELAY_IN_SECONDS
logger.info("[HTTP Delay] Delay {0} seconds for " \
"Status Code {1}".format(
delay, resp.status))
continue
if resp.status == httpclient.BAD_REQUEST:
raise BadRequestError()
return resp
except httpclient.HTTPException as e:
retry_msg = 'HTTP exception: {0} {1}'.format(log_msg, e)
retry_interval = 5
msg = '[HTTP Failed] HTTP {0} HttpException {1}'.format(method, e)
if _is_retry_exception(e):
continue
break
except IOError as e:
retry_msg = 'IO error: {0} {1}'.format(log_msg, e)
# error 101: network unreachable; when the adapter resets we may
# see this transient error for a short time, retry once.
if e.errno == 101:
retry_interval = RETRY_WAITING_INTERVAL
max_retry = 1
msg = '[HTTP Failed] HTTP {0} IOError {1}'.format(method, e)
if _is_retry_errno(e.errno):
continue
break
raise HttpError(msg)
def http_get(url, headers=None, chk_proxy=False,
max_retry=DEFAULT_RETRIES,
retry_codes=RETRY_CODES,
retry_delay=SHORT_DELAY_IN_SECONDS):
return http_request("GET",
url, None, headers=headers,
chk_proxy=chk_proxy,
max_retry=max_retry,
retry_codes=retry_codes,
retry_delay=retry_delay)
def http_head(url, headers=None, chk_proxy=False,
max_retry=DEFAULT_RETRIES,
retry_codes=RETRY_CODES,
retry_delay=SHORT_DELAY_IN_SECONDS):
return http_request("HEAD",
url, None, headers=headers,
chk_proxy=chk_proxy,
max_retry=max_retry,
retry_codes=retry_codes,
retry_delay=retry_delay)
def http_post(url, data, headers=None, chk_proxy=False,
max_retry=DEFAULT_RETRIES,
retry_codes=RETRY_CODES,
retry_delay=SHORT_DELAY_IN_SECONDS):
return http_request("POST",
url, data, headers=headers,
chk_proxy=chk_proxy,
max_retry=max_retry,
retry_codes=retry_codes,
retry_delay=retry_delay)
def http_put(url, data, headers=None, chk_proxy=False,
max_retry=DEFAULT_RETRIES,
retry_codes=RETRY_CODES,
retry_delay=SHORT_DELAY_IN_SECONDS):
return http_request("PUT",
url, data, headers=headers,
chk_proxy=chk_proxy,
max_retry=max_retry,
retry_codes=retry_codes,
retry_delay=retry_delay)
def http_delete(url, headers=None, chk_proxy=False,
max_retry=DEFAULT_RETRIES,
retry_codes=RETRY_CODES,
retry_delay=SHORT_DELAY_IN_SECONDS):
return http_request("DELETE",
url, None, headers=headers,
chk_proxy=chk_proxy,
max_retry=max_retry,
retry_codes=retry_codes,
retry_delay=retry_delay)
def request_failed(resp, ok_codes=OK_CODES):
return not request_succeeded(resp, ok_codes=ok_codes)
def request_succeeded(resp, ok_codes=OK_CODES):
return resp is not None and resp.status in ok_codes
def read_response_error(resp):
result = ''
if resp is not None:
try:
result = "[HTTP Failed] [{0}: {1}] {2}".format(
resp.status,
resp.reason,
resp.read())
# this result string is passed upstream to several methods
# which do a raise HttpError() or a format() of some kind;
# as a result it cannot have any unicode characters
if PY_VERSION_MAJOR < 3:
result = ustr(result, encoding='ascii', errors='ignore')
else:
retry_interval = 0
max_retry = 0
result = result\
.encode(encoding='ascii', errors='ignore')\
.decode(encoding='ascii', errors='ignore')
if retry < max_retry:
logger.info("Retry [{0}/{1} - {3}]",
retry+1,
max_retry,
retry_interval,
retry_msg)
time.sleep(retry_interval)
result = textutil.replace_non_ascii(result)
raise HttpError("{0} failed".format(log_msg))
def http_get(url, headers=None, max_retry=3, chk_proxy=False):
return http_request("GET", url, data=None, headers=headers,
max_retry=max_retry, chk_proxy=chk_proxy)
def http_head(url, headers=None, max_retry=3, chk_proxy=False):
return http_request("HEAD", url, None, headers=headers,
max_retry=max_retry, chk_proxy=chk_proxy)
def http_post(url, data, headers=None, max_retry=3, chk_proxy=False):
return http_request("POST", url, data, headers=headers,
max_retry=max_retry, chk_proxy=chk_proxy)
def http_put(url, data, headers=None, max_retry=3, chk_proxy=False):
return http_request("PUT", url, data, headers=headers,
max_retry=max_retry, chk_proxy=chk_proxy)
def http_delete(url, headers=None, max_retry=3, chk_proxy=False):
return http_request("DELETE", url, None, headers=headers,
max_retry=max_retry, chk_proxy=chk_proxy)
# End REST api util functions
except Exception:
logger.warn(traceback.format_exc())
return result
azurelinuxagent/common/utils/textutil.py
+11
View File
@@ -259,6 +259,17 @@ def set_ini_config(config, name, val):
config.insert(length - 1, text)
def replace_non_ascii(incoming, replace_char=''):
outgoing = ''
if incoming is not None:
for c in incoming:
if str_to_ord(c) > 128:
outgoing += replace_char
else:
outgoing += c
return outgoing
def remove_bom(c):
'''
bom is comprised of a sequence of three chars,0xef, 0xbb, 0xbf, in case of utf-8.
azurelinuxagent/common/version.py
+1 -1
View File
@@ -113,7 +113,7 @@ def get_distro():
AGENT_NAME = "WALinuxAgent"
AGENT_LONG_NAME = "Azure Linux Agent"
AGENT_VERSION = '2.2.14.1'
AGENT_VERSION = '2.2.14.2'
AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION)
AGENT_DESCRIPTION = """
The Azure Linux Agent supports the provisioning and running of Linux
azurelinuxagent/ga/update.py
+110 -90
View File
@@ -41,7 +41,8 @@ import azurelinuxagent.common.utils.textutil as textutil
from azurelinuxagent.common.event import add_event, add_periodic, \
elapsed_milliseconds, \
WALAEventOperation
from azurelinuxagent.common.exception import UpdateError, ProtocolError
from azurelinuxagent.common.exception import BadRequestError, \
ProtocolError, UpdateError
from azurelinuxagent.common.future import ustr
from azurelinuxagent.common.osutil import get_osutil
from azurelinuxagent.common.protocol import get_protocol_util
@@ -349,84 +350,6 @@ class UpdateHandler(object):
self._set_sentinal()
return
def _upgrade_available(self, base_version=CURRENT_VERSION):
# Emit an event expressing the state of AutoUpdate
# Note:
# - Duplicate events get suppressed; state transitions always emit
add_event(
AGENT_NAME,
version=CURRENT_VERSION,
op=WALAEventOperation.AutoUpdate,
is_success=conf.get_autoupdate_enabled())
# Ignore new agents if updating is disabled
if not conf.get_autoupdate_enabled():
return False
now = time.time()
if self.last_attempt_time is not None:
next_attempt_time = self.last_attempt_time + conf.get_autoupdate_frequency()
else:
next_attempt_time = now
if next_attempt_time > now:
return False
family = conf.get_autoupdate_gafamily()
logger.verbose("Checking for agent family {0} updates", family)
self.last_attempt_time = now
try:
protocol = self.protocol_util.get_protocol()
manifest_list, etag = protocol.get_vmagent_manifests()
except Exception as e:
msg = u"Exception retrieving agent manifests: {0}".format(ustr(e))
logger.warn(msg)
add_event(
AGENT_NAME,
op=WALAEventOperation.Download,
version=CURRENT_VERSION,
is_success=False,
message=msg)
return False
manifests = [m for m in manifest_list.vmAgentManifests \
if m.family == family and len(m.versionsManifestUris) > 0]
if len(manifests) == 0:
logger.verbose(u"Incarnation {0} has no agent family {1} updates", etag, family)
return False
try:
pkg_list = protocol.get_vmagent_pkgs(manifests[0])
except ProtocolError as e:
msg = u"Incarnation {0} failed to get {1} package list: " \
u"{2}".format(
etag,
family,
ustr(e))
logger.warn(msg)
add_event(
AGENT_NAME,
op=WALAEventOperation.Download,
version=CURRENT_VERSION,
is_success=False,
message=msg)
return False
# Set the agents to those available for download at least as current
# as the existing agent and remove from disk any agent no longer
# reported to the VM.
# Note:
# The code leaves on disk available, but blacklisted, agents so as to
# preserve the state. Otherwise, those agents could be again
# downloaded and inappropriately retried.
host = self._get_host_plugin(protocol=protocol)
self._set_agents([GuestAgent(pkg=pkg, host=host) for pkg in pkg_list.versions])
self._purge_agents()
self._filter_blacklisted_agents()
# Return True if agents more recent than the current are available
return len(self.agents) > 0 and self.agents[0].version > base_version
def _ensure_no_orphans(self, orphan_wait_interval=ORPHAN_WAIT_INTERVAL):
pid_files, ignored = self._write_pid_file()
for pid_file in pid_files:
@@ -615,6 +538,85 @@ class UpdateHandler(object):
str(e))
return
def _upgrade_available(self, base_version=CURRENT_VERSION):
# Emit an event expressing the state of AutoUpdate
# Note:
# - Duplicate events get suppressed; state transitions always emit
add_event(
AGENT_NAME,
version=CURRENT_VERSION,
op=WALAEventOperation.AutoUpdate,
is_success=conf.get_autoupdate_enabled())
# Ignore new agents if updating is disabled
if not conf.get_autoupdate_enabled():
return False
now = time.time()
if self.last_attempt_time is not None:
next_attempt_time = self.last_attempt_time + \
conf.get_autoupdate_frequency()
else:
next_attempt_time = now
if next_attempt_time > now:
return False
family = conf.get_autoupdate_gafamily()
logger.verbose("Checking for agent family {0} updates", family)
self.last_attempt_time = now
protocol = self.protocol_util.get_protocol()
for update_goal_state in [False, True]:
try:
if update_goal_state:
protocol.update_goal_state(forced=True)
manifest_list, etag = protocol.get_vmagent_manifests()
manifests = [m for m in manifest_list.vmAgentManifests \
if m.family == family and \
len(m.versionsManifestUris) > 0]
if len(manifests) == 0:
logger.verbose(u"Incarnation {0} has no {1} agent updates",
etag, family)
return False
pkg_list = protocol.get_vmagent_pkgs(manifests[0])
# Set the agents to those available for download at least as
# current as the existing agent and remove from disk any agent
# no longer reported to the VM.
# Note:
# The code leaves on disk available, but blacklisted, agents
# so as to preserve the state. Otherwise, those agents could be
# again downloaded and inappropriately retried.
host = self._get_host_plugin(protocol=protocol)
self._set_agents([GuestAgent(pkg=pkg, host=host) \
for pkg in pkg_list.versions])
self._purge_agents()
self._filter_blacklisted_agents()
# Return True if more recent agents are available
return len(self.agents) > 0 and \
self.agents[0].version > base_version
except Exception as e:
if isinstance(e, BadRequestError):
continue
msg = u"Exception retrieving agent manifests: {0}".format(
ustr(e))
logger.warn(msg)
add_event(
AGENT_NAME,
op=WALAEventOperation.Download,
version=CURRENT_VERSION,
is_success=False,
message=msg)
return False
def _write_pid_file(self):
pid_files = self._get_pid_files()
@@ -671,12 +673,16 @@ class GuestAgent(object):
self._ensure_downloaded()
self._ensure_loaded()
except Exception as e:
if isinstance(e, BadRequestError):
raise
# Note the failure, blacklist the agent if the package downloaded
# - An exception with a downloaded package indicates the package
# is corrupt (e.g., missing the HandlerManifest.json file)
self.mark_failure(is_fatal=os.path.isfile(self.get_agent_pkg_path()))
msg = u"Agent {0} download / load failed with exception: {1}".format(self.name, ustr(e))
msg = u"Agent {0} install failed with exception: {1}".format(
self.name, ustr(e))
logger.warn(msg)
add_event(
AGENT_NAME,
@@ -795,20 +801,29 @@ class GuestAgent(object):
for uri in self.pkg.uris:
if not HostPluginProtocol.is_default_channel() and self._fetch(uri.uri):
break
elif self.host is not None and self.host.ensure_initialized():
if not HostPluginProtocol.is_default_channel():
logger.warn("Download unsuccessful, falling back to host plugin")
logger.warn("Download failed, switching to host plugin")
else:
logger.verbose("Using host plugin as default channel")
uri, headers = self.host.get_artifact_request(uri.uri, self.host.manifest_uri)
if self._fetch(uri, headers=headers, chk_proxy=False):
if not HostPluginProtocol.is_default_channel():
logger.verbose("Setting host plugin as default channel")
HostPluginProtocol.set_default_channel(True)
break
else:
logger.warn("Host plugin download unsuccessful")
try:
if self._fetch(uri, headers=headers, chk_proxy=False):
if not HostPluginProtocol.is_default_channel():
logger.verbose("Setting host plugin as default channel")
HostPluginProtocol.set_default_channel(True)
break
else:
logger.warn("Host plugin download failed")
# If the HostPlugin rejects the request,
# let the error continue, but set to use the HostPlugin
except BadRequestError:
HostPluginProtocol.set_default_channel(True)
raise
else:
logger.error("No download channels available")
@@ -821,13 +836,14 @@ class GuestAgent(object):
is_success=False,
message=msg)
raise UpdateError(msg)
return
def _fetch(self, uri, headers=None, chk_proxy=True):
package = None
try:
resp = restutil.http_get(uri, chk_proxy=chk_proxy, headers=headers)
if resp.status == restutil.httpclient.OK:
if restutil.request_succeeded(resp):
package = resp.read()
fileutil.write_file(self.get_agent_pkg_path(),
bytearray(package),
@@ -835,12 +851,16 @@ class GuestAgent(object):
logger.verbose(u"Agent {0} downloaded from {1}", self.name, uri)
else:
logger.verbose("Fetch was unsuccessful [{0}]",
HostPluginProtocol.read_response_error(resp))
restutil.read_response_error(resp))
except restutil.HttpError as http_error:
if isinstance(http_error, BadRequestError):
raise
logger.verbose(u"Agent {0} download from {1} failed [{2}]",
self.name,
uri,
http_error)
return package is not None
def _load_error(self):
config/coreos/waagent.conf
+4
View File
@@ -107,3 +107,7 @@ OS.OpensslPath=None
# handling until inVMArtifactsProfile.OnHold is false.
# Default is disabled
# EnableOverProvisioning=n
# Allow fallback to HTTP if HTTPS is unavailable
# Note: Allowing HTTP (vs. HTTPS) may cause security risks
OS.AllowHTTP=y
config/waagent.conf
+4
View File
@@ -104,3 +104,7 @@ OS.SshDir=/etc/ssh
# handling until inVMArtifactsProfile.OnHold is false.
# Default is disabled
# EnableOverProvisioning=n
# Allow fallback to HTTP if HTTPS is unavailable
# Note: Allowing HTTP (vs. HTTPS) may cause security risks
# OS.AllowHTTP=n
tests/common/test_conf.py
+50
View File
@@ -24,6 +24,48 @@ from tests.tools import *
class TestConf(AgentTestCase):
# Note:
# -- These values *MUST* match those from data/test_waagent.conf
EXPECTED_CONFIGURATION = {
"Provisioning.Enabled" : True,
"Provisioning.UseCloudInit" : True,
"Provisioning.DeleteRootPassword" : True,
"Provisioning.RegenerateSshHostKeyPair" : True,
"Provisioning.SshHostKeyPairType" : "rsa",
"Provisioning.MonitorHostName" : True,
"Provisioning.DecodeCustomData" : False,
"Provisioning.ExecuteCustomData" : False,
"Provisioning.PasswordCryptId" : '6',
"Provisioning.PasswordCryptSaltLength" : 10,
"Provisioning.AllowResetSysUser" : False,
"ResourceDisk.Format" : True,
"ResourceDisk.Filesystem" : "ext4",
"ResourceDisk.MountPoint" : "/mnt/resource",
"ResourceDisk.EnableSwap" : False,
"ResourceDisk.SwapSizeMB" : 0,
"ResourceDisk.MountOptions" : None,
"Logs.Verbose" : False,
"OS.EnableFIPS" : True,
"OS.RootDeviceScsiTimeout" : '300',
"OS.OpensslPath" : '/usr/bin/openssl',
"OS.SshDir" : "/notareal/path",
"HttpProxy.Host" : None,
"HttpProxy.Port" : None,
"DetectScvmmEnv" : False,
"Lib.Dir" : "/var/lib/waagent",
"DVD.MountPoint" : "/mnt/cdrom/secure",
"Pid.File" : "/var/run/waagent.pid",
"Extension.LogDir" : "/var/log/azure",
"OS.HomeDir" : "/home",
"OS.EnableRDMA" : False,
"OS.UpdateRdmaDriver" : False,
"OS.CheckRdmaDriver" : False,
"AutoUpdate.Enabled" : True,
"AutoUpdate.GAFamily" : "Prod",
"EnableOverProvisioning" : False,
"OS.AllowHTTP" : False
}
def setUp(self):
AgentTestCase.setUp(self)
self.conf = ConfigurationProvider()
@@ -59,3 +101,11 @@ class TestConf(AgentTestCase):
def test_get_provision_cloudinit(self):
self.assertTrue(get_provision_cloudinit(self.conf))
def test_get_configuration(self):
configuration = conf.get_configuration(self.conf)
self.assertTrue(len(configuration.keys()) > 0)
for k in TestConf.EXPECTED_CONFIGURATION.keys():
self.assertEqual(
TestConf.EXPECTED_CONFIGURATION[k],
configuration[k])
tests/data/test_waagent.conf
+5 -2
View File
@@ -94,10 +94,13 @@ OS.SshDir=/notareal/path
# Extension.LogDir=/var/log/azure
#
# Home.Dir=/home
# OS.HomeDir=/home
# Enable RDMA management and set up, should only be used in HPC images
# OS.EnableRDMA=y
# OS.EnableRDMA=n
# OS.UpdateRdmaDriver=n
# OS.CheckRdmaDriver=n
# Enable or disable goal state processing auto-update, default is enabled
# AutoUpdate.Enabled=y
tests/ga/test_update.py
+40 -5
View File
@@ -828,6 +828,12 @@ class TestUpdate(UpdateTestCase):
self.event_patch.stop()
return
def _create_protocol(self, count=5, versions=None):
latest_version = self.prepare_agents(count=count)
if versions is None or len(versions) <= 0:
versions = [latest_version]
return ProtocolMock(versions=versions)
def _test_upgrade_available(
self,
base_version=FlexibleVersion(AGENT_VERSION),
@@ -835,12 +841,9 @@ class TestUpdate(UpdateTestCase):
versions=None,
count=5):
latest_version = self.prepare_agents(count=count)
if versions is None or len(versions) <= 0:
versions = [latest_version]
if protocol is None:
protocol = ProtocolMock(versions=versions)
protocol = self._create_protocol(count=count, versions=versions)
self.update_handler.protocol_util = protocol
conf.get_autoupdate_gafamily = Mock(return_value=protocol.family)
@@ -850,6 +853,16 @@ class TestUpdate(UpdateTestCase):
self.assertTrue(self._test_upgrade_available())
return
def test_upgrade_available_will_refresh_goal_state(self):
protocol = self._create_protocol()
protocol.emulate_stale_goal_state()
self.assertTrue(self._test_upgrade_available(protocol=protocol))
self.assertEqual(2, protocol.call_counts["get_vmagent_manifests"])
self.assertEqual(1, protocol.call_counts["get_vmagent_pkgs"])
self.assertEqual(1, protocol.call_counts["update_goal_state"])
self.assertTrue(protocol.goal_state_forced)
return
def test_get_latest_agent_excluded(self):
self.prepare_agent(AGENT_VERSION)
self.assertFalse(self._test_upgrade_available(
@@ -1549,12 +1562,22 @@ class ProtocolMock(object):
def __init__(self, family="TestAgent", etag=42, versions=None, client=None):
self.family = family
self.client = client
self.call_counts = {
"get_vmagent_manifests" : 0,
"get_vmagent_pkgs" : 0,
"update_goal_state" : 0
}
self.goal_state_is_stale = False
self.goal_state_forced = False
self.etag = etag
self.versions = versions if versions is not None else []
self.create_manifests()
self.create_packages()
return
def emulate_stale_goal_state(self):
self.goal_state_is_stale = True
def create_manifests(self):
self.agent_manifests = VMAgentManifestList()
if len(self.versions) <= 0:
@@ -1585,11 +1608,23 @@ class ProtocolMock(object):
return self
def get_vmagent_manifests(self):
self.call_counts["get_vmagent_manifests"] += 1
if self.goal_state_is_stale:
self.goal_state_is_stale = False
raise BadRequestError()
return self.agent_manifests, self.etag
def get_vmagent_pkgs(self, manifest):
self.call_counts["get_vmagent_pkgs"] += 1
if self.goal_state_is_stale:
self.goal_state_is_stale = False
raise BadRequestError()
return self.agent_packages
def update_goal_state(self, forced=False, max_retry=3):
self.call_counts["update_goal_state"] += 1
self.goal_state_forced = self.goal_state_forced or forced
return
class ResponseMock(Mock):
def __init__(self, status=restutil.httpclient.OK, response=None, reason=None):
tests/protocol/mockwiredata.py
+77 -24
View File
@@ -16,6 +16,7 @@
#
from tests.tools import *
from azurelinuxagent.common.exception import BadRequestError, HttpError
from azurelinuxagent.common.future import httpclient
from azurelinuxagent.common.utils.cryptutil import CryptUtil
@@ -53,6 +54,20 @@ DATA_FILE_EXT_AUTOUPGRADE_INTERNALVERSION["ext_conf"] = "wire/ext_conf_autoupgra
class WireProtocolData(object):
def __init__(self, data_files=DATA_FILE):
self.emulate_stale_goal_state = False
self.call_counts = {
"comp=versions" : 0,
"/versions" : 0,
"goalstate" : 0,
"hostingenvuri" : 0,
"sharedconfiguri" : 0,
"certificatesuri" : 0,
"extensionsconfiguri" : 0,
"extensionArtifact" : 0,
"manifest.xml" : 0,
"manifest_of_ga.xml" : 0,
"ExampleHandlerLinux" : 0
}
self.version_info = load_data(data_files.get("version_info"))
self.goal_state = load_data(data_files.get("goal_state"))
self.hosting_env = load_data(data_files.get("hosting_env"))
@@ -67,32 +82,70 @@ class WireProtocolData(object):
def mock_http_get(self, url, *args, **kwargs):
content = None
if "versions" in url:
content = self.version_info
elif "goalstate" in url:
content = self.goal_state
elif "hostingenvuri" in url:
content = self.hosting_env
elif "sharedconfiguri" in url:
content = self.shared_config
elif "certificatesuri" in url:
content = self.certs
elif "extensionsconfiguri" in url:
content = self.ext_conf
elif "manifest.xml" in url:
content = self.manifest
elif "manifest_of_ga.xml" in url:
content = self.ga_manifest
elif "ExampleHandlerLinux" in url:
content = self.ext
resp = MagicMock()
resp.status = httpclient.OK
resp.read = Mock(return_value=content)
return resp
else:
raise Exception("Bad url {0}".format(url))
resp = MagicMock()
resp.status = httpclient.OK
# wire server versions
if "comp=versions" in url:
content = self.version_info
self.call_counts["comp=versions"] += 1
# HostPlugin versions
elif "/versions" in url:
content = '["2015-09-01"]'
self.call_counts["/versions"] += 1
elif "goalstate" in url:
content = self.goal_state
self.call_counts["goalstate"] += 1
elif "hostingenvuri" in url:
content = self.hosting_env
self.call_counts["hostingenvuri"] += 1
elif "sharedconfiguri" in url:
content = self.shared_config
self.call_counts["sharedconfiguri"] += 1
elif "certificatesuri" in url:
content = self.certs
self.call_counts["certificatesuri"] += 1
elif "extensionsconfiguri" in url:
content = self.ext_conf
self.call_counts["extensionsconfiguri"] += 1
else:
# A stale GoalState results in a 400 from the HostPlugin
# for which the HTTP handler in restutil raises BadRequestError
if self.emulate_stale_goal_state:
if "extensionArtifact" in url:
self.emulate_stale_goal_state = False
self.call_counts["extensionArtifact"] += 1
raise BadRequestError()
else:
raise HttpError()
# For HostPlugin requests, replace the URL with that passed
# via the x-ms-artifact-location header
if "extensionArtifact" in url:
self.call_counts["extensionArtifact"] += 1
if "headers" not in kwargs or \
"x-ms-artifact-location" not in kwargs["headers"]:
raise Exception("Bad HEADERS passed to HostPlugin: {0}",
kwargs)
url = kwargs["headers"]["x-ms-artifact-location"]
if "manifest.xml" in url:
content = self.manifest
self.call_counts["manifest.xml"] += 1
elif "manifest_of_ga.xml" in url:
content = self.ga_manifest
self.call_counts["manifest_of_ga.xml"] += 1
elif "ExampleHandlerLinux" in url:
content = self.ext
self.call_counts["ExampleHandlerLinux"] += 1
resp.read = Mock(return_value=content)
return resp
else:
raise Exception("Bad url {0}".format(url))
resp.read = Mock(return_value=content.encode("utf-8"))
return resp
tests/protocol/test_hostplugin.py
+6 -55
View File
@@ -146,6 +146,7 @@ class TestHostPlugin(AgentTestCase):
test_goal_state = wire.GoalState(WireProtocolData(DATA_FILE).goal_state)
status = restapi.VMStatus(status="Ready",
message="Guest Agent is running")
wire.HostPluginProtocol.set_default_channel(False)
with patch.object(wire.HostPluginProtocol,
"ensure_initialized",
return_value=True):
@@ -173,6 +174,7 @@ class TestHostPlugin(AgentTestCase):
test_goal_state = wire.GoalState(WireProtocolData(DATA_FILE).goal_state)
status = restapi.VMStatus(status="Ready",
message="Guest Agent is running")
wire.HostPluginProtocol.set_default_channel(False)
with patch.object(wire.StatusBlob,
"upload",
return_value=False):
@@ -211,6 +213,8 @@ class TestHostPlugin(AgentTestCase):
bytearray(faux_status, encoding='utf-8'))
with patch.object(restutil, "http_request") as patch_http:
patch_http.return_value = Mock(status=httpclient.OK)
wire_protocol_client.get_goal_state = Mock(return_value=test_goal_state)
plugin = wire_protocol_client.get_host_plugin()
@@ -224,61 +228,6 @@ class TestHostPlugin(AgentTestCase):
test_goal_state,
exp_method, exp_url, exp_data)
def test_read_response_error(self):
"""
Validate the read_response_error method handles encoding correctly
"""
responses = ['message', b'message', '\x80message\x80']
response = MagicMock()
response.status = 'status'
response.reason = 'reason'
with patch.object(response, 'read') as patch_response:
for s in responses:
patch_response.return_value = s
result = hostplugin.HostPluginProtocol.read_response_error(response)
self.assertTrue('[status: reason]' in result)
self.assertTrue('message' in result)
def test_read_response_bytes(self):
response_bytes = '7b:0a:20:20:20:20:22:65:72:72:6f:72:43:6f:64:65:22:' \
'3a:20:22:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:' \
'69:73:20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:' \
'69:73:20:6f:70:65:72:61:74:69:6f:6e:2e:22:2c:0a:20:' \
'20:20:20:22:6d:65:73:73:61:67:65:22:3a:20:22:c3:af:' \
'c2:bb:c2:bf:3c:3f:78:6d:6c:20:76:65:72:73:69:6f:6e:' \
'3d:22:31:2e:30:22:20:65:6e:63:6f:64:69:6e:67:3d:22:' \
'75:74:66:2d:38:22:3f:3e:3c:45:72:72:6f:72:3e:3c:43:' \
'6f:64:65:3e:49:6e:76:61:6c:69:64:42:6c:6f:62:54:79:' \
'70:65:3c:2f:43:6f:64:65:3e:3c:4d:65:73:73:61:67:65:' \
'3e:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:69:73:' \
'20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:69:73:' \
'20:6f:70:65:72:61:74:69:6f:6e:2e:0a:52:65:71:75:65:' \
'73:74:49:64:3a:63:37:34:32:39:30:63:62:2d:30:30:30:' \
'31:2d:30:30:62:35:2d:30:36:64:61:2d:64:64:36:36:36:' \
'61:30:30:30:22:2c:0a:20:20:20:20:22:64:65:74:61:69:' \
'6c:73:22:3a:20:22:22:0a:7d'.split(':')
expected_response = '[status: reason] {\n "errorCode": "The blob ' \
'type is invalid for this operation.",\n ' \
'"message": "<?xml version="1.0" ' \
'encoding="utf-8"?>' \
'<Error><Code>InvalidBlobType</Code><Message>The ' \
'blob type is invalid for this operation.\n' \
'RequestId:c74290cb-0001-00b5-06da-dd666a000",' \
'\n "details": ""\n}'
response_string = ''.join(chr(int(b, 16)) for b in response_bytes)
response = MagicMock()
response.status = 'status'
response.reason = 'reason'
with patch.object(response, 'read') as patch_response:
patch_response.return_value = response_string
result = hostplugin.HostPluginProtocol.read_response_error(response)
self.assertEqual(result, expected_response)
try:
raise HttpError("{0}".format(result))
except HttpError as e:
self.assertTrue(result in ustr(e))
def test_no_fallback(self):
"""
Validate fallback to upload status using HostGAPlugin is not happening
@@ -318,6 +267,8 @@ class TestHostPlugin(AgentTestCase):
bytearray(faux_status, encoding='utf-8'))
with patch.object(restutil, "http_request") as patch_http:
patch_http.return_value = Mock(status=httpclient.OK)
with patch.object(wire.HostPluginProtocol,
"get_api_versions") as patch_get:
patch_get.return_value = api_versions
tests/protocol/test_metadata.py
+9 -11
View File
@@ -31,17 +31,15 @@ class TestMetadataProtocolGetters(AgentTestCase):
return json.loads(ustr(load_data(path)), encoding="utf-8")
@patch("time.sleep")
@patch("azurelinuxagent.common.protocol.metadata.restutil")
def _test_getters(self, test_data, mock_restutil ,_):
mock_restutil.http_get.side_effect = test_data.mock_http_get
protocol = MetadataProtocol()
protocol.detect()
protocol.get_vminfo()
protocol.get_certs()
ext_handlers, etag = protocol.get_ext_handlers()
for ext_handler in ext_handlers.extHandlers:
protocol.get_ext_handler_pkgs(ext_handler)
def _test_getters(self, test_data ,_):
with patch.object(restutil, 'http_get', test_data.mock_http_get):
protocol = MetadataProtocol()
protocol.detect()
protocol.get_vminfo()
protocol.get_certs()
ext_handlers, etag = protocol.get_ext_handlers()
for ext_handler in ext_handlers.extHandlers:
protocol.get_ext_handler_pkgs(ext_handler)
def test_getters(self, *args):
test_data = MetadataProtocolData(DATA_FILE)
tests/protocol/test_wire.py
+37 -20
View File
@@ -25,30 +25,34 @@ wireserver_url = '168.63.129.16'
@patch("time.sleep")
@patch("azurelinuxagent.common.protocol.wire.CryptUtil")
@patch("azurelinuxagent.common.protocol.wire.restutil")
class TestWireProtocolGetters(AgentTestCase):
def _test_getters(self, test_data, mock_restutil, MockCryptUtil, _):
mock_restutil.http_get.side_effect = test_data.mock_http_get
def setUp(self):
super(TestWireProtocolGetters, self).setUp()
HostPluginProtocol.set_default_channel(False)
def _test_getters(self, test_data, MockCryptUtil, _):
MockCryptUtil.side_effect = test_data.mock_crypt_util
protocol = WireProtocol(wireserver_url)
protocol.detect()
protocol.get_vminfo()
protocol.get_certs()
ext_handlers, etag = protocol.get_ext_handlers()
for ext_handler in ext_handlers.extHandlers:
protocol.get_ext_handler_pkgs(ext_handler)
with patch.object(restutil, 'http_get', test_data.mock_http_get):
protocol = WireProtocol(wireserver_url)
protocol.detect()
protocol.get_vminfo()
protocol.get_certs()
ext_handlers, etag = protocol.get_ext_handlers()
for ext_handler in ext_handlers.extHandlers:
protocol.get_ext_handler_pkgs(ext_handler)
crt1 = os.path.join(self.tmp_dir,
'33B0ABCE4673538650971C10F7D7397E71561F35.crt')
crt2 = os.path.join(self.tmp_dir,
'4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.crt')
prv2 = os.path.join(self.tmp_dir,
'4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.prv')
crt1 = os.path.join(self.tmp_dir,
'33B0ABCE4673538650971C10F7D7397E71561F35.crt')
crt2 = os.path.join(self.tmp_dir,
'4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.crt')
prv2 = os.path.join(self.tmp_dir,
'4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.prv')
self.assertTrue(os.path.isfile(crt1))
self.assertTrue(os.path.isfile(crt2))
self.assertTrue(os.path.isfile(prv2))
self.assertTrue(os.path.isfile(crt1))
self.assertTrue(os.path.isfile(crt2))
self.assertTrue(os.path.isfile(prv2))
def test_getters(self, *args):
"""Normal case"""
@@ -70,8 +74,21 @@ class TestWireProtocolGetters(AgentTestCase):
test_data = WireProtocolData(DATA_FILE_EXT_NO_PUBLIC)
self._test_getters(test_data, *args)
def test_getters_with_stale_goal_state(self, *args):
test_data = WireProtocolData(DATA_FILE)
test_data.emulate_stale_goal_state = True
self._test_getters(test_data, *args)
# Ensure HostPlugin was invoked
self.assertEqual(1, test_data.call_counts["/versions"])
self.assertEqual(2, test_data.call_counts["extensionArtifact"])
# Ensure the expected number of HTTP calls were made
# -- Tracking calls to retrieve GoalState is problematic since it is
# fetched often; however, the dependent documents, such as the
# HostingEnvironmentConfig, will be retrieved the expected number
self.assertEqual(2, test_data.call_counts["hostingenvuri"])
def test_call_storage_kwargs(self,
mock_restutil,
mock_cryptutil,
mock_sleep):
from azurelinuxagent.common.utils import restutil
tests/test_agent.py
+51
View File
@@ -17,12 +17,55 @@
import mock
import os.path
import sys
from azurelinuxagent.agent import *
from azurelinuxagent.common.conf import *
from tests.tools import *
EXPECTED_CONFIGURATION = \
"""AutoUpdate.Enabled = True
AutoUpdate.GAFamily = Prod
Autoupdate.Frequency = 3600
DVD.MountPoint = /mnt/cdrom/secure
DetectScvmmEnv = False
EnableOverProvisioning = False
Extension.LogDir = /var/log/azure
HttpProxy.Host = None
HttpProxy.Port = None
Lib.Dir = /var/lib/waagent
Logs.Verbose = False
OS.AllowHTTP = False
OS.CheckRdmaDriver = False
OS.EnableFIPS = True
OS.EnableRDMA = False
OS.HomeDir = /home
OS.OpensslPath = /usr/bin/openssl
OS.PasswordPath = /etc/shadow
OS.RootDeviceScsiTimeout = 300
OS.SshDir = /notareal/path
OS.SudoersDir = /etc/sudoers.d
OS.UpdateRdmaDriver = False
Pid.File = /var/run/waagent.pid
Provisioning.AllowResetSysUser = False
Provisioning.DecodeCustomData = False
Provisioning.DeleteRootPassword = True
Provisioning.Enabled = True
Provisioning.ExecuteCustomData = False
Provisioning.MonitorHostName = True
Provisioning.PasswordCryptId = 6
Provisioning.PasswordCryptSaltLength = 10
Provisioning.RegenerateSshHostKeyPair = True
Provisioning.SshHostKeyPairType = rsa
Provisioning.UseCloudInit = True
ResourceDisk.EnableSwap = False
ResourceDisk.Filesystem = ext4
ResourceDisk.Format = True
ResourceDisk.MountOptions = None
ResourceDisk.MountPoint = /mnt/resource
ResourceDisk.SwapSizeMB = 0
""".split('\n')
class TestAgent(AgentTestCase):
@@ -115,3 +158,11 @@ class TestAgent(AgentTestCase):
self.assertTrue(os.path.isfile(ext_log_dir))
self.assertFalse(os.path.isdir(ext_log_dir))
mock_log.assert_called_once()
def test_agent_show_configuration(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('Test requires at least Python 2.7 with buffered output')
agent = Agent(False,
conf_file_path=os.path.join(data_dir, "test_waagent.conf"))
agent.show_configuration()
self.assertEqual(EXPECTED_CONFIGURATION, sys.stdout.getvalue().split('\n'))
tests/utils/test_rest_util.py
+174 -1
View File
@@ -16,8 +16,12 @@
#
import unittest
from azurelinuxagent.common.exception import BadRequestError, \
HttpError, ProtocolError
import azurelinuxagent.common.utils.restutil as restutil
from azurelinuxagent.common.future import httpclient
from azurelinuxagent.common.future import httpclient, ustr
from tests.tools import AgentTestCase, patch, Mock, MagicMock
@@ -115,6 +119,175 @@ class TestHttpOperations(AgentTestCase):
self.assertRaises(restutil.HttpError, restutil.http_get,
"http://foo.bar")
@patch("time.sleep")
@patch("azurelinuxagent.common.utils.restutil._http_request")
def test_http_request_retries_status_codes(self, _http_request, _sleep):
_http_request.side_effect = [
Mock(status=httpclient.SERVICE_UNAVAILABLE),
Mock(status=httpclient.OK)
]
restutil.http_get("https://foo.bar")
self.assertEqual(2, _http_request.call_count)
self.assertEqual(1, _sleep.call_count)
@patch("time.sleep")
@patch("azurelinuxagent.common.utils.restutil._http_request")
def test_http_request_retries_passed_status_codes(self, _http_request, _sleep):
# Ensure the code is not part of the standard set
self.assertFalse(httpclient.UNAUTHORIZED in restutil.RETRY_CODES)
_http_request.side_effect = [
Mock(status=httpclient.UNAUTHORIZED),
Mock(status=httpclient.OK)
]
restutil.http_get("https://foo.bar", retry_codes=[httpclient.UNAUTHORIZED])
self.assertEqual(2, _http_request.call_count)
self.assertEqual(1, _sleep.call_count)
@patch("time.sleep")
@patch("azurelinuxagent.common.utils.restutil._http_request")
def test_http_request_raises_for_bad_request(self, _http_request, _sleep):
_http_request.side_effect = [
Mock(status=httpclient.BAD_REQUEST)
]
self.assertRaises(BadRequestError, restutil.http_get, "https://foo.bar")
self.assertEqual(1, _http_request.call_count)
@patch("time.sleep")
@patch("azurelinuxagent.common.utils.restutil._http_request")
def test_http_request_retries_exceptions(self, _http_request, _sleep):
# Testing each exception is difficult because they have varying
# signatures; for now, test one and ensure the set is unchanged
recognized_exceptions = [
httpclient.NotConnected,
httpclient.IncompleteRead,
httpclient.ImproperConnectionState,
httpclient.BadStatusLine
]
self.assertEqual(recognized_exceptions, restutil.RETRY_EXCEPTIONS)
_http_request.side_effect = [
httpclient.IncompleteRead(''),
Mock(status=httpclient.OK)
]
restutil.http_get("https://foo.bar")
self.assertEqual(2, _http_request.call_count)
self.assertEqual(1, _sleep.call_count)
@patch("time.sleep")
@patch("azurelinuxagent.common.utils.restutil._http_request")
def test_http_request_retries_ioerrors(self, _http_request, _sleep):
ioerror = IOError()
for errno in restutil.RETRY_IOERRORS:
_http_request.reset_mock()
_sleep.reset_mock()
ioerror.errno = errno
_http_request.side_effect = [
ioerror,
Mock(status=httpclient.OK)
]
restutil.http_get("https://foo.bar")
self.assertEqual(2, _http_request.call_count)
self.assertEqual(1, _sleep.call_count)
def test_request_failed(self):
self.assertTrue(restutil.request_failed(None))
resp = Mock()
for status in restutil.OK_CODES:
resp.status = status
self.assertFalse(restutil.request_failed(resp))
self.assertFalse(httpclient.BAD_REQUEST in restutil.OK_CODES)
resp.status = httpclient.BAD_REQUEST
self.assertTrue(restutil.request_failed(resp))
self.assertFalse(
restutil.request_failed(
resp, ok_codes=[httpclient.BAD_REQUEST]))
def test_request_succeeded(self):
self.assertFalse(restutil.request_succeeded(None))
resp = Mock()
for status in restutil.OK_CODES:
resp.status = status
self.assertTrue(restutil.request_succeeded(resp))
self.assertFalse(httpclient.BAD_REQUEST in restutil.OK_CODES)
resp.status = httpclient.BAD_REQUEST
self.assertFalse(restutil.request_succeeded(resp))
self.assertTrue(
restutil.request_succeeded(
resp, ok_codes=[httpclient.BAD_REQUEST]))
def test_read_response_error(self):
"""
Validate the read_response_error method handles encoding correctly
"""
responses = ['message', b'message', '\x80message\x80']
response = MagicMock()
response.status = 'status'
response.reason = 'reason'
with patch.object(response, 'read') as patch_response:
for s in responses:
patch_response.return_value = s
result = restutil.read_response_error(response)
print("RESPONSE: {0}".format(s))
print("RESULT: {0}".format(result))
print("PRESENT: {0}".format('[status: reason]' in result))
self.assertTrue('[status: reason]' in result)
self.assertTrue('message' in result)
def test_read_response_bytes(self):
response_bytes = '7b:0a:20:20:20:20:22:65:72:72:6f:72:43:6f:64:65:22:' \
'3a:20:22:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:' \
'69:73:20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:' \
'69:73:20:6f:70:65:72:61:74:69:6f:6e:2e:22:2c:0a:20:' \
'20:20:20:22:6d:65:73:73:61:67:65:22:3a:20:22:c3:af:' \
'c2:bb:c2:bf:3c:3f:78:6d:6c:20:76:65:72:73:69:6f:6e:' \
'3d:22:31:2e:30:22:20:65:6e:63:6f:64:69:6e:67:3d:22:' \
'75:74:66:2d:38:22:3f:3e:3c:45:72:72:6f:72:3e:3c:43:' \
'6f:64:65:3e:49:6e:76:61:6c:69:64:42:6c:6f:62:54:79:' \
'70:65:3c:2f:43:6f:64:65:3e:3c:4d:65:73:73:61:67:65:' \
'3e:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:69:73:' \
'20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:69:73:' \
'20:6f:70:65:72:61:74:69:6f:6e:2e:0a:52:65:71:75:65:' \
'73:74:49:64:3a:63:37:34:32:39:30:63:62:2d:30:30:30:' \
'31:2d:30:30:62:35:2d:30:36:64:61:2d:64:64:36:36:36:' \
'61:30:30:30:22:2c:0a:20:20:20:20:22:64:65:74:61:69:' \
'6c:73:22:3a:20:22:22:0a:7d'.split(':')
expected_response = '[HTTP Failed] [status: reason] {\n "errorCode": "The blob ' \
'type is invalid for this operation.",\n ' \
'"message": "<?xml version="1.0" ' \
'encoding="utf-8"?>' \
'<Error><Code>InvalidBlobType</Code><Message>The ' \
'blob type is invalid for this operation.\n' \
'RequestId:c74290cb-0001-00b5-06da-dd666a000",' \
'\n "details": ""\n}'
response_string = ''.join(chr(int(b, 16)) for b in response_bytes)
response = MagicMock()
response.status = 'status'
response.reason = 'reason'
with patch.object(response, 'read') as patch_response:
patch_response.return_value = response_string
result = restutil.read_response_error(response)
self.assertEqual(result, expected_response)
try:
raise HttpError("{0}".format(result))
except HttpError as e:
self.assertTrue(result in ustr(e))
if __name__ == '__main__':
unittest.main()
tests/utils/test_text_util.py
+13
View File
@@ -34,6 +34,19 @@ class TestTextUtil(AgentTestCase):
password_hash = textutil.gen_password_hash(data, 6, 10)
self.assertNotEquals(None, password_hash)
def test_replace_non_ascii(self):
data = ustr(b'\xef\xbb\xbfhehe', encoding='utf-8')
self.assertEqual('hehe', textutil.replace_non_ascii(data))
data = "abcd\xa0e\xf0fghijk\xbblm"
self.assertEqual("abcdefghijklm", textutil.replace_non_ascii(data))
data = "abcd\xa0e\xf0fghijk\xbblm"
self.assertEqual("abcdXeXfghijkXlm",
textutil.replace_non_ascii(data, replace_char='X'))
self.assertEqual('', textutil.replace_non_ascii(None))
def test_remove_bom(self):
#Test bom could be removed
data = ustr(b'\xef\xbb\xbfhehe', encoding='utf-8')