diff --git a/azurelinuxagent/agent.py b/azurelinuxagent/agent.py index 8394fcf..368ad12 100644 --- a/azurelinuxagent/agent.py +++ b/azurelinuxagent/agent.py @@ -85,16 +85,16 @@ def version(): """ Show agent version """ - print("{0} running on {1} {2}".format(AGENT_LONG_VERSION, DISTRO_NAME, - DISTRO_VERSION)) + print(("{0} running on {1} {2}".format(AGENT_LONG_VERSION, DISTRO_NAME, + DISTRO_VERSION))) def usage(): """ Show agent usage """ print("") - print(("usage: {0} [-verbose] [-force] [-help]" + print((("usage: {0} [-verbose] [-force] [-help]" "-deprovision[+user]|-register-service|-version|-daemon|-start]" - "").format(sys.argv[0])) + "").format(sys.argv[0]))) print("") def start(): @@ -109,9 +109,9 @@ def register_service(): """ Register agent as a service """ - print "Register {0} service".format(AGENT_NAME) + print("Register {0} service".format(AGENT_NAME)) OSUTIL.register_agent_service() - print "Start {0} service".format(AGENT_NAME) + print("Start {0} service".format(AGENT_NAME)) OSUTIL.start_agent_service() def main(): diff --git a/azurelinuxagent/distro/coreos/handlerFactory.py b/azurelinuxagent/distro/coreos/handlerFactory.py index 72ed965..f0490e8 100644 --- a/azurelinuxagent/distro/coreos/handlerFactory.py +++ b/azurelinuxagent/distro/coreos/handlerFactory.py @@ -17,7 +17,7 @@ # Requires Python 2.4+ and Openssl 1.0+ # -from deprovision import CoreOSDeprovisionHandler +from .deprovision import CoreOSDeprovisionHandler from azurelinuxagent.distro.default.handlerFactory import DefaultHandlerFactory class CoreOSHandlerFactory(DefaultHandlerFactory): diff --git a/azurelinuxagent/distro/default/deprovision.py b/azurelinuxagent/distro/default/deprovision.py index c0fb184..231f4eb 100644 --- a/azurelinuxagent/distro/default/deprovision.py +++ b/azurelinuxagent/distro/default/deprovision.py @@ -104,10 +104,10 @@ class DeprovisionHandler(object): def deprovision(self, force=False, deluser=False): warnings, actions = self.setup(deluser) for warning in warnings: - print warning + print(warning) if not force: - confirm = raw_input("Do you want to proceed (y/n)") + confirm = input("Do you want to proceed (y/n)") if not confirm.lower().startswith('y'): return diff --git a/azurelinuxagent/distro/default/extension.py b/azurelinuxagent/distro/default/extension.py index 9d7ac87..ac8c84a 100644 --- a/azurelinuxagent/distro/default/extension.py +++ b/azurelinuxagent/distro/default/extension.py @@ -183,7 +183,7 @@ class ExtensionInstance(object): def init_logger(self): #Init logger appender for extension - fileutil.mkdir(self.get_log_dir(), mode=0700) + fileutil.mkdir(self.get_log_dir(), mode=0o700) log_file = os.path.join(self.get_log_dir(), "CommandExecution.log") self.logger.add_appender(logger.AppenderType.FILE, logger.LogLevel.INFO, log_file) @@ -282,7 +282,7 @@ class ExtensionInstance(object): self.logger.info("Unpack extension package") pkg_file = os.path.join(self.lib_dir, os.path.basename(uri.uri) + ".zip") - fileutil.write_file(pkg_file, bytearray(package)) + fileutil.write_file(pkg_file, bytearray(package), asbin=True) zipfile.ZipFile(pkg_file).extractall(self.get_base_dir()) chmod = "find {0} -type f | xargs chmod u+x".format(self.get_base_dir()) shellutil.run(chmod) @@ -299,9 +299,9 @@ class ExtensionInstance(object): #Create status and config dir status_dir = self.get_status_dir() - fileutil.mkdir(status_dir, mode=0700) + fileutil.mkdir(status_dir, mode=0o700) conf_dir = self.get_conf_dir() - fileutil.mkdir(conf_dir, mode=0700) + fileutil.mkdir(conf_dir, mode=0o700) #Init handler state to uninstall self.set_handler_status("NotReady") @@ -508,8 +508,7 @@ class ExtensionInstance(object): if major is None: raise ExtensionError("Wrong version format: {0}".format(version)) - packages = filter(lambda x : x.version.startswith(major + "."), - self.pkg_list.versions) + packages = [x for x in self.pkg_list.versions if x.version.startswith(major + ".")] packages = sorted(packages, key=lambda x: x.version, reverse=True) if len(packages) <= 0: raise ExtensionError("Can't find version: {0}.*".format(major)) diff --git a/azurelinuxagent/distro/default/handlerFactory.py b/azurelinuxagent/distro/default/handlerFactory.py index 429e734..98b2380 100644 --- a/azurelinuxagent/distro/default/handlerFactory.py +++ b/azurelinuxagent/distro/default/handlerFactory.py @@ -16,15 +16,15 @@ # # Requires Python 2.4+ and Openssl 1.0+ # -from init import InitHandler -from run import MainHandler -from scvmm import ScvmmHandler -from dhcp import DhcpHandler -from env import EnvHandler -from provision import ProvisionHandler -from resourceDisk import ResourceDiskHandler -from extension import ExtensionsHandler -from deprovision import DeprovisionHandler +from .init import InitHandler +from .run import MainHandler +from .scvmm import ScvmmHandler +from .dhcp import DhcpHandler +from .env import EnvHandler +from .provision import ProvisionHandler +from .resourceDisk import ResourceDiskHandler +from .extension import ExtensionsHandler +from .deprovision import DeprovisionHandler class DefaultHandlerFactory(object): def __init__(self): diff --git a/azurelinuxagent/distro/default/init.py b/azurelinuxagent/distro/default/init.py index 9255597..337fdea 100644 --- a/azurelinuxagent/distro/default/init.py +++ b/azurelinuxagent/distro/default/init.py @@ -43,7 +43,7 @@ class InitHandler(object): path="/dev/console") #Create lib dir - fileutil.mkdir(OSUTIL.get_lib_dir(), mode=0700) + fileutil.mkdir(OSUTIL.get_lib_dir(), mode=0o700) os.chdir(OSUTIL.get_lib_dir()) diff --git a/azurelinuxagent/distro/default/osutil.py b/azurelinuxagent/distro/default/osutil.py index 4c3f187..194f213 100644 --- a/azurelinuxagent/distro/default/osutil.py +++ b/azurelinuxagent/distro/default/osutil.py @@ -126,7 +126,7 @@ class DefaultOSUtil(object): try: passwd_content = fileutil.read_file(self.passwd_file_path) passwd = passwd_content.split("\n") - new_passwd = filter(lambda x : not x.startswith(username), passwd) + new_passwd = [x for x in passwd if not x.startswith(username)] new_passwd.append("{0}:{1}:14600::::::".format(username, passwd_hash)) fileutil.write_file(self.passwd_file_path, "\n".join(new_passwd)) except IOError as e: @@ -147,13 +147,13 @@ class DefaultOSUtil(object): else: sudoer = "{0} ALL = (ALL) ALL\n".format(username) fileutil.append_file('/etc/sudoers.d/waagent', sudoer) - fileutil.chmod('/etc/sudoers.d/waagent', 0440) + fileutil.chmod('/etc/sudoers.d/waagent', 0o440) def del_root_password(self): try: passwd_content = fileutil.read_file(self.passwd_file_path) passwd = passwd_content.split('\n') - new_passwd = filter(lambda x : not x.startswith("root:"), passwd) + new_passwd = [x for x in passwd if not x.startswith("root:")] new_passwd.insert(0, "root:*LOCK*:14600::::::") fileutil.write_file(self.passwd_file_path, "\n".join(new_passwd)) except IOError as e: @@ -194,7 +194,7 @@ class DefaultOSUtil(object): path, thumbprint = keypair path = self._norm_path(path) dir_path = os.path.dirname(path) - fileutil.mkdir(dir_path, mode=0700, owner=username) + fileutil.mkdir(dir_path, mode=0o700, owner=username) lib_dir = self.get_lib_dir() prv_path = os.path.join(lib_dir, thumbprint + '.prv') if not os.path.isfile(prv_path): @@ -205,8 +205,8 @@ class DefaultOSUtil(object): fileutil.write_file(pub_path, pub) self.set_selinux_context(pub_path, 'unconfined_u:object_r:ssh_home_t:s0') self.set_selinux_context(path, 'unconfined_u:object_r:ssh_home_t:s0') - os.chmod(path, 0644) - os.chmod(pub_path, 0600) + os.chmod(path, 0o644) + os.chmod(pub_path, 0o600) def openssl_to_openssh(self, input_file, output_file): shellutil.run("ssh-keygen -i -m PKCS8 -f {0} >> {1}".format(input_file, @@ -222,7 +222,7 @@ class DefaultOSUtil(object): path = self._norm_path(path) dir_path = os.path.dirname(path) - fileutil.mkdir(dir_path, mode=0700, owner=username) + fileutil.mkdir(dir_path, mode=0o700, owner=username) if value is not None: if not value.startswith("ssh-"): raise OSUtilError("Bad public key: {0}".format(value)) @@ -238,13 +238,13 @@ class DefaultOSUtil(object): self.set_selinux_context(pub_path, 'unconfined_u:object_r:ssh_home_t:s0') self.openssl_to_openssh(pub_path, path) - fileutil.chmod(pub_path, 0600) + fileutil.chmod(pub_path, 0o600) else: raise OSUtilError("SSH public key Fingerprint and Value are None") self.set_selinux_context(path, 'unconfined_u:object_r:ssh_home_t:s0') fileutil.chowner(path, username) - fileutil.chmod(path, 0644) + fileutil.chmod(path, 0o644) def is_selinux_system(self): """ @@ -292,7 +292,7 @@ class DefaultOSUtil(object): conf_file_path = self.get_sshd_conf_file_path() conf = fileutil.read_file(conf_file_path).split("\n") textutil.set_ssh_config(conf, "ClientAliveInterval", "180") - fileutil.replace_file(conf_file_path, '\n'.join(conf)) + fileutil.write_file(conf_file_path, '\n'.join(conf)) logger.info("Configured SSH client probing to keep connections alive.") def conf_sshd(self, disable_password): @@ -301,7 +301,7 @@ class DefaultOSUtil(object): conf = fileutil.read_file(conf_file_path).split("\n") textutil.set_ssh_config(conf, "PasswordAuthentication", option) textutil.set_ssh_config(conf, "ChallengeResponseAuthentication", option) - fileutil.replace_file(conf_file_path, "\n".join(conf)) + fileutil.write_file(conf_file_path, "\n".join(conf)) logger.info("Disabled SSH password-based authentication methods.") @@ -469,7 +469,8 @@ class DefaultOSUtil(object): def is_missing_default_route(self): routes = shellutil.run_get_output("route -n")[1] - for route in routes: + for route in routes.split("\n"): + print(route) if route.startswith("0.0.0.0 ") or route.startswith("default "): return False return True @@ -625,7 +626,7 @@ class DefaultOSUtil(object): try: content = fileutil.read_file("/etc/sudoers.d/waagent") sudoers = content.split("\n") - sudoers = filter(lambda x : username not in x, sudoers) + sudoers = [x for x in sudoers if username not in x] fileutil.write_file("/etc/sudoers.d/waagent", "\n".join(sudoers)) except IOError as e: diff --git a/azurelinuxagent/distro/default/resourceDisk.py b/azurelinuxagent/distro/default/resourceDisk.py index f9b7777..6adf31b 100644 --- a/azurelinuxagent/distro/default/resourceDisk.py +++ b/azurelinuxagent/distro/default/resourceDisk.py @@ -91,7 +91,7 @@ class ResourceDiskHandler(object): logger.info("Resource disk {0}1 is already mounted", device) return existing - fileutil.mkdir(mount_point, mode=0755) + fileutil.mkdir(mount_point, mode=0o755) logger.info("Detect GPT...") partition = device + "1" @@ -102,8 +102,7 @@ class ResourceDiskHandler(object): if "gpt" in ret[1]: logger.info("GPT detected") logger.info("Get GPT partitions") - parts = filter(lambda x : re.match("^\s*[0-9]+", x), - ret[1].split("\n")) + parts = [x for x in ret[1].split("\n") if re.match("^\s*[0-9]+", x)] logger.info("Found more than {0} GPT partitions.", len(parts)) if len(parts) > 1: logger.info("Remove old GPT partitions") diff --git a/azurelinuxagent/distro/redhat/osutil.py b/azurelinuxagent/distro/redhat/osutil.py index b7340be..77a913b 100644 --- a/azurelinuxagent/distro/redhat/osutil.py +++ b/azurelinuxagent/distro/redhat/osutil.py @@ -60,30 +60,59 @@ class Redhat6xOSUtil(DefaultOSUtil): def asn1_to_ssh_rsa(self, pubkey): lines = pubkey.split("\n") - lines = filter(lambda x : not x.startswith("----"), lines) + lines = [x for x in lines if not x.startswith("----")] base64_encoded = "".join(lines) try: #TODO remove pyasn1 dependency from pyasn1.codec.der import decoder as der_decoder der_encoded = base64.b64decode(base64_encoded) der_encoded = der_decoder.decode(der_encoded)[0][1] - k = der_decoder.decode(textutil.bits_to_str(der_encoded))[0] - n=k[0] - e=k[1] - keydata="" - keydata += struct.pack('>I',len("ssh-rsa")) - keydata += "ssh-rsa" - keydata += struct.pack('>I',len(textutil.num_to_bytes(e))) - keydata += textutil.num_to_bytes(e) - keydata += struct.pack('>I',len(textutil.num_to_bytes(n)) + 1) - keydata += "\0" - keydata += textutil.num_to_bytes(n) - return "ssh-rsa " + base64.b64encode(keydata) + "\n" + key = der_decoder.decode(self.bits_to_bytes(der_encoded))[0] + n=key[0] + e=key[1] + print(n) + print(e) + keydata = bytearray() + keydata.extend(struct.pack('>I', len("ssh-rsa"))) + keydata.extend(b"ssh-rsa") + keydata.extend(struct.pack('>I', len(self.num_to_bytes(e)))) + keydata.extend(self.num_to_bytes(e)) + keydata.extend(struct.pack('>I', len(self.num_to_bytes(n)) + 1)) + keydata.extend(b"\0") + keydata.extend(self.num_to_bytes(n)) + return str(b"ssh-rsa " + base64.b64encode(keydata) + b"\n", + encoding='utf-8') except ImportError as e: raise OSUtilError("Failed to load pyasn1.codec.der") - except Exception as e: - raise OSUtilError(("Failed to convert public key: {0} {1}" - "").format(type(e).__name__, e)) + #except Exception as e: + #raise OSUtilError(("Failed to convert public key: {0} {1}" + #"").format(type(e).__name__, e)) + def num_to_bytes(self, num): + """ + Pack number into bytes. Retun as string. + """ + result = bytearray() + while num: + result.append(num & 0xFF) + num >>= 8 + result.reverse() + return result + + def bits_to_bytes(self, bits): + """ + Convert an array contains bits, [0,1] to a byte array + """ + index = 7 + byte_array = bytearray() + curr = 0 + for bit in bits: + curr = curr | (bit << index) + index = index - 1 + if index == -1: + byte_array.append(curr) + curr = 0 + index = 7 + return bytes(byte_array) def openssl_to_openssh(self, input_file, output_file): pubkey = fileutil.read_file(input_file) diff --git a/azurelinuxagent/event.py b/azurelinuxagent/event.py index c5fed91..4b54893 100644 --- a/azurelinuxagent/event.py +++ b/azurelinuxagent/event.py @@ -138,7 +138,7 @@ def save_event(data): event_dir = os.path.join(OSUTIL.get_lib_dir(), 'events') if not os.path.exists(event_dir): os.mkdir(event_dir) - os.chmod(event_dir,0700) + os.chmod(event_dir,0o700) if len(os.listdir(event_dir)) > 1000: raise EventError("Too many files under: {0}", event_dir) diff --git a/azurelinuxagent/logger.py b/azurelinuxagent/logger.py index d2e416d..3225140 100644 --- a/azurelinuxagent/logger.py +++ b/azurelinuxagent/logger.py @@ -48,13 +48,11 @@ class Logger(object): self.log(LogLevel.ERROR, msg_format, *args) def log(self, level, msg_format, *args): - msg_format = textutil.ascii(msg_format) - args = map(lambda x: textutil.ascii(x), args) if len(args) > 0: msg = msg_format.format(*args) else: msg = msg_format - time = datetime.now().strftime('%Y/%m/%d %H:%M:%S.%f') + time = datetime.now().strftime('%Y/%m/%d %H:%M:%S.%f\n') level_str = LogLevel.STRINGS[level] if self.prefix is not None: log_item = "{0} {1} {2} {3}".format(time, level_str, self.prefix, @@ -79,7 +77,7 @@ class ConsoleAppender(object): if self.level <= level: try: with open(self.path, "w") as console: - console.write(msg.encode('ascii', 'ignore') + "\n") + console.write(msg) except IOError: pass @@ -92,7 +90,7 @@ class FileAppender(object): if self.level <= level: try: with open(self.path, "a+") as log_file: - log_file.write(msg.encode('ascii', 'ignore') + "\n") + log_file.write(msg) except IOError: pass @@ -103,7 +101,7 @@ class StdoutAppender(object): def write(self, level, msg): if self.level <= level: try: - sys.stdout.write(msg.encode('ascii', 'ignore') + "\n") + sys.stdout.write(msg) except IOError: pass diff --git a/azurelinuxagent/protocol/common.py b/azurelinuxagent/protocol/common.py index aff4599..77247ab 100644 --- a/azurelinuxagent/protocol/common.py +++ b/azurelinuxagent/protocol/common.py @@ -42,7 +42,7 @@ def set_properties(obj, data): validata_param("data", data, dict) props = vars(obj) - for name, val in props.items(): + for name, val in list(props.items()): try: new_val = data[name] except KeyError: @@ -64,7 +64,7 @@ def get_properties(obj): data = {} props = vars(obj) - for name, val in props.items(): + for name, val in list(props.items()): if isinstance(val, DataContract): data[name] = get_properties(val) elif isinstance(val, DataContractList): diff --git a/azurelinuxagent/protocol/protocolFactory.py b/azurelinuxagent/protocol/protocolFactory.py index 7429974..a1efab9 100644 --- a/azurelinuxagent/protocol/protocolFactory.py +++ b/azurelinuxagent/protocol/protocolFactory.py @@ -46,11 +46,7 @@ def detect_wire_protocol(): OSUTIL.gen_transport_cert() protocol = WireProtocol(endpoint) protocol.initialize() - logger.info("Protocol V1 found.") - path = os.path.join(OSUTIL.get_lib_dir(), WireProtocol) - - fileutil.write_file(path, "") return protocol def detect_metadata_protocol(): diff --git a/azurelinuxagent/protocol/v1.py b/azurelinuxagent/protocol/v1.py index 6a3f02e..18e99d0 100644 --- a/azurelinuxagent/protocol/v1.py +++ b/azurelinuxagent/protocol/v1.py @@ -21,7 +21,7 @@ import json import re import time import traceback -import httplib +import http.client import xml.sax.saxutils as saxutils import xml.etree.ElementTree as ET import azurelinuxagent.logger as logger @@ -118,24 +118,24 @@ def _fetch_uri(uri, headers, chk_proxy=False): except restutil.HttpError as e: raise ProtocolError(str(e)) - if(resp.status == httplib.GONE): + if(resp.status == http.client.GONE): raise WireProtocolResourceGone(uri) - if(resp.status != httplib.OK): + if(resp.status != http.client.OK): raise ProtocolError("{0} - {1}".format(resp.status, uri)) - return resp.read() + return str(resp.read(), encoding='utf-8') def _fetch_manifest(version_uris): for version_uri in version_uris: try: xml_text = _fetch_uri(version_uri.uri, None, chk_proxy=True) return xml_text - except IOError, e: + except IOError as e: logger.warn("Failed to fetch ExtensionManifest: {0}, {1}", e, version_uri.uri) raise ProtocolError("Failed to fetch ExtensionManifest from all sources") def _build_role_properties(container_id, role_instance_id, thumbprint): - xml = (u"" + xml = ("" "" "" "{0}" @@ -160,7 +160,7 @@ def _build_health_report(incarnation, container_id, role_instance_id, "{0}" "{1}" "").format(substatus, description) - xml = (u"" + xml = ("" "" @@ -302,7 +302,7 @@ class StatusBlob(object): "x-ms-date" : timestamp, 'x-ms-version' : self.__class__.__storage_version__ }) - if resp is None or resp.status != httplib.OK: + if resp is None or resp.status != http.client.OK: raise ProtocolError(("Failed to get status blob type: {0}" "").format(resp.status)) @@ -319,7 +319,7 @@ class StatusBlob(object): "Content-Length": str(len(data)), "x-ms-version" : self.__class__.__storage_version__ }) - if resp is None or resp.status != httplib.CREATED: + if resp is None or resp.status != http.client.CREATED: raise ProtocolError(("Failed to upload block blob: {0}" "").format(resp.status)) @@ -335,7 +335,7 @@ class StatusBlob(object): "x-ms-blob-content-length" : str(page_blob_size), "x-ms-version" : self.__class__.__storage_version__ }) - if resp is None or resp.status != httplib.CREATED: + if resp is None or resp.status != http.client.CREATED: raise ProtocolError(("Failed to clean up page blob: {0}" "").format(resp.status)) @@ -352,36 +352,36 @@ class StatusBlob(object): end = min(len(data), start + page_max) content_size = end - start #Align to 512 bytes - page_end = ((end + 511) / 512) * 512 + page_end = int((end + 511) / 512) * 512 buf_size = page_end - start - buf = bytearray(buf_size) - buf[0 : content_size] = data[start : end] - resp = restutil.http_put(url, buffer(buf), { + buf = bytearray(source=data[start:end], encoding="utf-8") + #TODO buffer is not defined in python3, however we need this to make httplib to work on python 2.6 + resp = restutil.http_put(url, buf, { "x-ms-date" : timestamp, "x-ms-range" : "bytes={0}-{1}".format(start, page_end - 1), "x-ms-page-write" : "update", "x-ms-version" : self.__class__.__storage_version__, "Content-Length": str(page_end - start) }) - if resp is None or resp.status != httplib.CREATED: + if resp is None or resp.status != http.client.CREATED: raise ProtocolError(("Failed to upload page blob: {0}" "").format(resp.status)) start = end def event_param_to_v1(param): - param_format = u'' + param_format = '' param_type = type(param.value) attr_type = "" if param_type is int: - attr_type = u'mt:uint64' + attr_type = 'mt:uint64' elif param_type is str: - attr_type = u'mt:wstr' + attr_type = 'mt:wstr' elif str(param_type).count("'unicode'") > 0: - attr_type = u'mt:wstr' + attr_type = 'mt:wstr' elif param_type is bool: - attr_type = u'mt:bool' + attr_type = 'mt:bool' elif param_type is float: - attr_type = u'mt:float64' + attr_type = 'mt:float64' return param_format.format(param.name, saxutils.quoteattr(str(param.value)), attr_type) @@ -389,9 +389,9 @@ def event_to_v1(event): params = "" for param in event.parameters: params += event_param_to_v1(param) - event_str = (u'' - u'' - u'').format(event.eventId, params) + event_str = ('' + '' + '').format(event.eventId, params) return event_str class WireClient(object): @@ -568,11 +568,11 @@ class WireClient(object): def send_event(self, provider_id, event_str): uri = TELEMETRY_URI.format(self.endpoint) - data_format = (u'' - u'' - u'{1}' - u'' - u'') + data_format = ('' + '' + '{1}' + '' + '') data = data_format.format(provider_id, event_str) try: self.prevent_throttling() @@ -581,7 +581,7 @@ class WireClient(object): except restutil.HttpError as e: raise ProtocolError("Failed to send events:{0}".format(e)) - if resp.status != httplib.OK: + if resp.status != http.client.OK: logger.verb(resp.read()) raise ProtocolError("Failed to send events:{0}".format(resp.status)) @@ -601,7 +601,7 @@ class WireClient(object): buf[event.providerId] = buf[event.providerId] + event_str #Send out all events left in buffer. - for provider_id in buf.keys(): + for provider_id in list(buf.keys()): if len(buf[provider_id]) > 0: self.send_event(provider_id, buf[provider_id]) @@ -896,9 +896,9 @@ class ExtensionsConfig(object): name = ext.name version = ext.properties.version - settings = filter(lambda x: getattrib(x, "name") == name and \ - getattrib(x ,"version") == version, - plugin_settings) + settings = [x for x in plugin_settings \ + if getattrib(x, "name") == name and \ + getattrib(x ,"version") == version] if settings is None or len(settings) == 0: return @@ -938,7 +938,7 @@ class ExtensionManifest(object): version = findtext(package, "Version") uris = find(package, "Uris") uri_list = findall(uris, "Uri") - uri_list = map(lambda x : gettext(x), uri_list) + uri_list = [gettext(x) for x in uri_list] package = ExtensionPackage() package.version = version for uri in uri_list: diff --git a/azurelinuxagent/protocol/v2.py b/azurelinuxagent/protocol/v2.py index 4790b8e..b5feda4 100644 --- a/azurelinuxagent/protocol/v2.py +++ b/azurelinuxagent/protocol/v2.py @@ -16,7 +16,7 @@ # # Requires Python 2.4+ and Openssl 1.0+ -import httplib +import http.client import json import azurelinuxagent.utils.restutil as restutil from azurelinuxagent.protocol.common import * @@ -57,7 +57,7 @@ class MetadataProtocol(Protocol): except restutil.HttpError as e: raise ProtocolError(str(e)) - if resp.status != httplib.OK: + if resp.status != http.client.OK: raise ProtocolError("{0} - GET: {1}".format(resp.status, url)) try: data = json.loads(resp.read()) @@ -74,7 +74,7 @@ class MetadataProtocol(Protocol): resp = restutil.http_put(url, json.dumps(data), headers=headers) except restutil.HttpError as e: raise ProtocolError(str(e)) - if resp.status != httplib.OK: + if resp.status != http.client.OK: raise ProtocolError("{0} - PUT: {1}".format(resp.status, url)) def _post_data(self, url, obj, headers=None): @@ -84,7 +84,7 @@ class MetadataProtocol(Protocol): resp = restutil.http_post(url, json.dumps(data), headers=headers) except restutil.HttpError as e: raise ProtocolError(str(e)) - if resp.status != httplib.CREATED: + if resp.status != http.client.CREATED: raise ProtocolError("{0} - POST: {1}".format(resp.status, url)) def initialize(self): diff --git a/azurelinuxagent/utils/fileutil.py b/azurelinuxagent/utils/fileutil.py index 22d28e5..2f4db45 100644 --- a/azurelinuxagent/utils/fileutil.py +++ b/azurelinuxagent/utils/fileutil.py @@ -42,18 +42,30 @@ def read_file(filepath, asbin=False, remove_bom=False): contents = textutil.remove_bom(contents) return contents -def write_file(filepath, contents): +def write_file(filepath, contents, asbin=False): """ Write 'contents' to 'filepath'. """ - with open(filepath, "wb") as out_file: + if asbin: + mode = 'wb' + else: + mode = 'w' + if type(contents) != str: + contents = str(contents) + with open(filepath, mode) as out_file: out_file.write(contents) -def append_file(filepath, contents): +def append_file(filepath, contents, asbin=False): """ Append 'contents' to 'filepath'. """ - with open(filepath, "a+") as out_file: + if asbin: + mode = 'ab' + else: + mode = 'a' + if type(contents) != str: + contents = str(contents) + with open(filepath, mode) as out_file: out_file.write(contents) def replace_file(filepath, contents): @@ -84,7 +96,7 @@ def replace_file(filepath, contents): try: os.rename(temp, filepath) - except IOError, err: + except IOError as err: logger.error('Rename {0} to {1}, Exception is {2}', temp, filepath, err) return 1 @@ -143,7 +155,7 @@ def update_conf_file(path, line_start, val, chk_err=False): if not os.path.isfile(path) and chk_err: raise Exception("Can't find config file:{0}".format(path)) conf = read_file(path).split('\n') - conf = filter(lambda x: not x.startswith(line_start), conf) + conf = [x for x in conf if not x.startswith(line_start)] conf.append(val) replace_file(path, '\n'.join(conf)) diff --git a/azurelinuxagent/utils/restutil.py b/azurelinuxagent/utils/restutil.py index 4048a19..963e208 100644 --- a/azurelinuxagent/utils/restutil.py +++ b/azurelinuxagent/utils/restutil.py @@ -22,9 +22,9 @@ import os import subprocess import azurelinuxagent.logger as logger import azurelinuxagent.conf as conf -import httplib +import http.client import time -from urlparse import urlparse +from urllib.parse import urlparse """ REST api util functions @@ -61,21 +61,21 @@ def _http_request(method, host, rel_uri, port=None, data=None, secure=False, if secure: port = 443 if port is None else port if proxy_host is not None and proxy_port is not None: - conn = httplib.HTTPSConnection(proxy_host, proxy_port) + conn = http.client.HTTPSConnection(proxy_host, proxy_port) conn.set_tunnel(host, port) #If proxy is used, full url is needed. url = "https://{0}:{1}{2}".format(host, port, rel_uri) else: - conn = httplib.HTTPSConnection(host, port) + conn = http.client.HTTPSConnection(host, port) url = rel_uri else: port = 80 if port is None else port if proxy_host is not None and proxy_port is not None: - conn = httplib.HTTPConnection(proxy_host, proxy_port) + conn = http.client.HTTPConnection(proxy_host, proxy_port) #If proxy is used, full url is needed. url = "http://{0}:{1}{2}".format(host, port, rel_uri) else: - conn = httplib.HTTPConnection(host, port) + conn = http.client.HTTPConnection(host, port) url = rel_uri if headers == None: conn.request(method, url, data) @@ -100,7 +100,7 @@ def http_request(method, url, data, headers=None, max_retry=3, chk_proxy=False): proxy_host, proxy_port = get_http_proxy() #If httplib module is not built with ssl support. Fallback to http - if secure and not hasattr(httplib, "HTTPSConnection"): + if secure and not hasattr(http.client, "HTTPSConnection"): logger.warn("httplib is not built with ssl support") secure = False @@ -108,7 +108,7 @@ def http_request(method, url, data, headers=None, max_retry=3, chk_proxy=False): if secure and \ proxy_host is not None and \ proxy_port is not None and \ - not hasattr(httplib.HTTPSConnection, "set_tunnel"): + not hasattr(http.client.HTTPSConnection, "set_tunnel"): logger.warn("httplib doesn't support https tunnelling(new in python 2.7)") secure = False @@ -119,7 +119,7 @@ def http_request(method, url, data, headers=None, max_retry=3, chk_proxy=False): logger.verb("HTTP Resp: Status={0}", resp.status) logger.verb(" Header={0}", resp.getheaders()) return resp - except httplib.HTTPException as e: + except http.client.HTTPException as e: logger.warn('HTTPException {0}, args:{1}', e, repr(e.args)) except IOError as e: logger.warn('Socket IOError {0}, args:{1}', e, repr(e.args)) diff --git a/azurelinuxagent/utils/shellutil.py b/azurelinuxagent/utils/shellutil.py index 5e98b6e..342b123 100644 --- a/azurelinuxagent/utils/shellutil.py +++ b/azurelinuxagent/utils/shellutil.py @@ -73,12 +73,12 @@ def run_get_output(cmd, chk_err=True): logger.verb("run cmd '{0}'", cmd) try: output=subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) - except subprocess.CalledProcessError,e : + except subprocess.CalledProcessError as e : if chk_err : logger.error("run cmd '{0}' failed", e.cmd) logger.error("Error Code:{0}", e.returncode) logger.error("Result:{0}", e.output[:-1].decode('latin-1')) return e.returncode, e.output.decode('latin-1') - return 0, output + return 0, str(output, encoding="utf-8") #End shell command util functions diff --git a/azurelinuxagent/utils/textutil.py b/azurelinuxagent/utils/textutil.py index 6736808..66da372 100644 --- a/azurelinuxagent/utils/textutil.py +++ b/azurelinuxagent/utils/textutil.py @@ -188,9 +188,10 @@ def int_to_ip4_addr(a): def ascii(val): uni = None if type(val) == str: - uni = unicode(val, 'utf-8', errors='ignore') + pass + #uni = str(val, 'utf-8', errors='ignore') else: - uni = unicode(val) + uni = str(val) if uni is None: raise ValueError('') else: @@ -224,37 +225,11 @@ def remove_bom(c): return c def gen_password_hash(password, use_salt, salt_type, salt_len): - salt="$6$" - if use_salt: - collection = string.ascii_letters + string.digits - salt = ''.join(random.choice(collection) for _ in range(salt_len)) - salt = "${0}${1}".format(salt_type, salt) - return crypt.crypt(password, salt) + salt="$6$" + if use_salt: + collection = string.ascii_letters + string.digits + salt = ''.join(random.choice(collection) for _ in range(salt_len)) + salt = "${0}${1}".format(salt_type, salt) + return crypt.crypt(password, salt) -def num_to_bytes(i): - """ - Pack number into bytes. Retun as string. - """ - result = [] - while i: - result.append(chr(i & 0xFF)) - i >>= 8 - result.reverse() - return ''.join(result) - -def bits_to_str(a): - """ - Return string representation of bits in a. - """ - index=7 - s = "" - c = 0 - for bit in a: - c = c | (bit << index) - index = index - 1 - if index == -1: - s = s + struct.pack('>B', c) - c = 0 - index = 7 - return s diff --git a/bin/waagent b/bin/waagent index 7cdcce7..211a2bb 100755 --- a/bin/waagent +++ b/bin/waagent @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Windows Azure Linux Agent # @@ -29,6034 +29,3 @@ if __name__ == '__main__' : agent.main() sys.exit() -import array -import base64 -import httplib -import os -import os.path -import platform -import pwd -import re -import shutil -import socket -import SocketServer -import struct -import string -import subprocess -import sys -import tempfile -import textwrap -import threading -import time -import traceback -import xml.dom.minidom -import fcntl -import inspect -import zipfile -import json -import datetime -import xml.sax.saxutils - - -if not hasattr(subprocess,'check_output'): - def check_output(*popenargs, **kwargs): - r"""Backport from subprocess module from python 2.7""" - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd, output=output) - return output - - # Exception classes used by this module. - class CalledProcessError(Exception): - def __init__(self, returncode, cmd, output=None): - self.returncode = returncode - self.cmd = cmd - self.output = output - def __str__(self): - return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) - - subprocess.check_output=check_output - subprocess.CalledProcessError=CalledProcessError - -GuestAgentName = "WALinuxAgent" -GuestAgentLongName = "Windows Azure Linux Agent" -GuestAgentVersion = "WALinuxAgent-2.0.14" -ProtocolVersion = "2012-11-30" #WARNING this value is used to confirm the correct fabric protocol. - -Config = None -WaAgent = None -DiskActivated = False -Openssl = "openssl" -Children = [] -ExtensionChildren = [] -VMM_STARTUP_SCRIPT_NAME='install' -VMM_CONFIG_FILE_NAME='linuxosconfiguration.xml' -global RulesFiles -RulesFiles = [ "/lib/udev/rules.d/75-persistent-net-generator.rules", - "/etc/udev/rules.d/70-persistent-net.rules" ] -VarLibDhcpDirectories = ["/var/lib/dhclient", "/var/lib/dhcpcd", "/var/lib/dhcp"] -EtcDhcpClientConfFiles = ["/etc/dhcp/dhclient.conf", "/etc/dhcp3/dhclient.conf"] -global LibDir -LibDir = "/var/lib/waagent" -global provisioned -provisioned=False -global provisionError -provisionError=None -HandlerStatusToAggStatus = {"installed":"Installing", "enabled":"Ready", "unintalled":"NotReady", "disabled":"NotReady"} - -WaagentConf = """\ -# -# Windows Azure Linux Agent Configuration -# - -Role.StateConsumer=None # Specified program is invoked with the argument "Ready" when we report ready status - # to the endpoint server. -Role.ConfigurationConsumer=None # Specified program is invoked with XML file argument specifying role configuration. -Role.TopologyConsumer=None # Specified program is invoked with XML file argument specifying role topology. - -Provisioning.Enabled=y # -Provisioning.DeleteRootPassword=y # Password authentication for root account will be unavailable. -Provisioning.RegenerateSshHostKeyPair=y # Generate fresh host key pair. -Provisioning.SshHostKeyPairType=rsa # Supported values are "rsa", "dsa" and "ecdsa". -Provisioning.MonitorHostName=y # Monitor host name changes and publish changes via DHCP requests. - -ResourceDisk.Format=y # Format if unformatted. If 'n', resource disk will not be mounted. -ResourceDisk.Filesystem=ext4 # Typically ext3 or ext4. FreeBSD images should use 'ufs2' here. -ResourceDisk.MountPoint=/mnt/resource # -ResourceDisk.EnableSwap=n # Create and use swapfile on resource disk. -ResourceDisk.SwapSizeMB=0 # Size of the swapfile. - -LBProbeResponder=y # Respond to load balancer probes if requested by Windows Azure. - -Logs.Verbose=n # Enable verbose logs - -OS.RootDeviceScsiTimeout=300 # Root device timeout in seconds. -OS.OpensslPath=None # If "None", the system default version is used. -""" -README_FILENAME="DATALOSS_WARNING_README.txt" -README_FILECONTENT="""\ -WARNING: THIS IS A TEMPORARY DISK. - -Any data stored on this drive is SUBJECT TO LOSS and THERE IS NO WAY TO RECOVER IT. - -Please do not use this disk for storing any personal or application data. - -For additional details to please refer to the MSDN documentation at : http://msdn.microsoft.com/en-us/library/windowsazure/jj672979.aspx -""" - -############################################################ -# BEGIN DISTRO CLASS DEFS -############################################################ -############################################################ -# AbstractDistro -############################################################ -class AbstractDistro(object): - """ - AbstractDistro defines a skeleton neccesary for a concrete Distro class. - - Generic methods and attributes are kept here, distribution specific attributes - and behavior are to be placed in the concrete child named distroDistro, where - distro is the string returned by calling python platform.linux_distribution()[0]. - So for CentOS the derived class is called 'centosDistro'. - """ - - def __init__(self): - """ - Generic Attributes go here. These are based on 'majority rules'. - This __init__() may be called or overriden by the child. - """ - self.agent_service_name = os.path.basename(sys.argv[0]) - self.selinux=None - self.service_cmd='/usr/sbin/service' - self.ssh_service_restart_option='restart' - self.ssh_service_name='ssh' - self.ssh_config_file='/etc/ssh/sshd_config' - self.hostname_file_path='/etc/hostname' - self.dhcp_client_name='dhclient' - self.requiredDeps = [ 'route', 'shutdown', 'ssh-keygen', 'useradd', - 'openssl', 'sfdisk', 'fdisk', 'mkfs', 'chpasswd', - 'sed', 'grep', 'sudo', 'parted' ] - self.init_script_file='/etc/init.d/waagent' - self.agent_package_name='WALinuxAgent' - self.fileBlackList = [ "/root/.bash_history", "/var/log/waagent.log",'/etc/resolv.conf' ] - self.agent_files_to_uninstall = ["/etc/waagent.conf", "/etc/logrotate.d/waagent"] - self.grubKernelBootOptionsFile = '/etc/default/grub' - self.grubKernelBootOptionsLine = 'GRUB_CMDLINE_LINUX_DEFAULT=' - self.getpidcmd = 'pidof' - self.mount_dvd_cmd = 'mount' - self.sudoers_dir_base = '/etc' - self.waagent_conf_file = WaagentConf - self.shadow_file_mode=0600 - self.dhcp_enabled = False - - def isSelinuxSystem(self): - """ - Checks and sets self.selinux = True if SELinux is available on system. - """ - if self.selinux == None: - if Run("which getenforce",chk_err=False): - self.selinux = False - else: - self.selinux = True - return self.selinux - - def isSelinuxRunning(self): - """ - Calls shell command 'getenforce' and returns True if 'Enforcing'. - """ - if self.isSelinuxSystem(): - return RunGetOutput("getenforce")[1].startswith("Enforcing") - else: - return False - - def setSelinuxEnforce(self,state): - """ - Calls shell command 'setenforce' with 'state' and returns resulting exit code. - """ - if self.isSelinuxSystem(): - if state: s = '1' - else: s='0' - return Run("setenforce "+s) - - def setSelinuxContext(self,path,cn): - """ - Calls shell 'chcon' with 'path' and 'cn' context. - Returns exit result. - """ - if self.isSelinuxSystem(): - return Run('chcon ' + cn + ' ' + path) - - def setHostname(self,name): - """ - Shell call to hostname. - Returns resulting exit code. - """ - return Run('hostname ' + name) - - def publishHostname(self,name): - """ - Set the contents of the hostname file to 'name'. - Return 1 on failure. - """ - try: - r=SetFileContents(self.hostname_file_path, name) - for f in EtcDhcpClientConfFiles: - if os.path.exists(f) and FindStringInFile(f,r'^[^#]*?send\s*host-name.*?(|gethostname[(,)])') == None : - r=ReplaceFileContentsAtomic('/etc/dhcp/dhclient.conf', "send host-name \"" + name + "\";\n" - + "\n".join(filter(lambda a: not a.startswith("send host-name"), GetFileContents('/etc/dhcp/dhclient.conf').split('\n')))) - except: - return 1 - return r - - def installAgentServiceScriptFiles(self): - """ - Create the waagent support files for service installation. - Called by registerAgentService() - Abstract Virtual Function. Over-ridden in concrete Distro classes. - """ - pass - - def registerAgentService(self): - """ - Calls installAgentService to create service files. - Shell exec service registration commands. (e.g. chkconfig --add waagent) - Abstract Virtual Function. Over-ridden in concrete Distro classes. - """ - pass - - def uninstallAgentService(self): - """ - Call service subsystem to remove waagent script. - Abstract Virtual Function. Over-ridden in concrete Distro classes. - """ - pass - - def unregisterAgentService(self): - """ - Calls self.stopAgentService and call self.uninstallAgentService() - """ - self.stopAgentService() - self.uninstallAgentService() - - def startAgentService(self): - """ - Service call to start the Agent service - """ - return Run(self.service_cmd + ' ' + self.agent_service_name + ' start') - - def stopAgentService(self): - """ - Service call to stop the Agent service - """ - return Run(self.service_cmd + ' ' + self.agent_service_name + ' stop',False) - - def restartSshService(self): - """ - Service call to re(start) the SSH service - """ - sshRestartCmd = self.service_cmd + " " + self.ssh_service_name + " " + self.ssh_service_restart_option - retcode = Run(sshRestartCmd) - if retcode > 0: - Error("Failed to restart SSH service with return code:" + str(retcode)) - return retcode - - def sshDeployPublicKey(self,fprint,path): - """ - Generic sshDeployPublicKey - over-ridden in some concrete Distro classes due to minor differences in openssl packages deployed - """ - error=0 - SshPubKey = OvfEnv().OpensslToSsh(fprint) - if SshPubKey != None: - AppendFileContents(path, SshPubKey) - else: - Error("Failed: " + fprint + ".crt -> " + path) - error = 1 - return error - - def checkPackageInstalled(self,p): - """ - Query package database for prescence of an installed package. - Abstract Virtual Function. Over-ridden in concrete Distro classes. - """ - pass - - def checkPackageUpdateable(self,p): - """ - Online check if updated package of walinuxagent is available. - Abstract Virtual Function. Over-ridden in concrete Distro classes. - """ - pass - - def deleteRootPassword(self): - """ - Generic root password removal. - """ - filepath="/etc/shadow" - ReplaceFileContentsAtomic(filepath,"root:*LOCK*:14600::::::\n" - + "\n".join(filter(lambda a: not a.startswith("root:"),GetFileContents(filepath).split('\n')))) - os.chmod(filepath,self.shadow_file_mode) - if self.isSelinuxSystem(): - self.setSelinuxContext(filepath,'system_u:object_r:shadow_t:s0') - Log("Root password deleted.") - return 0 - - def changePass(self,user,password): - return RunSendStdin("chpasswd",(user + ":" + password + "\n")) - - def load_ata_piix(self): - return WaAgent.TryLoadAtapiix() - - def unload_ata_piix(self): - """ - Generic function to remove ata_piix.ko. - """ - return WaAgent.TryUnloadAtapiix() - - def deprovisionWarnUser(self): - """ - Generic user warnings used at deprovision. - """ - print("WARNING! Nameserver configuration in /etc/resolv.conf will be deleted.") - - def deprovisionDeleteFiles(self): - """ - Files to delete when VM is deprovisioned - """ - for a in VarLibDhcpDirectories: - Run("rm -f " + a + "/*") - - # Clear LibDir, remove nameserver and root bash history - - for f in os.listdir(LibDir) + self.fileBlackList: - try: - os.remove(f) - except: - pass - return 0 - - def uninstallDeleteFiles(self): - """ - Files to delete when agent is uninstalled. - """ - for f in self.agent_files_to_uninstall: - try: - os.remove(f) - except: - pass - return 0 - - def checkDependencies(self): - """ - Generic dependency check. - Return 1 unless all dependencies are satisfied. - """ - if self.checkPackageInstalled('NetworkManager'): - Error(GuestAgentLongName + " is not compatible with network-manager.") - return 1 - try: - m= __import__('pyasn1') - except ImportError: - Error(GuestAgentLongName + " requires python-pyasn1 for your Linux distribution.") - return 1 - for a in self.requiredDeps: - if Run("which " + a + " > /dev/null 2>&1",chk_err=False): - Error("Missing required dependency: " + a) - return 1 - return 0 - - def packagedInstall(self,buildroot): - """ - Called from setup.py for use by RPM. - Copies generated files waagent.conf, under the buildroot. - """ - if not os.path.exists(buildroot+'/etc'): - os.mkdir(buildroot+'/etc') - SetFileContents(buildroot+'/etc/waagent.conf', MyDistro.waagent_conf_file) - - if not os.path.exists(buildroot+'/etc/logrotate.d'): - os.mkdir(buildroot+'/etc/logrotate.d') - SetFileContents(buildroot+'/etc/logrotate.d/waagent', WaagentLogrotate) - - self.init_script_file=buildroot+self.init_script_file - # this allows us to call installAgentServiceScriptFiles() - if not os.path.exists(os.path.dirname(self.init_script_file)): - os.mkdir(os.path.dirname(self.init_script_file)) - self.installAgentServiceScriptFiles() - - def GetIpv4Address(self): - """ - Return the ip of the - first active non-loopback interface. - """ - addr='' - iface,addr=GetFirstActiveNetworkInterfaceNonLoopback() - return addr - - def GetMacAddress(self): - return GetMacAddress() - - def GetInterfaceName(self): - return GetFirstActiveNetworkInterfaceNonLoopback()[0] - - def RestartInterface(self, iface): - Run("ifdown " + iface + " && ifup " + iface) - - def CreateAccount(self,user, password, expiration, thumbprint): - return CreateAccount(user, password, expiration, thumbprint) - - def DeleteAccount(self,user): - return DeleteAccount(user) - - def ActivateResourceDisk(self): - """ - Format, mount, and if specified in the configuration - set resource disk as swap. - """ - global DiskActivated - format = Config.get("ResourceDisk.Format") - if format == None or format.lower().startswith("n"): - DiskActivated = True - return - device = DeviceForIdePort(1) - if device == None: - Error("ActivateResourceDisk: Unable to detect disk topology.") - return - device = "/dev/" + device - - mountlist = RunGetOutput("mount")[1] - mountpoint = GetMountPoint(mountlist, device) - - if(mountpoint): - Log("ActivateResourceDisk: " + device + "1 is already mounted.") - else: - mountpoint = Config.get("ResourceDisk.MountPoint") - if mountpoint == None: - mountpoint = "/mnt/resource" - CreateDir(mountpoint, "root", 0755) - fs = Config.get("ResourceDisk.Filesystem") - if fs == None: - fs = "ext3" - - partition = device + "1" - - #Check partition type - Log("Detect GPT...") - ret = RunGetOutput("parted {0} print".format(device)) - if ret[0] == 0 and "gpt" in ret[1]: - Log("GPT detected.") - #GPT(Guid Partition Table) is used. - #Get partitions. - parts = filter(lambda x : re.match("^\s*[0-9]+", x), ret[1].split("\n")) - #If there are more than 1 partitions, remove all partitions - #and create a new one using the entire disk space. - if len(parts) > 1: - for i in range(1, len(parts) + 1): - Run("parted {0} rm {1}".format(device, i)) - Run("parted {0} mkpart primary 0% 100%".format(device)) - Run("mkfs." + fs + " " + partition + " -F") - else: - existingFS = RunGetOutput("sfdisk -q -c " + device + " 1", chk_err=False)[1].rstrip() - if existingFS == "7" and fs != "ntfs": - Run("sfdisk -c " + device + " 1 83") - Run("mkfs." + fs + " " + partition) - if Run("mount " + partition + " " + mountpoint, chk_err=False): - #If mount failed, try to format the partition and mount again - Warn("Failed to mount resource disk. Retry mounting.") - Run("mkfs." + fs + " " + partition + " -F") - if Run("mount " + partition + " " + mountpoint): - Error("ActivateResourceDisk: Failed to mount resource disk (" + partition + ").") - return - Log("Resource disk (" + partition + ") is mounted at " + mountpoint + " with fstype " + fs) - - #Create README file under the root of resource disk - SetFileContents(os.path.join(mountpoint,README_FILENAME), README_FILECONTENT) - DiskActivated = True - - #Create swap space - swap = Config.get("ResourceDisk.EnableSwap") - if swap == None or swap.lower().startswith("n"): - return - sizeKB = int(Config.get("ResourceDisk.SwapSizeMB")) * 1024 - if os.path.isfile(mountpoint + "/swapfile") and os.path.getsize(mountpoint + "/swapfile") != (sizeKB * 1024): - os.remove(mountpoint + "/swapfile") - if not os.path.isfile(mountpoint + "/swapfile"): - Run("dd if=/dev/zero of=" + mountpoint + "/swapfile bs=1024 count=" + str(sizeKB)) - Run("mkswap " + mountpoint + "/swapfile") - if not Run("swapon " + mountpoint + "/swapfile"): - Log("Enabled " + str(sizeKB) + " KB of swap at " + mountpoint + "/swapfile") - else: - Error("ActivateResourceDisk: Failed to activate swap at " + mountpoint + "/swapfile") - - def Install(self): - return Install() - - def mediaHasFilesystem(self,dsk): - if len(dsk) == 0 : - return False - if Run("LC_ALL=C fdisk -l " + dsk + " | grep Disk"): - return False - return True - - def mountDVD(self,dvd,location): - return RunGetOutput(self.mount_dvd_cmd + ' ' + dvd + ' ' + location) - - def GetHome(self): - return GetHome() - - def getDhcpClientName(self): - return self.dhcp_client_name - - def initScsiDiskTimeout(self): - """ - Set the SCSI disk timeout when the agent starts running - """ - self.setScsiDiskTimeout() - - def setScsiDiskTimeout(self): - """ - Iterate all SCSI disks(include hot-add) and set their timeout if their value are different from the OS.RootDeviceScsiTimeout - """ - try: - scsiTimeout = Config.get("OS.RootDeviceScsiTimeout") - for diskName in [disk for disk in os.listdir("/sys/block") if disk.startswith("sd")]: - self.setBlockDeviceTimeout(diskName, scsiTimeout) - except: - pass - - def setBlockDeviceTimeout(self, device, timeout): - """ - Set SCSI disk timeout by set /sys/block/sd*/device/timeout - """ - if timeout != None and device: - filePath = "/sys/block/" + device + "/device/timeout" - if(GetFileContents(filePath).splitlines()[0].rstrip() != timeout): - SetFileContents(filePath,timeout) - Log("SetBlockDeviceTimeout: Update the device " + device + " with timeout " + timeout) - - def waitForSshHostKey(self, path): - """ - Provide a dummy waiting, since by default, ssh host key is created by waagent and the key - should already been created. - """ - if(os.path.isfile(path)): - return True - else: - Error("Can't find host key: {0}".format(path)) - return False - - def isDHCPEnabled(self): - return self.dhcp_enabled - - def stopDHCP(self): - """ - Stop the system DHCP client so that the agent can bind on its port. If - the distro has set dhcp_enabled to True, it will need to provide an - implementation of this method. - """ - raise NotImplementedError('stopDHCP method missing') - - def startDHCP(self): - """ - Start the system DHCP client. If the distro has set dhcp_enabled to - True, it will need to provide an implementation of this method. - """ - raise NotImplementedError('startDHCP method missing') - - def translateCustomData(self, data): - """ - Translate the custom data from a Base64 encoding. Default to no-op. - """ - decodeCustomData = Config.get("Provisioning.DecodeCustomData") - if decodeCustomData != None and decodeCustomData.lower().startswith("y"): - return base64.b64decode(data) - return data - - def getConfigurationPath(self): - return "/etc/waagent.conf" - - def getProcessorCores(self): - return int(RunGetOutput("grep 'processor.*:' /proc/cpuinfo |wc -l")[1]) - - def getTotalMemory(self): - return int(RunGetOutput("grep MemTotal /proc/meminfo |awk '{print $2}'")[1])/1024 - - def getInterfaceNameByMac(self, mac): - ret, output = RunGetOutput("ifconfig -a") - if ret != 0: - raise Exception("Failed to get network interface info") - output = output.replace('\n', '') - match = re.search(r"(eth\d).*(HWaddr|ether) {0}".format(mac), - output, re.IGNORECASE) - if match is None: - raise Exception("Failed to get ifname with mac: {0}".format(mac)) - output = match.group(0) - eths = re.findall(r"eth\d", output) - if eths is None or len(eths) == 0: - raise Exception("Failed to get ifname with mac: {0}".format(mac)) - return eths[-1] - - def configIpV4(self, ifName, addr, netmask=24): - ret, output = RunGetOutput("ifconfig {0} up".format(ifName)) - if ret != 0: - raise Exception("Failed to bring up {0}: {1}".format(ifName, - output)) - ret, output = RunGetOutput("ifconfig {0} {1}/{2}".format(ifName, addr, - netmask)) - if ret != 0: - raise Exception("Failed to config ipv4 for {0}: {1}".format(ifName, - output)) - -############################################################ -# GentooDistro -############################################################ -gentoo_init_file = """\ -#!/sbin/runscript - -command=/usr/sbin/waagent -pidfile=/var/run/waagent.pid -command_args=-daemon -command_background=true -name="Windows Azure Linux Agent" - -depend() -{ - need localmount - use logger network - after bootmisc modules -} - -""" -class gentooDistro(AbstractDistro): - """ - Gentoo distro concrete class - """ - - def __init__(self): # - super(gentooDistro,self).__init__() - self.service_cmd='/sbin/service' - self.ssh_service_name='sshd' - self.hostname_file_path='/etc/conf.d/hostname' - self.dhcp_client_name='dhcpcd' - self.shadow_file_mode=0640 - self.init_file=gentoo_init_file - - def publishHostname(self,name): - try: - if (os.path.isfile(self.hostname_file_path)): - r=ReplaceFileContentsAtomic(self.hostname_file_path, "hostname=\"" + name + "\"\n" - + "\n".join(filter(lambda a: not a.startswith("hostname="), GetFileContents(self.hostname_file_path).split("\n")))) - except: - return 1 - return r - - def installAgentServiceScriptFiles(self): - SetFileContents(self.init_script_file, self.init_file) - os.chmod(self.init_script_file, 0755) - - def registerAgentService(self): - self.installAgentServiceScriptFiles() - return Run('rc-update add ' + self.agent_service_name + ' default') - - def uninstallAgentService(self): - return Run('rc-update del ' + self.agent_service_name + ' default') - - def unregisterAgentService(self): - self.stopAgentService() - return self.uninstallAgentService() - - def checkPackageInstalled(self,p): - if Run('eix -I ^' + p + '$',chk_err=False): - return 0 - else: - return 1 - - def checkPackageUpdateable(self,p): - if Run('eix -u ^' + p + '$',chk_err=False): - return 0 - else: - return 1 - - def RestartInterface(self, iface): - Run("/etc/init.d/net." + iface + " restart") - -############################################################ -# SuSEDistro -############################################################ -suse_init_file = """\ -#! /bin/sh -# -# Windows Azure Linux Agent sysV init script -# -# Copyright 2013 Microsoft Corporation -# Copyright SUSE LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# /etc/init.d/waagent -# -# and symbolic link -# -# /usr/sbin/rcwaagent -# -# System startup script for the waagent -# -### BEGIN INIT INFO -# Provides: WindowsAzureLinuxAgent -# Required-Start: $network sshd -# Required-Stop: $network sshd -# Default-Start: 3 5 -# Default-Stop: 0 1 2 6 -# Description: Start the WindowsAzureLinuxAgent -### END INIT INFO - -PYTHON=/usr/bin/python -WAZD_BIN=/usr/sbin/waagent -WAZD_CONF=/etc/waagent.conf -WAZD_PIDFILE=/var/run/waagent.pid - -test -x "$WAZD_BIN" || { echo "$WAZD_BIN not installed"; exit 5; } -test -e "$WAZD_CONF" || { echo "$WAZD_CONF not found"; exit 6; } - -. /etc/rc.status - -# First reset status of this service -rc_reset - -# Return values acc. to LSB for all commands but status: -# 0 - success -# 1 - misc error -# 2 - invalid or excess args -# 3 - unimplemented feature (e.g. reload) -# 4 - insufficient privilege -# 5 - program not installed -# 6 - program not configured -# -# Note that starting an already running service, stopping -# or restarting a not-running service as well as the restart -# with force-reload (in case signalling is not supported) are -# considered a success. - - -case "$1" in - start) - echo -n "Starting WindowsAzureLinuxAgent" - ## Start daemon with startproc(8). If this fails - ## the echo return value is set appropriate. - startproc -f ${PYTHON} ${WAZD_BIN} -daemon - rc_status -v - ;; - stop) - echo -n "Shutting down WindowsAzureLinuxAgent" - ## Stop daemon with killproc(8) and if this fails - ## set echo the echo return value. - killproc -p ${WAZD_PIDFILE} ${PYTHON} ${WAZD_BIN} - rc_status -v - ;; - try-restart) - ## Stop the service and if this succeeds (i.e. the - ## service was running before), start it again. - $0 status >/dev/null && $0 restart - rc_status - ;; - restart) - ## Stop the service and regardless of whether it was - ## running or not, start it again. - $0 stop - sleep 1 - $0 start - rc_status - ;; - force-reload|reload) - rc_status - ;; - status) - echo -n "Checking for service WindowsAzureLinuxAgent " - ## Check status with checkproc(8), if process is running - ## checkproc will return with exit status 0. - - checkproc -p ${WAZD_PIDFILE} ${PYTHON} ${WAZD_BIN} - rc_status -v - ;; - probe) - ;; - *) - echo "Usage: $0 {start|stop|status|try-restart|restart|force-reload|reload}" - exit 1 - ;; -esac -rc_exit -""" -class SuSEDistro(AbstractDistro): - """ - SuSE Distro concrete class - Put SuSE specific behavior here... - """ - def __init__(self): - super(SuSEDistro,self).__init__() - self.service_cmd='/sbin/service' - self.ssh_service_name='sshd' - self.kernel_boot_options_file='/boot/grub/menu.lst' - self.hostname_file_path='/etc/HOSTNAME' - self.requiredDeps += [ "/sbin/insserv" ] - self.init_file=suse_init_file - self.dhcp_client_name='dhcpcd' - if ((DistInfo(fullname=1)[0] == 'SUSE Linux Enterprise Server' and DistInfo()[1] >= '12') or \ - (DistInfo(fullname=1)[0] == 'openSUSE' and DistInfo()[1] >= '13.2')): - self.dhcp_client_name='wickedd-dhcp4' - self.grubKernelBootOptionsFile = '/boot/grub/menu.lst' - self.grubKernelBootOptionsLine = 'kernel' - self.getpidcmd='pidof ' - self.dhcp_enabled=True - - def checkPackageInstalled(self,p): - if Run("rpm -q " + p,chk_err=False): - return 0 - else: - return 1 - - def checkPackageUpdateable(self,p): - if Run("zypper list-updates | grep " + p,chk_err=False): - return 1 - else: - return 0 - - - def installAgentServiceScriptFiles(self): - try: - SetFileContents(self.init_script_file, self.init_file) - os.chmod(self.init_script_file, 0744) - except: - pass - - def registerAgentService(self): - self.installAgentServiceScriptFiles() - return Run('insserv ' + self.agent_service_name) - - def uninstallAgentService(self): - return Run('insserv -r ' + self.agent_service_name) - - def unregisterAgentService(self): - self.stopAgentService() - return self.uninstallAgentService() - - def startDHCP(self): - Run("service " + self.dhcp_client_name + " start", chk_err=False) - - def stopDHCP(self): - Run("service " + self.dhcp_client_name + " stop", chk_err=False) - -############################################################ -# redhatDistro -############################################################ - -redhat_init_file= """\ -#!/bin/bash -# -# Init file for WindowsAzureLinuxAgent. -# -# chkconfig: 2345 60 80 -# description: WindowsAzureLinuxAgent -# - -# source function library -. /etc/rc.d/init.d/functions - -RETVAL=0 -FriendlyName="WindowsAzureLinuxAgent" -WAZD_BIN=/usr/sbin/waagent - -start() -{ - echo -n $"Starting $FriendlyName: " - $WAZD_BIN -daemon & -} - -stop() -{ - echo -n $"Stopping $FriendlyName: " - killproc -p /var/run/waagent.pid $WAZD_BIN - RETVAL=$? - echo - return $RETVAL -} - -case "$1" in - start) - start - ;; - stop) - stop - ;; - restart) - stop - start - ;; - reload) - ;; - report) - ;; - status) - status $WAZD_BIN - RETVAL=$? - ;; - *) - echo $"Usage: $0 {start|stop|restart|status}" - RETVAL=1 -esac -exit $RETVAL -""" - -class redhatDistro(AbstractDistro): - """ - Redhat Distro concrete class - Put Redhat specific behavior here... - """ - def __init__(self): - super(redhatDistro,self).__init__() - self.service_cmd='/sbin/service' - self.ssh_service_restart_option='condrestart' - self.ssh_service_name='sshd' - self.hostname_file_path= None if DistInfo()[1] < '7.0' else '/etc/hostname' - self.init_file=redhat_init_file - self.grubKernelBootOptionsFile = '/boot/grub/menu.lst' - self.grubKernelBootOptionsLine = 'kernel' - - def publishHostname(self,name): - super(redhatDistro,self).publishHostname(name) - if DistInfo()[1] < '7.0' : - filepath = "/etc/sysconfig/network" - if os.path.isfile(filepath): - ReplaceFileContentsAtomic(filepath, "HOSTNAME=" + name + "\n" - + "\n".join(filter(lambda a: not a.startswith("HOSTNAME"), GetFileContents(filepath).split('\n')))) - - ethernetInterface = MyDistro.GetInterfaceName() - filepath = "/etc/sysconfig/network-scripts/ifcfg-" + ethernetInterface - if os.path.isfile(filepath): - ReplaceFileContentsAtomic(filepath, "DHCP_HOSTNAME=" + name + "\n" - + "\n".join(filter(lambda a: not a.startswith("DHCP_HOSTNAME"), GetFileContents(filepath).split('\n')))) - return 0 - - def installAgentServiceScriptFiles(self): - SetFileContents(self.init_script_file, self.init_file) - os.chmod(self.init_script_file, 0744) - return 0 - - def registerAgentService(self): - self.installAgentServiceScriptFiles() - return Run('chkconfig --add waagent') - - def uninstallAgentService(self): - return Run('chkconfig --del ' + self.agent_service_name) - - def unregisterAgentService(self): - self.stopAgentService() - return self.uninstallAgentService() - - def checkPackageInstalled(self,p): - if Run("yum list installed " + p,chk_err=False): - return 0 - else: - return 1 - - def checkPackageUpdateable(self,p): - if Run("yum check-update | grep "+ p,chk_err=False): - return 1 - else: - return 0 - - - -############################################################ -# centosDistro -############################################################ - -class centosDistro(redhatDistro): - """ - CentOS Distro concrete class - Put CentOS specific behavior here... - """ - def __init__(self): - super(centosDistro,self).__init__() - - -############################################################ -# CoreOSDistro -############################################################ - -class CoreOSDistro(AbstractDistro): - """ - CoreOS Distro concrete class - Put CoreOS specific behavior here... - """ - CORE_UID = 500 - - def __init__(self): - super(CoreOSDistro,self).__init__() - self.requiredDeps += [ "/usr/bin/systemctl" ] - self.agent_service_name = 'waagent' - self.init_script_file='/etc/systemd/system/waagent.service' - self.fileBlackList.append("/etc/machine-id") - self.dhcp_client_name='systemd-networkd' - self.getpidcmd='pidof ' - self.shadow_file_mode=0640 - self.waagent_path='/usr/share/oem/bin' - self.python_path='/usr/share/oem/python/bin' - self.dhcp_enabled=True - if 'PATH' in os.environ: - os.environ['PATH'] = "{0}:{1}".format(os.environ['PATH'], self.python_path) - else: - os.environ['PATH'] = self.python_path - - if 'PYTHONPATH' in os.environ: - os.environ['PYTHONPATH'] = "{0}:{1}".format(os.environ['PYTHONPATH'], self.waagent_path) - else: - os.environ['PYTHONPATH'] = self.waagent_path - - def checkPackageInstalled(self,p): - """ - There is no package manager in CoreOS. Return 1 since it must be preinstalled. - """ - return 1 - - def checkDependencies(self): - for a in self.requiredDeps: - if Run("which " + a + " > /dev/null 2>&1",chk_err=False): - Error("Missing required dependency: " + a) - return 1 - return 0 - - - def checkPackageUpdateable(self,p): - """ - There is no package manager in CoreOS. Return 0 since it can't be updated via package. - """ - return 0 - - def startAgentService(self): - return Run('systemctl start ' + self.agent_service_name) - - def stopAgentService(self): - return Run('systemctl stop ' + self.agent_service_name) - - def restartSshService(self): - """ - SSH is socket activated on CoreOS. No need to restart it. - """ - return 0 - - def sshDeployPublicKey(self,fprint,path): - """ - We support PKCS8. - """ - if Run("ssh-keygen -i -m PKCS8 -f " + fprint + " >> " + path): - return 1 - else : - return 0 - - def RestartInterface(self, iface): - Run("systemctl restart systemd-networkd") - - def CreateAccount(self, user, password, expiration, thumbprint): - """ - Create a user account, with 'user', 'password', 'expiration', ssh keys - and sudo permissions. - Returns None if successful, error string on failure. - """ - userentry = None - try: - userentry = pwd.getpwnam(user) - except: - pass - uidmin = None - try: - uidmin = int(GetLineStartingWith("UID_MIN", "/etc/login.defs").split()[1]) - except: - pass - if uidmin == None: - uidmin = 100 - if userentry != None and userentry[2] < uidmin and userentry[2] != self.CORE_UID: - Error("CreateAccount: " + user + " is a system user. Will not set password.") - return "Failed to set password for system user: " + user + " (0x06)." - if userentry == None: - command = "useradd --create-home --password '*' " + user - if expiration != None: - command += " --expiredate " + expiration.split('.')[0] - if Run(command): - Error("Failed to create user account: " + user) - return "Failed to create user account: " + user + " (0x07)." - else: - Log("CreateAccount: " + user + " already exists. Will update password.") - if password != None: - RunSendStdin("chpasswd", user + ":" + password + "\n") - try: - if password == None: - SetFileContents("/etc/sudoers.d/waagent", user + " ALL = (ALL) NOPASSWD: ALL\n") - else: - SetFileContents("/etc/sudoers.d/waagent", user + " ALL = (ALL) ALL\n") - os.chmod("/etc/sudoers.d/waagent", 0440) - except: - Error("CreateAccount: Failed to configure sudo access for user.") - return "Failed to configure sudo privileges (0x08)." - home = MyDistro.GetHome() - if thumbprint != None: - dir = home + "/" + user + "/.ssh" - CreateDir(dir, user, 0700) - pub = dir + "/id_rsa.pub" - prv = dir + "/id_rsa" - Run("ssh-keygen -y -f " + thumbprint + ".prv > " + pub) - SetFileContents(prv, GetFileContents(thumbprint + ".prv")) - for f in [pub, prv]: - os.chmod(f, 0600) - ChangeOwner(f, user) - SetFileContents(dir + "/authorized_keys", GetFileContents(pub)) - ChangeOwner(dir + "/authorized_keys", user) - Log("Created user account: " + user) - return None - - def startDHCP(self): - Run("systemctl start " + self.dhcp_client_name, chk_err=False) - - def stopDHCP(self): - Run("systemctl stop " + self.dhcp_client_name, chk_err=False) - - def translateCustomData(self, data): - return base64.b64decode(data) - - def getConfigurationPath(self): - return "/usr/share/oem/waagent.conf" - -############################################################ -# debianDistro -############################################################ -debian_init_file = """\ -#!/bin/sh -### BEGIN INIT INFO -# Provides: WindowsAzureLinuxAgent -# Required-Start: $network $syslog -# Required-Stop: $network $syslog -# Should-Start: $network $syslog -# Should-Stop: $network $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: WindowsAzureLinuxAgent -# Description: WindowsAzureLinuxAgent -### END INIT INFO - -. /lib/lsb/init-functions - -OPTIONS="-daemon" -WAZD_BIN=/usr/sbin/waagent -WAZD_PID=/var/run/waagent.pid - -case "$1" in - start) - log_begin_msg "Starting WindowsAzureLinuxAgent..." - pid=$( pidofproc $WAZD_BIN ) - if [ -n "$pid" ] ; then - log_begin_msg "Already running." - log_end_msg 0 - exit 0 - fi - start-stop-daemon --start --quiet --oknodo --background --exec $WAZD_BIN -- $OPTIONS - log_end_msg $? - ;; - - stop) - log_begin_msg "Stopping WindowsAzureLinuxAgent..." - start-stop-daemon --stop --quiet --oknodo --pidfile $WAZD_PID - ret=$? - rm -f $WAZD_PID - log_end_msg $ret - ;; - force-reload) - $0 restart - ;; - restart) - $0 stop - $0 start - ;; - status) - status_of_proc $WAZD_BIN && exit 0 || exit $? - ;; - *) - log_success_msg "Usage: /etc/init.d/waagent {start|stop|force-reload|restart|status}" - exit 1 - ;; -esac - -exit 0 -""" - -class debianDistro(AbstractDistro): - """ - debian Distro concrete class - Put debian specific behavior here... - """ - def __init__(self): - super(debianDistro,self).__init__() - self.requiredDeps += [ "/usr/sbin/update-rc.d" ] - self.init_file=debian_init_file - self.agent_package_name='walinuxagent' - self.dhcp_client_name='dhclient' - self.getpidcmd='pidof ' - self.shadow_file_mode=0640 - - def checkPackageInstalled(self,p): - """ - Check that the package is installed. - Return 1 if installed, 0 if not installed. - This method of using dpkg-query - allows wildcards to be present in the - package name. - """ - if not Run("dpkg-query -W -f='${Status}\n' '" + p + "' | grep ' installed' 2>&1",chk_err=False): - return 1 - else: - return 0 - - def checkDependencies(self): - """ - Debian dependency check. python-pyasn1 is NOT needed. - Return 1 unless all dependencies are satisfied. - NOTE: using network*manager will catch either package name in Ubuntu or debian. - """ - if self.checkPackageInstalled('network*manager'): - Error(GuestAgentLongName + " is not compatible with network-manager.") - return 1 - for a in self.requiredDeps: - if Run("which " + a + " > /dev/null 2>&1",chk_err=False): - Error("Missing required dependency: " + a) - return 1 - return 0 - - def checkPackageUpdateable(self,p): - if Run("apt-get update ; apt-get upgrade -us | grep " + p,chk_err=False): - return 1 - else: - return 0 - - def installAgentServiceScriptFiles(self): - """ - If we are packaged - the service name is walinuxagent, do nothing. - """ - if self.agent_service_name == 'walinuxagent': - return 0 - try: - SetFileContents(self.init_script_file, self.init_file) - os.chmod(self.init_script_file, 0744) - except OSError, e: - ErrorWithPrefix('installAgentServiceScriptFiles','Exception: '+str(e)+' occured creating ' + self.init_script_file) - return 1 - return 0 - - def registerAgentService(self): - if self.installAgentServiceScriptFiles() == 0: - return Run('update-rc.d waagent defaults') - else : - return 1 - - def uninstallAgentService(self): - return Run('update-rc.d -f ' + self.agent_service_name + ' remove') - - def unregisterAgentService(self): - self.stopAgentService() - return self.uninstallAgentService() - - def sshDeployPublicKey(self,fprint,path): - """ - We support PKCS8. - """ - if Run("ssh-keygen -i -m PKCS8 -f " + fprint + " >> " + path): - return 1 - else : - return 0 - -############################################################ -# KaliDistro - WIP -# Functioning on Kali 1.1.0a so far -############################################################ -class KaliDistro(debianDistro): - """ - Kali Distro concrete class - Put Kali specific behavior here... - """ - def __init__(self): - super(KaliDistro,self).__init__() - -############################################################ -# UbuntuDistro -############################################################ -ubuntu_upstart_file = """\ -#walinuxagent - start Windows Azure agent - -description "walinuxagent" -author "Ben Howard " - -start on (filesystem and started rsyslog) - -pre-start script - - WALINUXAGENT_ENABLED=1 - [ -r /etc/default/walinuxagent ] && . /etc/default/walinuxagent - - if [ "$WALINUXAGENT_ENABLED" != "1" ]; then - exit 1 - fi - - if [ ! -x /usr/sbin/waagent ]; then - exit 1 - fi - - #Load the udf module - modprobe -b udf -end script - -exec /usr/sbin/waagent -daemon -""" - -class UbuntuDistro(debianDistro): - """ - Ubuntu Distro concrete class - Put Ubuntu specific behavior here... - """ - def __init__(self): - super(UbuntuDistro,self).__init__() - self.init_script_file='/etc/init/waagent.conf' - self.init_file=ubuntu_upstart_file - self.fileBlackList = [ "/root/.bash_history", "/var/log/waagent.log"] - self.dhcp_client_name=None - self.getpidcmd='pidof ' - - def registerAgentService(self): - return self.installAgentServiceScriptFiles() - - def uninstallAgentService(self): - """ - If we are packaged - the service name is walinuxagent, do nothing. - """ - if self.agent_service_name == 'walinuxagent': - return 0 - os.remove('/etc/init/' + self.agent_service_name + '.conf') - - def unregisterAgentService(self): - """ - If we are packaged - the service name is walinuxagent, do nothing. - """ - if self.agent_service_name == 'walinuxagent': - return - self.stopAgentService() - return self.uninstallAgentService() - - def deprovisionWarnUser(self): - """ - Ubuntu specific warning string from Deprovision. - """ - print("WARNING! Nameserver configuration in /etc/resolvconf/resolv.conf.d/{tail,original} will be deleted.") - - def deprovisionDeleteFiles(self): - """ - Ubuntu uses resolv.conf by default, so removing /etc/resolv.conf will - break resolvconf. Therefore, we check to see if resolvconf is in use, - and if so, we remove the resolvconf artifacts. - """ - if os.path.realpath('/etc/resolv.conf') != '/run/resolvconf/resolv.conf': - Log("resolvconf is not configured. Removing /etc/resolv.conf") - self.fileBlackList.append('/etc/resolv.conf') - else: - Log("resolvconf is enabled; leaving /etc/resolv.conf intact") - resolvConfD = '/etc/resolvconf/resolv.conf.d/' - self.fileBlackList.extend([resolvConfD + 'tail', resolvConfD + 'original']) - for f in os.listdir(LibDir)+self.fileBlackList: - try: - os.remove(f) - except: - pass - return 0 - - def getDhcpClientName(self): - if self.dhcp_client_name != None : - return self.dhcp_client_name - if DistInfo()[1] == '12.04' : - self.dhcp_client_name='dhclient3' - else : - self.dhcp_client_name='dhclient' - return self.dhcp_client_name - - def waitForSshHostKey(self, path): - """ - Wait until the ssh host key is generated by cloud init. - """ - for retry in range(0, 10): - if(os.path.isfile(path)): - return True - time.sleep(1) - Error("Can't find host key: {0}".format(path)) - return False - - -############################################################ -# LinuxMintDistro -############################################################ - -class LinuxMintDistro(UbuntuDistro): - """ - LinuxMint Distro concrete class - Put LinuxMint specific behavior here... - """ - def __init__(self): - super(LinuxMintDistro,self).__init__() - -############################################################ -# fedoraDistro -############################################################ -fedora_systemd_service = """\ -[Unit] -Description=Windows Azure Linux Agent -After=network.target -After=sshd.service -ConditionFileIsExecutable=/usr/sbin/waagent -ConditionPathExists=/etc/waagent.conf - -[Service] -Type=simple -ExecStart=/usr/sbin/waagent -daemon - -[Install] -WantedBy=multi-user.target -""" - -class fedoraDistro(redhatDistro): - """ - FedoraDistro concrete class - Put Fedora specific behavior here... - """ - def __init__(self): - super(fedoraDistro,self).__init__() - self.service_cmd = '/usr/bin/systemctl' - self.hostname_file_path = '/etc/hostname' - self.init_script_file = '/usr/lib/systemd/system/' + self.agent_service_name + '.service' - self.init_file = fedora_systemd_service - self.grubKernelBootOptionsFile = '/etc/default/grub' - self.grubKernelBootOptionsLine = 'GRUB_CMDLINE_LINUX=' - - def publishHostname(self, name): - SetFileContents(self.hostname_file_path, name + '\n') - ethernetInterface = MyDistro.GetInterfaceName() - filepath = "/etc/sysconfig/network-scripts/ifcfg-" + ethernetInterface - if os.path.isfile(filepath): - ReplaceFileContentsAtomic(filepath, "DHCP_HOSTNAME=" + name + "\n" - + "\n".join(filter(lambda a: not a.startswith("DHCP_HOSTNAME"), GetFileContents(filepath).split('\n')))) - return 0 - - def installAgentServiceScriptFiles(self): - SetFileContents(self.init_script_file, self.init_file) - os.chmod(self.init_script_file, 0644) - return Run(self.service_cmd + ' daemon-reload') - - def registerAgentService(self): - self.installAgentServiceScriptFiles() - return Run(self.service_cmd + ' enable ' + self.agent_service_name) - - def uninstallAgentService(self): - """ - Call service subsystem to remove waagent script. - """ - return Run(self.service_cmd + ' disable ' + self.agent_service_name) - - def unregisterAgentService(self): - """ - Calls self.stopAgentService and call self.uninstallAgentService() - """ - self.stopAgentService() - self.uninstallAgentService() - - def startAgentService(self): - """ - Service call to start the Agent service - """ - return Run(self.service_cmd + ' start ' + self.agent_service_name) - - def stopAgentService(self): - """ - Service call to stop the Agent service - """ - return Run(self.service_cmd + ' stop ' + self.agent_service_name, False) - - def restartSshService(self): - """ - Service call to re(start) the SSH service - """ - sshRestartCmd = self.service_cmd + " " + self.ssh_service_restart_option + " " + self.ssh_service_name - retcode = Run(sshRestartCmd) - if retcode > 0: - Error("Failed to restart SSH service with return code:" + str(retcode)) - return retcode - - def checkPackageInstalled(self, p): - """ - Query package database for prescence of an installed package. - """ - import rpm - ts = rpm.TransactionSet() - rpms = ts.dbMatch(rpm.RPMTAG_PROVIDES, p) - return bool(len(rpms) > 0) - - def deleteRootPassword(self): - return Run("/sbin/usermod root -p '!!'") - - def packagedInstall(self,buildroot): - """ - Called from setup.py for use by RPM. - Copies generated files waagent.conf, under the buildroot. - """ - if not os.path.exists(buildroot+'/etc'): - os.mkdir(buildroot+'/etc') - SetFileContents(buildroot+'/etc/waagent.conf', MyDistro.waagent_conf_file) - - if not os.path.exists(buildroot+'/etc/logrotate.d'): - os.mkdir(buildroot+'/etc/logrotate.d') - SetFileContents(buildroot+'/etc/logrotate.d/WALinuxAgent', WaagentLogrotate) - - self.init_script_file=buildroot+self.init_script_file - # this allows us to call installAgentServiceScriptFiles() - if not os.path.exists(os.path.dirname(self.init_script_file)): - os.mkdir(os.path.dirname(self.init_script_file)) - self.installAgentServiceScriptFiles() - - def CreateAccount(self, user, password, expiration, thumbprint): - super(fedoraDistro, self).CreateAccount(user, password, expiration, thumbprint) - Run('/sbin/usermod ' + user + ' -G wheel') - - def DeleteAccount(self, user): - Run('/sbin/usermod ' + user + ' -G ""') - super(fedoraDistro, self).DeleteAccount(user) - -############################################################ -# FreeBSD -############################################################ -FreeBSDWaagentConf = """\ -# -# Windows Azure Linux Agent Configuration -# - -Role.StateConsumer=None # Specified program is invoked with the argument "Ready" when we report ready status - # to the endpoint server. -Role.ConfigurationConsumer=None # Specified program is invoked with XML file argument specifying role configuration. -Role.TopologyConsumer=None # Specified program is invoked with XML file argument specifying role topology. - -Provisioning.Enabled=y # -Provisioning.DeleteRootPassword=y # Password authentication for root account will be unavailable. -Provisioning.RegenerateSshHostKeyPair=y # Generate fresh host key pair. -Provisioning.SshHostKeyPairType=rsa # Supported values are "rsa", "dsa" and "ecdsa". -Provisioning.MonitorHostName=y # Monitor host name changes and publish changes via DHCP requests. - -ResourceDisk.Format=y # Format if unformatted. If 'n', resource disk will not be mounted. -ResourceDisk.Filesystem=ufs2 # -ResourceDisk.MountPoint=/mnt/resource # -ResourceDisk.EnableSwap=n # Create and use swapfile on resource disk. -ResourceDisk.SwapSizeMB=0 # Size of the swapfile. - -LBProbeResponder=y # Respond to load balancer probes if requested by Windows Azure. - -Logs.Verbose=n # Enable verbose logs - -OS.RootDeviceScsiTimeout=300 # Root device timeout in seconds. -OS.OpensslPath=None # If "None", the system default version is used. -""" - -bsd_init_file="""\ -#! /bin/sh - -# PROVIDE: waagent -# REQUIRE: DAEMON cleanvar sshd -# BEFORE: LOGIN -# KEYWORD: nojail - -. /etc/rc.subr -export PATH=$PATH:/usr/local/bin -name="waagent" -rcvar="waagent_enable" -command="/usr/sbin/${name}" -command_interpreter="/usr/local/bin/python" -waagent_flags=" daemon &" - -pidfile="/var/run/waagent.pid" - -load_rc_config $name -run_rc_command "$1" - -""" -bsd_activate_resource_disk_txt="""\ -#!/usr/bin/env python - -import os -import sys -import imp - -# waagent has no '.py' therefore create waagent module import manually. -__name__='setupmain' #prevent waagent.__main__ from executing -waagent=imp.load_source('waagent','/tmp/waagent') -waagent.LoggerInit('/var/log/waagent.log','/dev/console') -from waagent import RunGetOutput,Run -Config=waagent.ConfigurationProvider() -format = Config.get("ResourceDisk.Format") -if format == None or format.lower().startswith("n"): - sys.exit(0) -device_base = 'da1' -device = "/dev/" + device_base -for entry in RunGetOutput("mount")[1].split(): - if entry.startswith(device + "s1"): - waagent.Log("ActivateResourceDisk: " + device + "s1 is already mounted.") - sys.exit(0) -mountpoint = Config.get("ResourceDisk.MountPoint") -if mountpoint == None: - mountpoint = "/mnt/resource" -waagent.CreateDir(mountpoint, "root", 0755) -fs = Config.get("ResourceDisk.Filesystem") -if waagent.FreeBSDDistro().mediaHasFilesystem(device) == False : - Run("newfs " + device + "s1") -if Run("mount " + device + "s1 " + mountpoint): - waagent.Error("ActivateResourceDisk: Failed to mount resource disk (" + device + "s1).") - sys.exit(0) -waagent.Log("Resource disk (" + device + "s1) is mounted at " + mountpoint + " with fstype " + fs) -waagent.SetFileContents(os.path.join(mountpoint,waagent.README_FILENAME), waagent.README_FILECONTENT) -swap = Config.get("ResourceDisk.EnableSwap") -if swap == None or swap.lower().startswith("n"): - sys.exit(0) -sizeKB = int(Config.get("ResourceDisk.SwapSizeMB")) * 1024 -if os.path.isfile(mountpoint + "/swapfile") and os.path.getsize(mountpoint + "/swapfile") != (sizeKB * 1024): - os.remove(mountpoint + "/swapfile") -if not os.path.isfile(mountpoint + "/swapfile"): - Run("dd if=/dev/zero of=" + mountpoint + "/swapfile bs=1024 count=" + str(sizeKB)) -if Run("mdconfig -a -t vnode -f " + mountpoint + "/swapfile -u 0"): - waagent.Error("ActivateResourceDisk: Configuring swap - Failed to create md0") -if not Run("swapon /dev/md0"): - waagent.Log("Enabled " + str(sizeKB) + " KB of swap at " + mountpoint + "/swapfile") -else: - waagent.Error("ActivateResourceDisk: Failed to activate swap at " + mountpoint + "/swapfile") -""" - -class FreeBSDDistro(AbstractDistro): - """ - """ - def __init__(self): - """ - Generic Attributes go here. These are based on 'majority rules'. - This __init__() may be called or overriden by the child. - """ - super(FreeBSDDistro,self).__init__() - self.agent_service_name = os.path.basename(sys.argv[0]) - self.selinux=False - self.ssh_service_name='sshd' - self.ssh_config_file='/etc/ssh/sshd_config' - self.hostname_file_path='/etc/hostname' - self.dhcp_client_name='dhclient' - self.requiredDeps = [ 'route', 'shutdown', 'ssh-keygen', 'pw' - , 'openssl', 'fdisk', 'sed', 'grep' , 'sudo'] - self.init_script_file='/etc/rc.d/waagent' - self.init_file=bsd_init_file - self.agent_package_name='WALinuxAgent' - self.fileBlackList = [ "/root/.bash_history", "/var/log/waagent.log",'/etc/resolv.conf' ] - self.agent_files_to_uninstall = ["/etc/waagent.conf"] - self.grubKernelBootOptionsFile = '/boot/loader.conf' - self.grubKernelBootOptionsLine = '' - self.getpidcmd = 'pgrep -n' - self.mount_dvd_cmd = 'dd bs=2048 count=33 skip=295 if=' # custom data max len is 64k - self.sudoers_dir_base = '/usr/local/etc' - self.waagent_conf_file = FreeBSDWaagentConf - - def installAgentServiceScriptFiles(self): - SetFileContents(self.init_script_file, self.init_file) - os.chmod(self.init_script_file, 0777) - AppendFileContents("/etc/rc.conf","waagent_enable='YES'\n") - return 0 - - def registerAgentService(self): - self.installAgentServiceScriptFiles() - return Run("services_mkdb " + self.init_script_file) - - - def sshDeployPublicKey(self,fprint,path): - """ - We support PKCS8. - """ - if Run("ssh-keygen -i -m PKCS8 -f " + fprint + " >> " + path): - return 1 - else : - return 0 - - def deleteRootPassword(self): - """ - BSD root password removal. - """ - filepath="/etc/master.passwd" - ReplaceStringInFile(filepath,r'root:.*?:','root::') - #ReplaceFileContentsAtomic(filepath,"root:*LOCK*:14600::::::\n" - # + "\n".join(filter(lambda a: not a.startswith("root:"),GetFileContents(filepath).split('\n')))) - os.chmod(filepath,self.shadow_file_mode) - if self.isSelinuxSystem(): - self.setSelinuxContext(filepath,'system_u:object_r:shadow_t:s0') - RunGetOutput("pwd_mkdb -u root /etc/master.passwd") - Log("Root password deleted.") - return 0 - - def changePass(self,user,password): - return RunSendStdin("pw usermod " + user + " -h 0 ",password) - - def load_ata_piix(self): - return 0 - - def unload_ata_piix(self): - return 0 - - def checkDependencies(self): - """ - FreeBSD dependency check. - Return 1 unless all dependencies are satisfied. - """ - for a in self.requiredDeps: - if Run("which " + a + " > /dev/null 2>&1",chk_err=False): - Error("Missing required dependency: " + a) - return 1 - return 0 - - def packagedInstall(self,buildroot): - pass - - def GetInterfaceName(self): - """ - Return the ip of the - active ethernet interface. - """ - iface,inet,mac=self.GetFreeBSDEthernetInfo() - return iface - - def RestartInterface(self, iface): - Run("service netif restart") - - def GetIpv4Address(self): - """ - Return the ip of the - active ethernet interface. - """ - iface,inet,mac=self.GetFreeBSDEthernetInfo() - return inet - - def GetMacAddress(self): - """ - Return the ip of the - active ethernet interface. - """ - iface,inet,mac=self.GetFreeBSDEthernetInfo() - l=mac.split(':') - r=[] - for i in l: - r.append(string.atoi(i,16)) - return r - - def GetFreeBSDEthernetInfo(self): - """ - There is no SIOCGIFCONF - on freeBSD - just parse ifconfig. - Returns strings: iface, inet4_addr, and mac - or 'None,None,None' if unable to parse. - We will sleep and retry as the network must be up. - """ - code,output=RunGetOutput("ifconfig",chk_err=False) - Log(output) - retries=10 - cmd='ifconfig | grep -A2 -B2 ether | grep -B3 inet | grep -A4 UP ' - code=1 - - while code > 0 : - if code > 0 and retries == 0: - Error("GetFreeBSDEthernetInfo - Failed to detect ethernet interface") - return None, None, None - code,output=RunGetOutput(cmd,chk_err=False) - retries-=1 - if code > 0 and retries > 0 : - Log("GetFreeBSDEthernetInfo - Error: retry ethernet detection " + str(retries)) - if retries == 9 : - c,o=RunGetOutput("ifconfig | grep -A1 -B2 ether",chk_err=False) - if c == 0: - t=o.replace('\n',' ') - t=t.split() - i=t[0][:-1] - Log(RunGetOutput('id')[1]) - Run('dhclient '+i) - time.sleep(10) - - j=output.replace('\n',' ') - j=j.split() - iface=j[0][:-1] - - for i in range(len(j)): - if j[i] == 'inet' : - inet=j[i+1] - elif j[i] == 'ether' : - mac=j[i+1] - - return iface, inet, mac - - def CreateAccount(self,user, password, expiration, thumbprint): - """ - Create a user account, with 'user', 'password', 'expiration', ssh keys - and sudo permissions. - Returns None if successful, error string on failure. - """ - userentry = None - try: - userentry = pwd.getpwnam(user) - except: - pass - uidmin = None - try: - if os.path.isfile("/etc/login.defs"): - uidmin = int(GetLineStartingWith("UID_MIN", "/etc/login.defs").split()[1]) - except: - pass - if uidmin == None: - uidmin = 100 - if userentry != None and userentry[2] < uidmin: - Error("CreateAccount: " + user + " is a system user. Will not set password.") - return "Failed to set password for system user: " + user + " (0x06)." - if userentry == None: - command = "pw useradd " + user + " -m" - if expiration != None: - command += " -e " + expiration.split('.')[0] - if Run(command): - Error("Failed to create user account: " + user) - return "Failed to create user account: " + user + " (0x07)." - else: - Log("CreateAccount: " + user + " already exists. Will update password.") - - if password != None: - self.changePass(user,password) - try: - # for older distros create sudoers.d - if not os.path.isdir(MyDistro.sudoers_dir_base+'/sudoers.d/'): - # create the /etc/sudoers.d/ directory - os.mkdir(MyDistro.sudoers_dir_base+'/sudoers.d') - # add the include of sudoers.d to the /etc/sudoers - SetFileContents(MyDistro.sudoers_dir_base+'/sudoers',GetFileContents(MyDistro.sudoers_dir_base+'/sudoers')+'\n#includedir ' + MyDistro.sudoers_dir_base + '/sudoers.d\n') - if password == None: - SetFileContents(MyDistro.sudoers_dir_base+"/sudoers.d/waagent", user + " ALL = (ALL) NOPASSWD: ALL\n") - else: - SetFileContents(MyDistro.sudoers_dir_base+"/sudoers.d/waagent", user + " ALL = (ALL) ALL\n") - os.chmod(MyDistro.sudoers_dir_base+"/sudoers.d/waagent", 0440) - except: - Error("CreateAccount: Failed to configure sudo access for user.") - return "Failed to configure sudo privileges (0x08)." - home = MyDistro.GetHome() - if thumbprint != None: - dir = home + "/" + user + "/.ssh" - CreateDir(dir, user, 0700) - pub = dir + "/id_rsa.pub" - prv = dir + "/id_rsa" - Run("ssh-keygen -y -f " + thumbprint + ".prv > " + pub) - SetFileContents(prv, GetFileContents(thumbprint + ".prv")) - for f in [pub, prv]: - os.chmod(f, 0600) - ChangeOwner(f, user) - SetFileContents(dir + "/authorized_keys", GetFileContents(pub)) - ChangeOwner(dir + "/authorized_keys", user) - Log("Created user account: " + user) - return None - - def DeleteAccount(self,user): - """ - Delete the 'user'. - Clear utmp first, to avoid error. - Removes the /etc/sudoers.d/waagent file. - """ - userentry = None - try: - userentry = pwd.getpwnam(user) - except: - pass - if userentry == None: - Error("DeleteAccount: " + user + " not found.") - return - uidmin = None - try: - if os.path.isfile("/etc/login.defs"): - uidmin = int(GetLineStartingWith("UID_MIN", "/etc/login.defs").split()[1]) - except: - pass - if uidmin == None: - uidmin = 100 - if userentry[2] < uidmin: - Error("DeleteAccount: " + user + " is a system user. Will not delete account.") - return - Run("> /var/run/utmp") #Delete utmp to prevent error if we are the 'user' deleted - pid = subprocess.Popen(['rmuser', '-y', user], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE).pid - try: - os.remove(MyDistro.sudoers_dir_base+"/sudoers.d/waagent") - except: - pass - return - - def ActivateResourceDiskNoThread(self): - """ - Format, mount, and if specified in the configuration - set resource disk as swap. - """ - global DiskActivated - Run('cp /usr/sbin/waagent /tmp/') - SetFileContents('/tmp/bsd_activate_resource_disk.py',bsd_activate_resource_disk_txt) - Run('chmod +x /tmp/bsd_activate_resource_disk.py') - pid = subprocess.Popen(["/tmp/bsd_activate_resource_disk.py", ""]).pid - Log("Spawning bsd_activate_resource_disk.py") - DiskActivated = True - return - - def Install(self): - """ - Install the agent service. - Check dependencies. - Create /etc/waagent.conf and move old version to - /etc/waagent.conf.old - Copy RulesFiles to /var/lib/waagent - Create /etc/logrotate.d/waagent - Set /etc/ssh/sshd_config ClientAliveInterval to 180 - Call ApplyVNUMAWorkaround() - """ - if MyDistro.checkDependencies(): - return 1 - os.chmod(sys.argv[0], 0755) - SwitchCwd() - for a in RulesFiles: - if os.path.isfile(a): - if os.path.isfile(GetLastPathElement(a)): - os.remove(GetLastPathElement(a)) - shutil.move(a, ".") - Warn("Moved " + a + " -> " + LibDir + "/" + GetLastPathElement(a) ) - MyDistro.registerAgentService() - if os.path.isfile("/etc/waagent.conf"): - try: - os.remove("/etc/waagent.conf.old") - except: - pass - try: - os.rename("/etc/waagent.conf", "/etc/waagent.conf.old") - Warn("Existing /etc/waagent.conf has been renamed to /etc/waagent.conf.old") - except: - pass - SetFileContents("/etc/waagent.conf", self.waagent_conf_file) - if os.path.exists('/usr/local/etc/logrotate.d/'): - SetFileContents("/usr/local/etc/logrotate.d/waagent", WaagentLogrotate) - filepath = "/etc/ssh/sshd_config" - ReplaceFileContentsAtomic(filepath, "\n".join(filter(lambda a: not - a.startswith("ClientAliveInterval"), - GetFileContents(filepath).split('\n'))) + "\nClientAliveInterval 180\n") - Log("Configured SSH client probing to keep connections alive.") - #ApplyVNUMAWorkaround() - return 0 - - def mediaHasFilesystem(self,dsk): - if Run('LC_ALL=C fdisk -p ' + dsk + ' | grep "invalid fdisk partition table found" ',False): - return False - return True - - def mountDVD(self,dvd,location): - #At this point we cannot read a joliet option udf DVD in freebsd10 - so we 'dd' it into our location - retcode,out = RunGetOutput(self.mount_dvd_cmd + dvd + ' of=' + location + '/ovf-env.xml') - if retcode != 0: - return retcode,out - - ovfxml = (GetFileContents(location+"/ovf-env.xml",asbin=False)) - if ord(ovfxml[0]) > 128 and ord(ovfxml[1]) > 128 and ord(ovfxml[2]) > 128 : - ovfxml = ovfxml[3:] # BOM is not stripped. First three bytes are > 128 and not unicode chars so we ignore them. - ovfxml = ovfxml.strip(chr(0x00)) - ovfxml = "".join(filter(lambda x: ord(x)<128, ovfxml)) - ovfxml = re.sub(r'.*\Z','',ovfxml,0,re.DOTALL) - ovfxml += '' - SetFileContents(location+"/ovf-env.xml", ovfxml) - return retcode,out - - def GetHome(self): - return '/home' - - def initScsiDiskTimeout(self): - """ - Set the SCSI disk timeout by updating the kernal config - """ - timeout = Config.get("OS.RootDeviceScsiTimeout") - if timeout: - Run("sysctl kern.cam.da.default_timeout=" + timeout) - - def setScsiDiskTimeout(self): - return - - def setBlockDeviceTimeout(self, device, timeout): - return - - def getProcessorCores(self): - return int(RunGetOutput("sysctl hw.ncpu | awk '{print $2}'")[1]) - - def getTotalMemory(self): - return int(RunGetOutput("sysctl hw.realmem | awk '{print $2}'")[1])/1024 - -############################################################ -# END DISTRO CLASS DEFS -############################################################ - -# This lets us index into a string or an array of integers transparently. -def Ord(a): - """ - Allows indexing into a string or an array of integers transparently. - Generic utility function. - """ - if type(a) == type("a"): - a = ord(a) - return a - -def IsLinux(): - """ - Returns True if platform is Linux. - Generic utility function. - """ - return (platform.uname()[0] == "Linux") - -def GetLastPathElement(path): - """ - Similar to basename. - Generic utility function. - """ - return path.rsplit('/', 1)[1] - -def GetFileContents(filepath,asbin=False): - """ - Read and return contents of 'filepath'. - """ - mode='r' - if asbin: - mode+='b' - c=None - try: - with open(filepath, mode) as F : - c=F.read() - except IOError, e: - ErrorWithPrefix('GetFileContents','Reading from file ' + filepath + ' Exception is ' + str(e)) - return None - return c - -def SetFileContents(filepath, contents): - """ - Write 'contents' to 'filepath'. - """ - if type(contents) == str : - contents=contents.encode('latin-1', 'ignore') - try: - with open(filepath, "wb+") as F : - F.write(contents) - except IOError, e: - ErrorWithPrefix('SetFileContents','Writing to file ' + filepath + ' Exception is ' + str(e)) - return None - return 0 - -def AppendFileContents(filepath, contents): - """ - Append 'contents' to 'filepath'. - """ - if type(contents) == str : - contents=contents.encode('latin-1') - try: - with open(filepath, "a+") as F : - F.write(contents) - except IOError, e: - ErrorWithPrefix('AppendFileContents','Appending to file ' + filepath + ' Exception is ' + str(e)) - return None - return 0 - -def ReplaceFileContentsAtomic(filepath, contents): - """ - Write 'contents' to 'filepath' by creating a temp file, and replacing original. - """ - handle, temp = tempfile.mkstemp(dir = os.path.dirname(filepath)) - if type(contents) == str : - contents=contents.encode('latin-1') - try: - os.write(handle, contents) - except IOError, e: - ErrorWithPrefix('ReplaceFileContentsAtomic','Writing to file ' + filepath + ' Exception is ' + str(e)) - return None - finally: - os.close(handle) - try: - os.rename(temp, filepath) - return None - except IOError, e: - ErrorWithPrefix('ReplaceFileContentsAtomic','Renaming ' + temp+ ' to ' + filepath + ' Exception is ' + str(e)) - try: - os.remove(filepath) - except IOError, e: - ErrorWithPrefix('ReplaceFileContentsAtomic','Removing '+ filepath + ' Exception is ' + str(e)) - try: - os.rename(temp,filepath) - except IOError, e: - ErrorWithPrefix('ReplaceFileContentsAtomic','Removing '+ filepath + ' Exception is ' + str(e)) - return 1 - return 0 - -def GetLineStartingWith(prefix, filepath): - """ - Return line from 'filepath' if the line startswith 'prefix' - """ - for line in GetFileContents(filepath).split('\n'): - if line.startswith(prefix): - return line - return None - -def Run(cmd,chk_err=True): - """ - Calls RunGetOutput on 'cmd', returning only the return code. - If chk_err=True then errors will be reported in the log. - If chk_err=False then errors will be suppressed from the log. - """ - retcode,out=RunGetOutput(cmd,chk_err) - return retcode - -def RunGetOutput(cmd,chk_err=True): - """ - Wrapper for subprocess.check_output. - Execute 'cmd'. Returns return code and STDOUT, trapping expected exceptions. - Reports exceptions to Error if chk_err parameter is True - """ - LogIfVerbose(cmd) - try: - output=subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) - except subprocess.CalledProcessError,e : - if chk_err : - Error('CalledProcessError. Error Code is ' + str(e.returncode) ) - Error('CalledProcessError. Command string was ' + e.cmd ) - Error('CalledProcessError. Command result was ' + (e.output[:-1]).decode('latin-1')) - return e.returncode,e.output.decode('latin-1') - return 0,output.decode('latin-1') - -def RunSendStdin(cmd,input,chk_err=True): - """ - Wrapper for subprocess.Popen. - Execute 'cmd', sending 'input' to STDIN of 'cmd'. - Returns return code and STDOUT, trapping expected exceptions. - Reports exceptions to Error if chk_err parameter is True - """ - LogIfVerbose(cmd+input) - try: - me=subprocess.Popen([cmd], shell=True, stdin=subprocess.PIPE,stderr=subprocess.STDOUT,stdout=subprocess.PIPE) - output=me.communicate(input) - except OSError , e : - if chk_err : - Error('CalledProcessError. Error Code is ' + str(me.returncode) ) - Error('CalledProcessError. Command string was ' + cmd ) - Error('CalledProcessError. Command result was ' + output[0].decode('latin-1')) - return 1,output[0].decode('latin-1') - if me.returncode is not 0 and chk_err is True: - Error('CalledProcessError. Error Code is ' + str(me.returncode) ) - Error('CalledProcessError. Command string was ' + cmd ) - Error('CalledProcessError. Command result was ' + output[0].decode('latin-1')) - return me.returncode,output[0].decode('latin-1') - -def GetNodeTextData(a): - """ - Filter non-text nodes from DOM tree - """ - for b in a.childNodes: - if b.nodeType == b.TEXT_NODE: - return b.data - -def GetHome(): - """ - Attempt to guess the $HOME location. - Return the path string. - """ - home = None - try: - home = GetLineStartingWith("HOME", "/etc/default/useradd").split('=')[1].strip() - except: - pass - if (home == None) or (home.startswith("/") == False): - home = "/home" - return home - -def ChangeOwner(filepath, user): - """ - Lookup user. Attempt chown 'filepath' to 'user'. - """ - p = None - try: - p = pwd.getpwnam(user) - except: - pass - if p != None: - os.chown(filepath, p[2], p[3]) - -def CreateDir(dirpath, user, mode): - """ - Attempt os.makedirs, catch all exceptions. - Call ChangeOwner afterwards. - """ - try: - os.makedirs(dirpath, mode) - except: - pass - ChangeOwner(dirpath, user) - -def CreateAccount(user, password, expiration, thumbprint): - """ - Create a user account, with 'user', 'password', 'expiration', ssh keys - and sudo permissions. - Returns None if successful, error string on failure. - """ - userentry = None - try: - userentry = pwd.getpwnam(user) - except: - pass - uidmin = None - try: - uidmin = int(GetLineStartingWith("UID_MIN", "/etc/login.defs").split()[1]) - except: - pass - if uidmin == None: - uidmin = 100 - if userentry != None and userentry[2] < uidmin: - Error("CreateAccount: " + user + " is a system user. Will not set password.") - return "Failed to set password for system user: " + user + " (0x06)." - if userentry == None: - command = "useradd -m " + user - if expiration != None: - command += " -e " + expiration.split('.')[0] - if Run(command): - Error("Failed to create user account: " + user) - return "Failed to create user account: " + user + " (0x07)." - else: - Log("CreateAccount: " + user + " already exists. Will update password.") - if password != None: - RunSendStdin("chpasswd",(user + ":" + password + "\n")) - try: - # for older distros create sudoers.d - if not os.path.isdir('/etc/sudoers.d/'): - # create the /etc/sudoers.d/ directory - os.mkdir('/etc/sudoers.d/') - # add the include of sudoers.d to the /etc/sudoers - SetFileContents('/etc/sudoers',GetFileContents('/etc/sudoers')+'\n#includedir /etc/sudoers.d\n') - if password == None: - SetFileContents("/etc/sudoers.d/waagent", user + " ALL = (ALL) NOPASSWD: ALL\n") - else: - SetFileContents("/etc/sudoers.d/waagent", user + " ALL = (ALL) ALL\n") - os.chmod("/etc/sudoers.d/waagent", 0440) - except: - Error("CreateAccount: Failed to configure sudo access for user.") - return "Failed to configure sudo privileges (0x08)." - home = MyDistro.GetHome() - if thumbprint != None: - dir = home + "/" + user + "/.ssh" - CreateDir(dir, user, 0700) - pub = dir + "/id_rsa.pub" - prv = dir + "/id_rsa" - Run("ssh-keygen -y -f " + thumbprint + ".prv > " + pub) - SetFileContents(prv, GetFileContents(thumbprint + ".prv")) - for f in [pub, prv]: - os.chmod(f, 0600) - ChangeOwner(f, user) - SetFileContents(dir + "/authorized_keys", GetFileContents(pub)) - ChangeOwner(dir + "/authorized_keys", user) - Log("Created user account: " + user) - return None - -def DeleteAccount(user): - """ - Delete the 'user'. - Clear utmp first, to avoid error. - Removes the /etc/sudoers.d/waagent file. - """ - userentry = None - try: - userentry = pwd.getpwnam(user) - except: - pass - if userentry == None: - Error("DeleteAccount: " + user + " not found.") - return - uidmin = None - try: - uidmin = int(GetLineStartingWith("UID_MIN", "/etc/login.defs").split()[1]) - except: - pass - if uidmin == None: - uidmin = 100 - if userentry[2] < uidmin: - Error("DeleteAccount: " + user + " is a system user. Will not delete account.") - return - Run("> /var/run/utmp") #Delete utmp to prevent error if we are the 'user' deleted - Run("userdel -f -r " + user) - try: - os.remove("/etc/sudoers.d/waagent") - except: - pass - return - -def IsInRangeInclusive(a, low, high): - """ - Return True if 'a' in 'low' <= a >= 'high' - """ - return (a >= low and a <= high) - -def IsPrintable(ch): - """ - Return True if character is displayable. - """ - return IsInRangeInclusive(ch, Ord('A'), Ord('Z')) or IsInRangeInclusive(ch, Ord('a'), Ord('z')) or IsInRangeInclusive(ch, Ord('0'), Ord('9')) - -def HexDump(buffer, size): - """ - Return Hex formated dump of a 'buffer' of 'size'. - """ - if size < 0: - size = len(buffer) - result = "" - for i in range(0, size): - if (i % 16) == 0: - result += "%06X: " % i - byte = buffer[i] - if type(byte) == str: - byte = ord(byte.decode('latin1')) - result += "%02X " % byte - if (i & 15) == 7: - result += " " - if ((i + 1) % 16) == 0 or (i + 1) == size: - j = i - while ((j + 1) % 16) != 0: - result += " " - if (j & 7) == 7: - result += " " - j += 1 - result += " " - for j in range(i - (i % 16), i + 1): - byte=buffer[j] - if type(byte) == str: - byte = ord(byte.decode('latin1')) - k = '.' - if IsPrintable(byte): - k = chr(byte) - result += k - if (i + 1) != size: - result += "\n" - return result - -def SimpleLog(file_path,message): - if not file_path or len(message) < 1: - return - t = time.localtime() - t = "%04u/%02u/%02u %02u:%02u:%02u " % (t.tm_year, t.tm_mon, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec) - lines=re.sub(re.compile(r'^(.)',re.MULTILINE),t+r'\1',message) - with open(file_path, "a") as F : - lines = filter(lambda x : x in string.printable, lines) - F.write(lines.encode('ascii','ignore') + "\n") - -class Logger(object): - """ - The Agent's logging assumptions are: - For Log, and LogWithPrefix all messages are logged to the - self.file_path and to the self.con_path. Setting either path - parameter to None skips that log. If Verbose is enabled, messages - calling the LogIfVerbose method will be logged to file_path yet - not to con_path. Error and Warn messages are normal log messages - with the 'ERROR:' or 'WARNING:' prefix added. - """ - - def __init__(self,filepath,conpath,verbose=False): - """ - Construct an instance of Logger. - """ - self.file_path=filepath - self.con_path=conpath - self.verbose=verbose - - def ThrottleLog(self,counter): - """ - Log everything up to 10, every 10 up to 100, then every 100. - """ - return (counter < 10) or ((counter < 100) and ((counter % 10) == 0)) or ((counter % 100) == 0) - - def LogToFile(self,message): - """ - Write 'message' to logfile. - """ - if self.file_path: - try: - with open(self.file_path, "a") as F : - message = filter(lambda x : x in string.printable, message) - F.write(message.encode('ascii','ignore') + "\n") - except IOError, e: - print e - pass - - def LogToCon(self,message): - """ - Write 'message' to /dev/console. - This supports serial port logging if the /dev/console - is redirected to ttys0 in kernel boot options. - """ - if self.con_path: - try: - with open(self.con_path, "w") as C : - message = filter(lambda x : x in string.printable, message) - C.write(message.encode('ascii','ignore') + "\n") - except IOError, e: - print e - pass - - def Log(self,message): - """ - Standard Log function. - Logs to self.file_path, and con_path - """ - self.LogWithPrefix("", message) - - def LogWithPrefix(self,prefix, message): - """ - Prefix each line of 'message' with current time+'prefix'. - """ - t = time.localtime() - t = "%04u/%02u/%02u %02u:%02u:%02u " % (t.tm_year, t.tm_mon, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec) - t += prefix - for line in message.split('\n'): - line = t + line - self.LogToFile(line) - self.LogToCon(line) - - def NoLog(self,message): - """ - Don't Log. - """ - pass - - def LogIfVerbose(self,message): - """ - Only log 'message' if global Verbose is True. - """ - self.LogWithPrefixIfVerbose('',message) - - def LogWithPrefixIfVerbose(self,prefix, message): - """ - Only log 'message' if global Verbose is True. - Prefix each line of 'message' with current time+'prefix'. - """ - if self.verbose == True: - t = time.localtime() - t = "%04u/%02u/%02u %02u:%02u:%02u " % (t.tm_year, t.tm_mon, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec) - t += prefix - for line in message.split('\n'): - line = t + line - self.LogToFile(line) - self.LogToCon(line) - - def Warn(self,message): - """ - Prepend the text "WARNING:" to the prefix for each line in 'message'. - """ - self.LogWithPrefix("WARNING:", message) - - def Error(self,message): - """ - Call ErrorWithPrefix(message). - """ - ErrorWithPrefix("", message) - - def ErrorWithPrefix(self,prefix, message): - """ - Prepend the text "ERROR:" to the prefix for each line in 'message'. - Errors written to logfile, and /dev/console - """ - self.LogWithPrefix("ERROR:", message) - -def LoggerInit(log_file_path,log_con_path,verbose=False): - """ - Create log object and export its methods to global scope. - """ - global Log,LogWithPrefix,LogIfVerbose,LogWithPrefixIfVerbose,Error,ErrorWithPrefix,Warn,NoLog,ThrottleLog,myLogger - l=Logger(log_file_path,log_con_path,verbose) - Log,LogWithPrefix,LogIfVerbose,LogWithPrefixIfVerbose,Error,ErrorWithPrefix,Warn,NoLog,ThrottleLog,myLogger = l.Log,l.LogWithPrefix,l.LogIfVerbose,l.LogWithPrefixIfVerbose,l.Error,l.ErrorWithPrefix,l.Warn,l.NoLog,l.ThrottleLog,l - -def Linux_ioctl_GetInterfaceMac(ifname): - """ - Return the mac-address bound to the socket. - """ - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - info = fcntl.ioctl(s.fileno(), 0x8927, struct.pack('256s', (ifname[:15]+('\0'*241)).encode('latin-1'))) - return ''.join(['%02X' % Ord(char) for char in info[18:24]]) - -def GetFirstActiveNetworkInterfaceNonLoopback(): - """ - Return the interface name, and ip addr of the - first active non-loopback interface. - """ - iface='' - expected=16 # how many devices should I expect... - is_64bits = sys.maxsize > 2**32 - struct_size=40 if is_64bits else 32 # for 64bit the size is 40 bytes, for 32bits it is 32 bytes. - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - buff=array.array('B', b'\0' * (expected*struct_size)) - retsize=(struct.unpack('iL', fcntl.ioctl(s.fileno(), 0x8912, struct.pack('iL',expected*struct_size,buff.buffer_info()[0]))))[0] - if retsize == (expected*struct_size) : - Warn('SIOCGIFCONF returned more than ' + str(expected) + ' up network interfaces.') - s=buff.tostring() - preferred_nic = Config.get("Network.Interface") - for i in range(0,struct_size*expected,struct_size): - iface=s[i:i+16].split(b'\0', 1)[0] - if iface == b'lo': - continue - elif preferred_nic is None: - break - elif iface == preferred_nic: - break - return iface.decode('latin-1'), socket.inet_ntoa(s[i+20:i+24]) - -def GetIpv4Address(): - """ - Return the ip of the - first active non-loopback interface. - """ - iface,addr=GetFirstActiveNetworkInterfaceNonLoopback() - return addr - -def HexStringToByteArray(a): - """ - Return hex string packed into a binary struct. - """ - b = b"" - for c in range(0, len(a) // 2): - b += struct.pack("B", int(a[c * 2:c * 2 + 2], 16)) - return b - -def GetMacAddress(): - """ - Convienience function, returns mac addr bound to - first non-loobback interface. - """ - ifname='' - while len(ifname) < 2 : - ifname=GetFirstActiveNetworkInterfaceNonLoopback()[0] - a = Linux_ioctl_GetInterfaceMac(ifname) - return HexStringToByteArray(a) - -def DeviceForIdePort(n): - """ - Return device name attached to ide port 'n'. - """ - if n > 3: - return None - g0 = "00000000" - if n > 1: - g0 = "00000001" - n = n - 2 - device = None - path = "/sys/bus/vmbus/devices/" - for vmbus in os.listdir(path): - guid = GetFileContents(path + vmbus + "/device_id").lstrip('{').split('-') - if guid[0] == g0 and guid[1] == "000" + str(n): - for root, dirs, files in os.walk(path + vmbus): - if root.endswith("/block"): - device = dirs[0] - break - else : #older distros - for d in dirs: - if ':' in d and "block" == d.split(':')[0]: - device = d.split(':')[1] - break - break - return device - -class HttpResourceGoneError(Exception): - pass - -class Util(object): - """ - Http communication class. - Base of GoalState, and Agent classes. - """ - RetryWaitingInterval=10 - - def __init__(self): - self.Endpoint = None - - def _ParseUrl(self, url): - secure = False - host = self.Endpoint - path = url - port = None - - #"http[s]://hostname[:port][/]" - if url.startswith("http://"): - url = url[7:] - if "/" in url: - host = url[0: url.index("/")] - path = url[url.index("/"):] - else: - host = url - path = "/" - elif url.startswith("https://"): - secure = True - url = url[8:] - if "/" in url: - host = url[0: url.index("/")] - path = url[url.index("/"):] - else: - host = url - path = "/" - - if host is None: - raise ValueError("Host is invalid:{0}".format(url)) - - if(":" in host): - pos = host.rfind(":") - port = int(host[pos + 1:]) - host = host[0:pos] - - return host, port, secure, path - - def GetHttpProxy(self, secure): - """ - Get http_proxy and https_proxy from environment variables. - Username and password is not supported now. - """ - host = Config.get("HttpProxy.Host") - port = Config.get("HttpProxy.Port") - return (host, port) - - def _HttpRequest(self, method, host, path, port=None, data=None, secure=False, - headers=None, proxyHost=None, proxyPort=None): - resp = None - conn = None - try: - if secure: - port = 443 if port is None else port - if proxyHost is not None and proxyPort is not None: - conn = httplib.HTTPSConnection(proxyHost, proxyPort) - conn.set_tunnel(host, port) - #If proxy is used, full url is needed. - path = "https://{0}:{1}{2}".format(host, port, path) - else: - conn = httplib.HTTPSConnection(host, port) - else: - port = 80 if port is None else port - if proxyHost is not None and proxyPort is not None: - conn = httplib.HTTPConnection(proxyHost, proxyPort) - #If proxy is used, full url is needed. - path = "http://{0}:{1}{2}".format(host, port, path) - else: - conn = httplib.HTTPConnection(host, port) - if headers == None: - conn.request(method, path, data) - else: - conn.request(method, path, data, headers) - resp = conn.getresponse() - except httplib.HTTPException, e: - Error('HTTPException {0}, args:{1}'.format(e, repr(e.args))) - except IOError, e: - Error('Socket IOError {0}, args:{1}'.format(e, repr(e.args))) - return resp - - def HttpRequest(self, method, url, data=None, - headers=None, maxRetry=3, chkProxy=False): - """ - Sending http request to server - On error, sleep 10 and maxRetry times. - Return the output buffer or None. - """ - LogIfVerbose("HTTP Req: {0} {1}".format(method, url)) - LogIfVerbose("HTTP Req: Data={0}".format(data)) - LogIfVerbose("HTTP Req: Header={0}".format(headers)) - try: - host, port, secure, path = self._ParseUrl(url) - except ValueError, e: - Error("Failed to parse url:{0}".format(url)) - return None - - #Check proxy - proxyHost, proxyPort = (None, None) - if chkProxy: - proxyHost, proxyPort = self.GetHttpProxy(secure) - - #If httplib module is not built with ssl support. Fallback to http - if secure and not hasattr(httplib, "HTTPSConnection"): - Warn("httplib is not built with ssl support") - secure = False - proxyHost, proxyPort = self.GetHttpProxy(secure) - - #If httplib module doesn't support https tunnelling. Fallback to http - if secure and \ - proxyHost is not None and \ - proxyPort is not None and \ - not hasattr(httplib.HTTPSConnection, "set_tunnel"): - Warn("httplib doesn't support https tunnelling(new in python 2.7)") - secure = False - proxyHost, proxyPort = self.GetHttpProxy(secure) - - resp = self._HttpRequest(method, host, path, port=port, data=data, - secure=secure, headers=headers, - proxyHost=proxyHost, proxyPort=proxyPort) - for retry in range(0, maxRetry): - if resp is not None and \ - (resp.status == httplib.OK or \ - resp.status == httplib.CREATED or \ - resp.status == httplib.ACCEPTED): - return resp; - - if resp is not None and resp.status == httplib.GONE: - raise HttpResourceGoneError("Http resource gone.") - - Error("Retry={0}".format(retry)) - Error("HTTP Req: {0} {1}".format(method, url)) - Error("HTTP Req: Data={0}".format(data)) - Error("HTTP Req: Header={0}".format(headers)) - if resp is None: - Error("HTTP Err: response is empty.".format(retry)) - else: - Error("HTTP Err: Status={0}".format(resp.status)) - Error("HTTP Err: Reason={0}".format(resp.reason)) - Error("HTTP Err: Header={0}".format(resp.getheaders())) - Error("HTTP Err: Body={0}".format(resp.read())) - - time.sleep(self.__class__.RetryWaitingInterval) - resp = self._HttpRequest(method, host, path, port=port, data=data, - secure=secure, headers=headers, - proxyHost=proxyHost, proxyPort=proxyPort) - - return None - - def HttpGet(self, url, headers=None, maxRetry=3, chkProxy=False): - return self.HttpRequest("GET", url, headers=headers, - maxRetry=maxRetry, chkProxy=chkProxy) - - def HttpHead(self, url, headers=None, maxRetry=3, chkProxy=False): - return self.HttpRequest("HEAD", url, headers=headers, - maxRetry=maxRetry, chkProxy=chkProxy) - - def HttpPost(self, url, data, headers=None, maxRetry=3, chkProxy=False): - return self.HttpRequest("POST", url, data=data, headers=headers, - maxRetry=maxRetry, chkProxy=chkProxy) - - def HttpPut(self, url, data, headers=None, maxRetry=3, chkProxy=False): - return self.HttpRequest("PUT", url, data=data, headers=headers, - maxRetry=maxRetry, chkProxy=chkProxy) - - def HttpDelete(self, url, headers=None, maxRetry=3, chkProxy=False): - return self.HttpRequest("DELETE", url, headers=headers, - maxRetry=maxRetry, chkProxy=chkProxy) - - def HttpGetWithoutHeaders(self, url, maxRetry=3, chkProxy=False): - """ - Return data from an HTTP get on 'url'. - """ - resp = self.HttpGet(url, headers=None, maxRetry=maxRetry, - chkProxy=chkProxy) - return resp.read() if resp is not None else None - - def HttpGetWithHeaders(self, url, maxRetry=3, chkProxy=False): - """ - Return data from an HTTP get on 'url' with - x-ms-agent-name and x-ms-version - headers. - """ - resp = self.HttpGet(url, headers={ - "x-ms-agent-name": GuestAgentName, - "x-ms-version": ProtocolVersion - }, maxRetry=maxRetry, chkProxy=chkProxy) - return resp.read() if resp is not None else None - - def HttpSecureGetWithHeaders(self, url, transportCert, maxRetry=3, - chkProxy=False): - """ - Return output of get using ssl cert. - """ - resp = self.HttpGet(url, headers={ - "x-ms-agent-name": GuestAgentName, - "x-ms-version": ProtocolVersion, - "x-ms-cipher-name": "DES_EDE3_CBC", - "x-ms-guest-agent-public-x509-cert": transportCert - }, maxRetry=maxRetry, chkProxy=chkProxy) - return resp.read() if resp is not None else None - - def HttpPostWithHeaders(self, url, data, maxRetry=3, chkProxy=False): - headers = { - "x-ms-agent-name": GuestAgentName, - "Content-Type": "text/xml; charset=utf-8", - "x-ms-version": ProtocolVersion - } - return self.HttpPost(url, data=data, headers=headers, - maxRetry=maxRetry, chkProxy=chkProxy) - -__StorageVersion="2014-02-14" - -def GetBlobType(url): - restutil = Util() - #Check blob type - LogIfVerbose("Check blob type.") - timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - blobPropResp = restutil.HttpHead(url, { - "x-ms-date" : timestamp, - 'x-ms-version' : __StorageVersion - }, chkProxy=True); - blobType = None - if blobPropResp is None: - Error("Can't get status blob type.") - return None - blobType = blobPropResp.getheader("x-ms-blob-type") - LogIfVerbose("Blob type={0}".format(blobType)) - return blobType - -def PutBlockBlob(url, data): - restutil = Util() - LogIfVerbose("Upload block blob") - timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - ret = restutil.HttpPut(url, data, { - "x-ms-date" : timestamp, - "x-ms-blob-type" : "BlockBlob", - "Content-Length": str(len(data)), - "x-ms-version" : __StorageVersion - }, chkProxy=True) - if ret is None: - Error("Failed to upload block blob for status.") - -def PutPageBlob(url, data): - restutil = Util() - LogIfVerbose("Replace old page blob") - timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - #Align to 512 bytes - pageBlobSize = ((len(data) + 511) / 512) * 512 - ret = restutil.HttpPut(url, "", { - "x-ms-date" : timestamp, - "x-ms-blob-type" : "PageBlob", - "Content-Length": "0", - "x-ms-blob-content-length" : str(pageBlobSize), - "x-ms-version" : __StorageVersion - }, chkProxy=True) - if ret is None: - Error("Failed to clean up page blob for status") - return - - if url.index('?') < 0: - url = "{0}?comp=page".format(url) - else: - url = "{0}&comp=page".format(url) - - LogIfVerbose("Upload page blob") - pageMax = 4 * 1024 * 1024 #Max page size: 4MB - start = 0 - end = 0 - while end < len(data): - end = min(len(data), start + pageMax) - contentSize = end - start - #Align to 512 bytes - pageEnd = ((end + 511) / 512) * 512 - bufSize = pageEnd - start - buf = bytearray(bufSize) - buf[0 : contentSize] = data[start : end] - ret = restutil.HttpPut(url, buffer(buf), { - "x-ms-date" : timestamp, - "x-ms-range" : "bytes={0}-{1}".format(start, pageEnd - 1), - "x-ms-page-write" : "update", - "x-ms-version" : __StorageVersion, - "Content-Length": str(pageEnd - start) - }, chkProxy=True) - if ret is None: - Error("Failed to upload page blob for status") - return - start = end - -def UploadStatusBlob(url, data): - LogIfVerbose("Upload status blob") - LogIfVerbose("Status={0}".format(data)) - blobType = GetBlobType(url) - - if blobType == "BlockBlob": - PutBlockBlob(url, data) - elif blobType == "PageBlob": - PutPageBlob(url, data) - else: - Error("Unknown blob type: {0}".format(blobType)) - return None - -class TCPHandler(SocketServer.BaseRequestHandler): - """ - Callback object for LoadBalancerProbeServer. - Recv and send LB probe messages. - """ - def __init__(self,lb_probe): - super(TCPHandler,self).__init__() - self.lb_probe=lb_probe - - def GetHttpDateTimeNow(self): - """ - Return formatted gmtime "Date: Fri, 25 Mar 2011 04:53:10 GMT" - """ - return time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) - - def handle(self): - """ - Log LB probe messages, read the socket buffer, - send LB probe response back to server. - """ - self.lb_probe.ProbeCounter = (self.lb_probe.ProbeCounter + 1) % 1000000 - log = [NoLog, LogIfVerbose][ThrottleLog(self.lb_probe.ProbeCounter)] - strCounter = str(self.lb_probe.ProbeCounter) - if self.lb_probe.ProbeCounter == 1: - Log("Receiving LB probes.") - log("Received LB probe # " + strCounter) - self.request.recv(1024) - self.request.send("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nContent-Type: text/html\r\nDate: " + self.GetHttpDateTimeNow() + "\r\n\r\nOK") - -class LoadBalancerProbeServer(object): - """ - Threaded object to receive and send LB probe messages. - Load Balancer messages but be recv'd by - the load balancing server, or this node may be shut-down. - """ - def __init__(self, port): - self.ProbeCounter = 0 - self.server = SocketServer.TCPServer((self.get_ip(), port), TCPHandler) - self.server_thread = threading.Thread(target = self.server.serve_forever) - self.server_thread.setDaemon(True) - self.server_thread.start() - - def shutdown(self): - self.server.shutdown() - - def get_ip(self): - for retry in range(1,6): - ip = MyDistro.GetIpv4Address() - if ip == None : - Log("LoadBalancerProbeServer: GetIpv4Address() returned None, sleeping 10 before retry " + str(retry+1) ) - time.sleep(10) - else: - return ip - -class ConfigurationProvider(object): - """ - Parse amd store key:values in waagent.conf - """ - def __init__(self, walaConfigFile): - self.values = dict() - if 'MyDistro' not in globals(): - global MyDistro - MyDistro = GetMyDistro() - if walaConfigFile is None: - walaConfigFile = MyDistro.getConfigurationPath() - if os.path.isfile(walaConfigFile) == False: - raise Exception("Missing configuration in {0}".format(walaConfigFile)) - try: - for line in GetFileContents(walaConfigFile).split('\n'): - if not line.startswith("#") and "=" in line: - parts = line.split()[0].split('=') - value = parts[1].strip("\" ") - if value != "None": - self.values[parts[0]] = value - else: - self.values[parts[0]] = None - except: - Error("Unable to parse {0}".format(walaConfigFile)) - raise - return - - def get(self, key): - return self.values.get(key) - -class EnvMonitor(object): - """ - Montor changes to dhcp and hostname. - If dhcp clinet process re-start has occurred, reset routes, dhcp with fabric. - """ - def __init__(self): - self.shutdown = False - self.HostName = socket.gethostname() - self.server_thread = threading.Thread(target = self.monitor) - self.server_thread.setDaemon(True) - self.server_thread.start() - self.published = False - - def monitor(self): - """ - Monitor dhcp client pid and hostname. - If dhcp clinet process re-start has occurred, reset routes, dhcp with fabric. - """ - publish = Config.get("Provisioning.MonitorHostName") - dhcpcmd = MyDistro.getpidcmd+ ' ' + MyDistro.getDhcpClientName() - dhcppid = RunGetOutput(dhcpcmd)[1] - while not self.shutdown: - for a in RulesFiles: - if os.path.isfile(a): - if os.path.isfile(GetLastPathElement(a)): - os.remove(GetLastPathElement(a)) - shutil.move(a, ".") - Log("EnvMonitor: Moved " + a + " -> " + LibDir) - MyDistro.setScsiDiskTimeout() - if publish != None and publish.lower().startswith("y"): - try: - if socket.gethostname() != self.HostName: - Log("EnvMonitor: Detected host name change: " + self.HostName + " -> " + socket.gethostname()) - self.HostName = socket.gethostname() - WaAgent.UpdateAndPublishHostName(self.HostName) - dhcppid = RunGetOutput(dhcpcmd)[1] - self.published = True - except: - pass - else: - self.published = True - pid = "" - if not os.path.isdir("/proc/" + dhcppid.strip()): - pid = RunGetOutput(dhcpcmd)[1] - if pid != "" and pid != dhcppid: - Log("EnvMonitor: Detected dhcp client restart. Restoring routing table.") - WaAgent.RestoreRoutes() - dhcppid = pid - for child in Children: - if child.poll() != None: - Children.remove(child) - time.sleep(5) - - def SetHostName(self, name): - """ - Generic call to MyDistro.setHostname(name). - Complian to Log on error. - """ - if socket.gethostname() == name: - self.published = True - elif MyDistro.setHostname(name): - Error("Error: SetHostName: Cannot set hostname to " + name) - return ("Error: SetHostName: Cannot set hostname to " + name) - - def IsHostnamePublished(self): - """ - Return self.published - """ - return self.published - - def ShutdownService(self): - """ - Stop server comminucation and join the thread to main thread. - """ - self.shutdown = True - self.server_thread.join() - -class Certificates(object): - """ - Object containing certificates of host and provisioned user. - Parses and splits certificates into files. - """ - # - # 2010-12-15 - # 2 - # Pkcs7BlobWithPfxContents - # MIILTAY... - # - # - - def __init__(self): - self.reinitialize() - - def reinitialize(self): - """ - Reset the Role, Incarnation - """ - self.Incarnation = None - self.Role = None - - def Parse(self, xmlText): - """ - Parse multiple certificates into seperate files. - """ - self.reinitialize() - SetFileContents("Certificates.xml", xmlText) - dom = xml.dom.minidom.parseString(xmlText) - for a in [ "CertificateFile", "Version", "Incarnation", - "Format", "Data", ]: - if not dom.getElementsByTagName(a): - Error("Certificates.Parse: Missing " + a) - return None - node = dom.childNodes[0] - if node.localName != "CertificateFile": - Error("Certificates.Parse: root not CertificateFile") - return None - SetFileContents("Certificates.p7m", - "MIME-Version: 1.0\n" - + "Content-Disposition: attachment; filename=\"Certificates.p7m\"\n" - + "Content-Type: application/x-pkcs7-mime; name=\"Certificates.p7m\"\n" - + "Content-Transfer-Encoding: base64\n\n" - + GetNodeTextData(dom.getElementsByTagName("Data")[0])) - if Run(Openssl + " cms -decrypt -in Certificates.p7m -inkey TransportPrivate.pem -recip TransportCert.pem | " + Openssl + " pkcs12 -nodes -password pass: -out Certificates.pem"): - Error("Certificates.Parse: Failed to extract certificates from CMS message.") - return self - # There may be multiple certificates in this package. Split them. - file = open("Certificates.pem") - pindex = 1 - cindex = 1 - output = open("temp.pem", "w") - for line in file.readlines(): - output.write(line) - if re.match(r'[-]+END .*?(KEY|CERTIFICATE)[-]+$',line): - output.close() - if re.match(r'[-]+END .*?KEY[-]+$',line): - os.rename("temp.pem", str(pindex) + ".prv") - pindex += 1 - else: - os.rename("temp.pem", str(cindex) + ".crt") - cindex += 1 - output = open("temp.pem", "w") - output.close() - os.remove("temp.pem") - keys = dict() - index = 1 - filename = str(index) + ".crt" - while os.path.isfile(filename): - thumbprint = (RunGetOutput(Openssl + " x509 -in " + filename + " -fingerprint -noout")[1]).rstrip().split('=')[1].replace(':', '').upper() - pubkey=RunGetOutput(Openssl + " x509 -in " + filename + " -pubkey -noout")[1] - keys[pubkey] = thumbprint - os.rename(filename, thumbprint + ".crt") - os.chmod(thumbprint + ".crt", 0600) - MyDistro.setSelinuxContext(thumbprint + '.crt','unconfined_u:object_r:ssh_home_t:s0') - index += 1 - filename = str(index) + ".crt" - index = 1 - filename = str(index) + ".prv" - while os.path.isfile(filename): - pubkey = RunGetOutput(Openssl + " rsa -in " + filename + " -pubout 2> /dev/null ")[1] - os.rename(filename, keys[pubkey] + ".prv") - os.chmod(keys[pubkey] + ".prv", 0600) - MyDistro.setSelinuxContext( keys[pubkey] + '.prv','unconfined_u:object_r:ssh_home_t:s0') - index += 1 - filename = str(index) + ".prv" - return self - -class SharedConfig(object): - """ - Parse role endpoint server and goal state config. - """ - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - def __init__(self): - self.reinitialize() - - def reinitialize(self): - """ - Reset members. - """ - self.RdmaMacAddress = None - self.RdmaIPv4Address = None - self.xmlText = None - - def Parse(self, xmlText): - """ - Parse and write configuration to file SharedConfig.xml. - """ - LogIfVerbose(xmlText) - self.reinitialize() - self.xmlText = xmlText - dom = xml.dom.minidom.parseString(xmlText) - for a in [ "SharedConfig", "Deployment", "Service", - "ServiceInstance", "Incarnation", "Role", ]: - if not dom.getElementsByTagName(a): - Error("SharedConfig.Parse: Missing " + a) - - node = dom.childNodes[0] - if node.localName != "SharedConfig": - Error("SharedConfig.Parse: root not SharedConfig") - - nodes = dom.getElementsByTagName("Instance") - if nodes is not None and len(nodes) != 0: - node = nodes[0] - if node.hasAttribute("rdmaMacAddress"): - addr = node.getAttribute("rdmaMacAddress") - self.RdmaMacAddress = addr[0:2] - for i in range(1, 6): - self.RdmaMacAddress += ":" + addr[2 * i : 2 *i + 2] - if node.hasAttribute("rdmaIPv4Address"): - self.RdmaIPv4Address = node.getAttribute("rdmaIPv4Address") - return self - - def Save(self): - LogIfVerbose("Save SharedConfig.xml") - SetFileContents("SharedConfig.xml", self.xmlText) - - def InvokeTopologyConsumer(self): - program = Config.get("Role.TopologyConsumer") - if program != None: - try: - Children.append(subprocess.Popen([program, LibDir + "/SharedConfig.xml"])) - except OSError, e : - ErrorWithPrefix('Agent.Run','Exception: '+ str(e) +' occured launching ' + program ) - - def Process(self): - global rdma_configured - if not rdma_configured and self.RdmaMacAddress is not None and self.RdmaIPv4Address is not None: - handler = RdmaHandler(self.RdmaMacAddress, self.RdmaIPv4Address) - handler.start() - rdma_configured = True - self.InvokeTopologyConsumer() - -rdma_configured = False - -class RdmaError(Exception): - pass - -class RdmaHandler(object): - """ - Handle rdma configuration. - """ - - def __init__(self, mac, ip_addr, dev="/dev/hvnd_rdma", - dat_conf_files=['/etc/dat.conf', '/etc/rdma/dat.conf', - '/usr/local/etc/dat.conf']): - self.mac = mac - self.ip_addr = ip_addr - self.dev = dev - self.dat_conf_files = dat_conf_files - self.data = ('rdmaMacAddress="{0}" rdmaIPv4Address="{1}"' - '').format(self.mac, self.ip_addr) - - def start(self): - """ - Start a new thread to process rdma - """ - threading.Thread(target=self.process).start() - - def process(self): - try: - self.set_dat_conf() - self.set_rdma_dev() - self.set_rdma_ip() - except RdmaError as e: - Error("Failed to config rdma device: {0}".format(e)) - - def set_dat_conf(self): - """ - Agent needs to search all possible locations for dat.conf - """ - Log("Set dat.conf") - for dat_conf_file in self.dat_conf_files: - if not os.path.isfile(dat_conf_file): - continue - try: - self.write_dat_conf(dat_conf_file) - except IOError as e: - raise RdmaError("Failed to write to dat.conf: {0}".format(e)) - - def write_dat_conf(self, dat_conf_file): - Log("Write config to {0}".format(dat_conf_file)) - old = ("ofa-v2-ib0 u2.0 nonthreadsafe default libdaplofa.so.2 " - "dapl.2.0 \"\S+ 0\"") - new = ("ofa-v2-ib0 u2.0 nonthreadsafe default libdaplofa.so.2 " - "dapl.2.0 \"{0} 0\"").format(self.ip_addr) - lines = GetFileContents(dat_conf_file) - lines = re.sub(old, new, lines) - SetFileContents(dat_conf_file, lines) - - def set_rdma_dev(self): - """ - Write config string to /dev/hvnd_rdma - """ - Log("Set /dev/hvnd_rdma") - self.wait_rdma_dev() - self.write_rdma_dev_conf() - - def write_rdma_dev_conf(self): - Log("Write rdma config to {0}: {1}".format(self.dev, self.data)) - try: - with open(self.dev, "w") as c: - c.write(self.data) - except IOError, e: - raise RdmaError("Error writing {0}, {1}".format(self.dev, e)) - - def wait_rdma_dev(self): - Log("Wait for /dev/hvnd_rdma") - retry = 0 - while retry < 120: - if os.path.exists(self.dev): - return - time.sleep(1) - retry += 1 - raise RdmaError("The device doesn't show up in 120 seconds") - - def set_rdma_ip(self): - Log("Set ip addr for rdma") - try: - if_name = MyDistro.getInterfaceNameByMac(self.mac) - #Azure is using 12 bits network mask for infiniband. - MyDistro.configIpV4(if_name, self.ip_addr, 12) - except Exception as e: - raise RdmaError("Failed to config rdma device: {0}".format(e)) - -class ExtensionsConfig(object): - """ - Parse ExtensionsConfig, downloading and unpacking them to /var/lib/waagent. - Install if true, remove if it is set to false. - """ - # - # - # - # - # - # - # {"runtimeSettings":[{"handlerSettings":{"protectedSettingsCertThumbprint":"1BE9A13AA1321C7C515EF109746998BAB6D86FD1", - #"protectedSettings":"MIIByAYJKoZIhvcNAQcDoIIBuTCCAbUCAQAxggFxMIIBbQIBADBVMEExPzA9BgoJkiaJk/IsZAEZFi9XaW5kb3dzIEF6dXJlIFNlcnZpY2UgTWFuYWdlbWVudCBmb3IgR - #Xh0ZW5zaW9ucwIQZi7dw+nhc6VHQTQpCiiV2zANBgkqhkiG9w0BAQEFAASCAQCKr09QKMGhwYe+O4/a8td+vpB4eTR+BQso84cV5KCAnD6iUIMcSYTrn9aveY6v6ykRLEw8GRKfri2d6 - #tvVDggUrBqDwIgzejGTlCstcMJItWa8Je8gHZVSDfoN80AEOTws9Fp+wNXAbSuMJNb8EnpkpvigAWU2v6pGLEFvSKC0MCjDTkjpjqciGMcbe/r85RG3Zo21HLl0xNOpjDs/qqikc/ri43Y76E/X - #v1vBSHEGMFprPy/Hwo3PqZCnulcbVzNnaXN3qi/kxV897xGMPPC3IrO7Nc++AT9qRLFI0841JLcLTlnoVG1okPzK9w6ttksDQmKBSHt3mfYV+skqs+EOMDsGCSqGSIb3DQEHATAUBggqh - #kiG9w0DBwQITgu0Nu3iFPuAGD6/QzKdtrnCI5425fIUy7LtpXJGmpWDUA==","publicSettings":{"port":"3000"}}}]} - # - # - #https://ostcextensions.blob.core.test-cint.azure-test.net/vhds/eg-plugin7-vm.eg-plugin7-vm.eg-plugin7-vm.status?sr=b&sp=rw& - #se=9999-01-01&sk=key1&sv=2012-02-12&sig=wRUIDN1x2GC06FWaetBP9sjjifOWvRzS2y2XBB4qoBU%3D - - def __init__(self): - self.reinitialize() - - def reinitialize(self): - """ - Reset members. - """ - self.Extensions = None - self.Plugins = None - self.Util = None - - def Parse(self, xmlText): - """ - Write configuration to file ExtensionsConfig.xml. - Log plugin specific activity to /var/log/azure/.//CommandExecution.log. - If state is enabled: - if the plugin is installed: - if the new plugin's version is higher - if DisallowMajorVersionUpgrade is false or if true, the version is a minor version do upgrade: - download the new archive - do the updateCommand. - disable the old plugin and remove - enable the new plugin - if the new plugin's version is the same or lower: - create the new .settings file from the configuration received - do the enableCommand - if the plugin is not installed: - download/unpack archive and call the installCommand/Enable - if state is disabled: - call disableCommand - if state is uninstall: - call uninstallCommand - remove old plugin directory. - """ - self.reinitialize() - self.Util=Util() - dom = xml.dom.minidom.parseString(xmlText) - LogIfVerbose(xmlText) - self.plugin_log_dir='/var/log/azure' - if not os.path.exists(self.plugin_log_dir): - os.mkdir(self.plugin_log_dir) - try: - self.Extensions=dom.getElementsByTagName("Extensions") - pg = dom.getElementsByTagName("Plugins") - if len(pg) > 0: - self.Plugins = pg[0].getElementsByTagName("Plugin") - else: - self.Plugins = [] - incarnation=self.Extensions[0].getAttribute("goalStateIncarnation") - SetFileContents('ExtensionsConfig.'+incarnation+'.xml', xmlText) - except Exception, e: - Error('ERROR: Error parsing ExtensionsConfig: {0}.'.format(e)) - return None - for p in self.Plugins: - if len(p.getAttribute("location"))<1: # this plugin is inside the PluginSettings - continue - p.setAttribute('restricted','false') - previous_version = None - version=p.getAttribute("version") - name=p.getAttribute("name") - plog_dir=self.plugin_log_dir+'/'+name +'/'+ version - if not os.path.exists(plog_dir): - os.makedirs(plog_dir) - p.plugin_log=plog_dir+'/CommandExecution.log' - handler=name + '-' + version - if p.getAttribute("isJson") != 'true': - Error("Plugin " + name+" version: " +version+" is not a JSON Extension. Skipping.") - continue - Log("Found Plugin: " + name + ' version: ' + version) - if p.getAttribute("state") == 'disabled' or p.getAttribute("state") == 'uninstall': - #disable - zip_dir=LibDir+"/" + name + '-' + version - mfile=None - for root, dirs, files in os.walk(zip_dir): - for f in files: - if f in ('HandlerManifest.json'): - mfile=os.path.join(root,f) - if mfile != None: - break - if mfile == None : - Error('HandlerManifest.json not found.') - continue - manifest = GetFileContents(mfile) - p.setAttribute('manifestdata',manifest) - if self.launchCommand(p.plugin_log,name,version,'disableCommand') == None : - self.SetHandlerState(handler, 'Enabled') - Error('Unable to disable '+name) - SimpleLog(p.plugin_log,'ERROR: Unable to disable '+name) - else : - self.SetHandlerState(handler, 'Disabled') - Log(name+' is disabled') - SimpleLog(p.plugin_log,name+' is disabled') - - # uninstall if needed - if p.getAttribute("state") == 'uninstall': - if self.launchCommand(p.plugin_log,name,version,'uninstallCommand') == None : - self.SetHandlerState(handler, 'Installed') - Error('Unable to uninstall '+name) - SimpleLog(p.plugin_log,'Unable to uninstall '+name) - else : - self.SetHandlerState(handler, 'NotInstalled') - Log(name+' uninstallCommand completed .') - # remove the plugin - Run('rm -rf ' + LibDir + '/' + name +'-'+ version + '*') - Log(name +'-'+ version + ' extension files deleted.') - SimpleLog(p.plugin_log,name +'-'+ version + ' extension files deleted.') - - continue - # state is enabled - # if the same plugin exists and the version is newer or - # does not exist then download and unzip the new plugin - plg_dir=None - for root, dirs, files in os.walk(LibDir): - for d in dirs: - if name in d: - plg_dir=os.path.join(root,d) - if plg_dir != None: - break - if plg_dir != None : - previous_version=plg_dir.rsplit('-')[-1] - if plg_dir == None or version > previous_version : - location=p.getAttribute("location") - Log("Downloading plugin manifest: " + name + " from " + location) - SimpleLog(p.plugin_log,"Downloading plugin manifest: " + name + " from " + location) - - self.Util.Endpoint=location.split('/')[2] - Log("Plugin server is: " + self.Util.Endpoint) - SimpleLog(p.plugin_log,"Plugin server is: " + self.Util.Endpoint) - - manifest=self.Util.HttpGetWithoutHeaders(location, chkProxy=True) - if manifest == None: - Error("Unable to download plugin manifest" + name + " from primary location. Attempting with failover location.") - SimpleLog(p.plugin_log,"Unable to download plugin manifest" + name + " from primary location. Attempting with failover location.") - failoverlocation=p.getAttribute("failoverlocation") - self.Util.Endpoint=failoverlocation.split('/')[2] - Log("Plugin failover server is: " + self.Util.Endpoint) - SimpleLog(p.plugin_log,"Plugin failover server is: " + self.Util.Endpoint) - - manifest=self.Util.HttpGetWithoutHeaders(failoverlocation, chkProxy=True) - #if failoverlocation also fail what to do then? - if manifest == None: - AddExtensionEvent(name,WALAEventOperation.Download,False,0,version,"Download mainfest fail "+failoverlocation) - Log("Plugin manifest " + name + " downloading failed from failover location.") - SimpleLog(p.plugin_log,"Plugin manifest " + name + " downloading failed from failover location.") - - filepath=LibDir+"/" + name + '.' + incarnation + '.manifest' - if os.path.splitext(location)[-1] == '.xml' : #if this is an xml file we may have a BOM - if ord(manifest[0]) > 128 and ord(manifest[1]) > 128 and ord(manifest[2]) > 128: - manifest=manifest[3:] - SetFileContents(filepath,manifest) - #Get the bundle url from the manifest - p.setAttribute('manifestdata',manifest) - man_dom = xml.dom.minidom.parseString(manifest) - bundle_uri = "" - for mp in man_dom.getElementsByTagName("Plugin"): - if GetNodeTextData(mp.getElementsByTagName("Version")[0]) == version: - bundle_uri = GetNodeTextData(mp.getElementsByTagName("Uri")[0]) - break - if len(mp.getElementsByTagName("DisallowMajorVersionUpgrade")): - if GetNodeTextData(mp.getElementsByTagName("DisallowMajorVersionUpgrade")[0]) == 'true' and previous_version !=None and previous_version.split('.')[0] != version.split('.')[0] : - Log('DisallowMajorVersionUpgrade is true, this major version is restricted from upgrade.') - SimpleLog(p.plugin_log,'DisallowMajorVersionUpgrade is true, this major version is restricted from upgrade.') - p.setAttribute('restricted','true') - continue - if len(bundle_uri) < 1 : - Error("Unable to fetch Bundle URI from manifest for " + name + " v " + version) - SimpleLog(p.plugin_log,"Unable to fetch Bundle URI from manifest for " + name + " v " + version) - continue - Log("Bundle URI = " + bundle_uri) - SimpleLog(p.plugin_log,"Bundle URI = " + bundle_uri) - - # Download the zipfile archive and save as '.zip' - bundle=self.Util.HttpGetWithoutHeaders(bundle_uri, chkProxy=True) - if bundle == None: - AddExtensionEvent(name,WALAEventOperation.Download,True,0,version,"Download zip fail "+bundle_uri) - Error("Unable to download plugin bundle" + bundle_uri ) - SimpleLog(p.plugin_log,"Unable to download plugin bundle" + bundle_uri ) - continue - AddExtensionEvent(name,WALAEventOperation.Download,True,0,version,"Download Success") - b=bytearray(bundle) - filepath=LibDir+"/" + os.path.basename(bundle_uri) + '.zip' - SetFileContents(filepath,b) - Log("Plugin bundle" + bundle_uri + "downloaded successfully length = " + str(len(bundle))) - SimpleLog(p.plugin_log,"Plugin bundle" + bundle_uri + "downloaded successfully length = " + str(len(bundle))) - - # unpack the archive - z=zipfile.ZipFile(filepath) - zip_dir=LibDir+"/" + name + '-' + version - z.extractall(zip_dir) - Log('Extracted ' + bundle_uri + ' to ' + zip_dir) - SimpleLog(p.plugin_log,'Extracted ' + bundle_uri + ' to ' + zip_dir) - - # zip no file perms in .zip so set all the scripts to +x - Run( "find " + zip_dir +" -type f | xargs chmod u+x ") - #write out the base64 config data so the plugin can process it. - mfile=None - for root, dirs, files in os.walk(zip_dir): - for f in files: - if f in ('HandlerManifest.json'): - mfile=os.path.join(root,f) - if mfile != None: - break - if mfile == None : - Error('HandlerManifest.json not found.') - SimpleLog(p.plugin_log,'HandlerManifest.json not found.') - continue - manifest = GetFileContents(mfile) - p.setAttribute('manifestdata',manifest) - # create the status and config dirs - Run('mkdir -p ' + root + '/status') - Run('mkdir -p ' + root + '/config') - # write out the configuration data to goalStateIncarnation.settings file in the config path. - config='' - seqNo='0' - if len(dom.getElementsByTagName("PluginSettings")) != 0 : - pslist=dom.getElementsByTagName("PluginSettings")[0].getElementsByTagName("Plugin") - for ps in pslist: - if name == ps.getAttribute("name") and version == ps.getAttribute("version"): - Log("Found RuntimeSettings for " + name + " V " + version) - SimpleLog(p.plugin_log,"Found RuntimeSettings for " + name + " V " + version) - - config=GetNodeTextData(ps.getElementsByTagName("RuntimeSettings")[0]) - seqNo=ps.getElementsByTagName("RuntimeSettings")[0].getAttribute("seqNo") - break - if config == '': - Log("No RuntimeSettings for " + name + " V " + version) - SimpleLog(p.plugin_log,"No RuntimeSettings for " + name + " V " + version) - - SetFileContents(root +"/config/" + seqNo +".settings", config ) - #create HandlerEnvironment.json - handler_env='[{ "name": "'+name+'", "seqNo": "'+seqNo+'", "version": 1.0, "handlerEnvironment": { "logFolder": "'+os.path.dirname(p.plugin_log)+'", "configFolder": "' + root + '/config", "statusFolder": "' + root + '/status", "heartbeatFile": "'+ root + '/heartbeat.log"}}]' - SetFileContents(root+'/HandlerEnvironment.json',handler_env) - self.SetHandlerState(handler, 'NotInstalled') - - cmd = '' - getcmd='installCommand' - if plg_dir != None and previous_version != None and version > previous_version : - previous_handler=name+'-'+previous_version - if self.GetHandlerState(previous_handler) != 'NotInstalled': - getcmd='updateCommand' - # disable the old plugin if it exists - if self.launchCommand(p.plugin_log,name,previous_version,'disableCommand') == None : - self.SetHandlerState(previous_handler, 'Enabled') - Error('Unable to disable old plugin '+name+' version ' + previous_version) - SimpleLog(p.plugin_log,'Unable to disable old plugin '+name+' version ' + previous_version) - else : - self.SetHandlerState(previous_handler, 'Disabled') - Log(name+' version ' + previous_version + ' is disabled') - SimpleLog(p.plugin_log,name+' version ' + previous_version + ' is disabled') - - isupgradeSuccess = True - if getcmd=='updateCommand': - if self.launchCommand(p.plugin_log,name,version,getcmd,previous_version) == None : - Error('Update failed for '+name+'-'+version) - SimpleLog(p.plugin_log,'Update failed for '+name+'-'+version) - isupgradeSuccess=False - else : - Log('Update complete'+name+'-'+version) - SimpleLog(p.plugin_log,'Update complete'+name+'-'+version) - - # if we updated - call unistall for the old plugin - if self.launchCommand(p.plugin_log,name,previous_version,'uninstallCommand') == None : - self.SetHandlerState(previous_handler, 'Installed') - Error('Uninstall failed for '+name+'-'+previous_version) - SimpleLog(p.plugin_log,'Uninstall failed for '+name+'-'+previous_version) - isupgradeSuccess=False - else : - self.SetHandlerState(previous_handler, 'NotInstalled') - Log('Uninstall complete'+ previous_handler ) - SimpleLog(p.plugin_log,'Uninstall complete'+ name +'-' + previous_version) - AddExtensionEvent(name,WALAEventOperation.Upgrade,isupgradeSuccess,0,previous_version) - else : # run install - if self.launchCommand(p.plugin_log,name,version,getcmd) == None : - self.SetHandlerState(handler, 'NotInstalled') - Error('Installation failed for '+name+'-'+version) - SimpleLog(p.plugin_log,'Installation failed for '+name+'-'+version) - else : - self.SetHandlerState(handler, 'Installed') - Log('Installation completed for '+name+'-'+version) - SimpleLog(p.plugin_log,'Installation completed for '+name+'-'+version) - - #end if plg_dir == none or version > = prev - # change incarnation of settings file so it knows how to name status... - zip_dir=LibDir+"/" + name + '-' + version - mfile=None - for root, dirs, files in os.walk(zip_dir): - for f in files: - if f in ('HandlerManifest.json'): - mfile=os.path.join(root,f) - if mfile != None: - break - if mfile == None : - Error('HandlerManifest.json not found.') - SimpleLog(p.plugin_log,'HandlerManifest.json not found.') - - continue - manifest = GetFileContents(mfile) - p.setAttribute('manifestdata',manifest) - config='' - seqNo='0' - if len(dom.getElementsByTagName("PluginSettings")) != 0 : - try: - pslist=dom.getElementsByTagName("PluginSettings")[0].getElementsByTagName("Plugin") - except: - Error('Error parsing ExtensionsConfig.') - SimpleLog(p.plugin_log,'Error parsing ExtensionsConfig.') - - continue - for ps in pslist: - if name == ps.getAttribute("name") and version == ps.getAttribute("version"): - Log("Found RuntimeSettings for " + name + " V " + version) - SimpleLog(p.plugin_log,"Found RuntimeSettings for " + name + " V " + version) - - config=GetNodeTextData(ps.getElementsByTagName("RuntimeSettings")[0]) - seqNo=ps.getElementsByTagName("RuntimeSettings")[0].getAttribute("seqNo") - break - if config == '': - Error("No RuntimeSettings for " + name + " V " + version) - SimpleLog(p.plugin_log,"No RuntimeSettings for " + name + " V " + version) - - SetFileContents(root +"/config/" + seqNo +".settings", config ) - - # state is still enable - if (self.GetHandlerState(handler) == 'NotInstalled'): # run install first if true - if self.launchCommand(p.plugin_log,name,version,'installCommand') == None : - self.SetHandlerState(handler, 'NotInstalled') - Error('Installation failed for '+name+'-'+version) - SimpleLog(p.plugin_log,'Installation failed for '+name+'-'+version) - - else : - self.SetHandlerState(handler, 'Installed') - Log('Installation completed for '+name+'-'+version) - SimpleLog(p.plugin_log,'Installation completed for '+name+'-'+version) - - - if (self.GetHandlerState(handler) != 'NotInstalled'): - if self.launchCommand(p.plugin_log,name,version,'enableCommand') == None : - self.SetHandlerState(handler, 'Installed') - Error('Enable failed for '+name+'-'+version) - SimpleLog(p.plugin_log,'Enable failed for '+name+'-'+version) - - else : - self.SetHandlerState(handler, 'Enabled') - Log('Enable completed for '+name+'-'+version) - SimpleLog(p.plugin_log,'Enable completed for '+name+'-'+version) - - # this plugin processing is complete - Log('Processing completed for '+name+'-'+version) - SimpleLog(p.plugin_log,'Processing completed for '+name+'-'+version) - - #end plugin processing loop - Log('Finished processing ExtensionsConfig.xml') - try: - SimpleLog(p.plugin_log,'Finished processing ExtensionsConfig.xml') - except: - pass - - return self - def launchCommand(self,plugin_log,name,version,command,prev_version=None): - commandToEventOperation={ - "installCommand":WALAEventOperation.Install, - "uninstallCommand":WALAEventOperation.UnIsntall, - "updateCommand": WALAEventOperation.Upgrade, - "enableCommand": WALAEventOperation.Enable, - "disableCommand": WALAEventOperation.Disable, - } - isSuccess=True - start = datetime.datetime.now() - r=self.__launchCommandWithoutEventLog(plugin_log,name,version,command,prev_version) - if r==None: - isSuccess=False - Duration = int((datetime.datetime.now() - start).seconds) - if commandToEventOperation.get(command): - AddExtensionEvent(name,commandToEventOperation[command],isSuccess,Duration,version) - return r - - def __launchCommandWithoutEventLog(self,plugin_log,name,version,command,prev_version=None): - # get the manifest and read the command - mfile=None - zip_dir=LibDir+"/" + name + '-' + version - for root, dirs, files in os.walk(zip_dir): - for f in files: - if f in ('HandlerManifest.json'): - mfile=os.path.join(root,f) - if mfile != None: - break - if mfile == None : - Error('HandlerManifest.json not found.') - SimpleLog(plugin_log,'HandlerManifest.json not found.') - - return None - manifest = GetFileContents(mfile) - try: - jsn = json.loads(manifest) - except: - Error('Error parsing HandlerManifest.json.') - SimpleLog(plugin_log,'Error parsing HandlerManifest.json.') - - return None - if type(jsn)==list: - jsn=jsn[0] - if jsn.has_key('handlerManifest') : - cmd = jsn['handlerManifest'][command] - else : - Error('Key handlerManifest not found. Handler cannot be installed.') - SimpleLog(plugin_log,'Key handlerManifest not found. Handler cannot be installed.') - - if len(cmd) == 0 : - Error('Unable to read ' + command ) - SimpleLog(plugin_log,'Unable to read ' + command ) - - return None - - # for update we send the path of the old installation - arg='' - if prev_version != None : - arg=' ' + LibDir+'/' + name + '-' + prev_version - dirpath=os.path.dirname(mfile) - LogIfVerbose('Command is '+ dirpath+'/'+ cmd) - # launch - pid=None - try: - child = subprocess.Popen(dirpath+'/'+cmd+arg,shell=True,cwd=dirpath,stdout=subprocess.PIPE) - except Exception as e: - Error('Exception launching ' + cmd + str(e)) - SimpleLog(plugin_log,'Exception launching ' + cmd + str(e)) - - pid = child.pid - if pid == None or pid < 1 : - ExtensionChildren.append((-1,root)) - Error('Error launching ' + cmd + '.') - SimpleLog(plugin_log,'Error launching ' + cmd + '.') - - else : - ExtensionChildren.append((pid,root)) - Log("Spawned "+ cmd + " PID " + str(pid)) - SimpleLog(plugin_log,"Spawned "+ cmd + " PID " + str(pid)) - - - # wait until install/upgrade is finished - timeout = 300 # 5 minutes - retry = timeout/5 - while retry > 0 and child.poll() == None: - LogIfVerbose(cmd + ' still running with PID ' + str(pid)) - time.sleep(5) - retry-=1 - if retry==0: - Error('Process exceeded timeout of ' + str(timeout) + ' seconds. Terminating process ' + str(pid)) - SimpleLog(plugin_log,'Process exceeded timeout of ' + str(timeout) + ' seconds. Terminating process ' + str(pid)) - - os.kill(pid,9) - return None - code = child.wait() - if code == None or code != 0: - Error('Process ' + str(pid) + ' returned non-zero exit code (' + str(code) + ')') - SimpleLog(plugin_log,'Process ' + str(pid) + ' returned non-zero exit code (' + str(code) + ')') - - return None - Log(command + ' completed.') - SimpleLog(plugin_log,command + ' completed.') - - return 0 - - def ReportHandlerStatus(self): - """ - Collect all status reports. - """ - # { "version": "1.0", "timestampUTC": "2014-03-31T21:28:58Z", - # "aggregateStatus": { - # "guestAgentStatus": { "version": "2.0.4PRE", "status": "Ready", "formattedMessage": { "lang": "en-US", "message": "GuestAgent is running and accepting new configurations." } }, - # "handlerAggregateStatus": [{ - # "handlerName": "ExampleHandlerLinux", "handlerVersion": "1.0", "status": "Ready", "runtimeSettingsStatus": { - # "sequenceNumber": "2", "settingsStatus": { "timestampUTC": "2014-03-31T23:46:00Z", "status": { "name": "ExampleHandlerLinux", "operation": "Command Execution Finished", "configurationAppliedTime": "2014-03-31T23:46:00Z", "status": "success", "formattedMessage": { "lang": "en-US", "message": "Finished executing command" }, - # "substatus": [ - # { "name": "StdOut", "status": "success", "formattedMessage": { "lang": "en-US", "message": "Goodbye world!" } }, - # { "name": "StdErr", "status": "success", "formattedMessage": { "lang": "en-US", "message": "" } } - # ] - # } } } } - # ] - # }} - - try: - incarnation=self.Extensions[0].getAttribute("goalStateIncarnation") - except: - Error('Error parsing ExtensionsConfig. Unable to send status reports') - return None - status='' - statuses='' - for p in self.Plugins: - if p.getAttribute("state") == 'uninstall' or p.getAttribute("restricted") == 'true' : - continue - version=p.getAttribute("version") - name=p.getAttribute("name") - if p.getAttribute("isJson") != 'true': - LogIfVerbose("Plugin " + name+" version: " +version+" is not a JSON Extension. Skipping.") - continue - reportHeartbeat = False - if len(p.getAttribute("manifestdata"))<1: - Error("Failed to get manifestdata.") - else: - reportHeartbeat = json.loads(p.getAttribute("manifestdata"))[0]['handlerManifest']['reportHeartbeat'] - if len(statuses)>0: - statuses+=',' - statuses+=self.GenerateAggStatus(name, version, reportHeartbeat) - tstamp=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - #header - #agent state - if provisioned == False: - if provisionError == None : - agent_state='Provisioning' - agent_msg='Guest Agent is starting.' - else: - agent_state='Provisioning Error.' - agent_msg=provisionError - else: - agent_state='Ready' - agent_msg='GuestAgent is running and accepting new configurations.' - - status='{"version":"1.0","timestampUTC":"'+tstamp+'","aggregateStatus":{"guestAgentStatus":{"version":"'+GuestAgentVersion+'","status":"'+agent_state+'","formattedMessage":{"lang":"en-US","message":"'+agent_msg+'"}},"handlerAggregateStatus":['+statuses+']}}' - try: - uri=GetNodeTextData(self.Extensions[0].getElementsByTagName("StatusUploadBlob")[0]).replace('&','&') - except: - Error('Error parsing ExtensionsConfig. Unable to send status reports') - return None - - UploadStatusBlob(uri, status.encode("utf-8")) - LogIfVerbose('Status report '+status+' sent to ' + uri) - return True - - def GetCurrentSequenceNumber(self, plugin_base_dir): - """ - Get the settings file with biggest file number in config folder - """ - config_dir = os.path.join(plugin_base_dir, 'config') - seq_no = 0 - for subdir, dirs, files in os.walk(config_dir): - for file in files: - try: - cur_seq_no = int(os.path.basename(file).split('.')[0]) - if cur_seq_no > seq_no: - seq_no = cur_seq_no - except ValueError: - continue - return str(seq_no) - - - def GenerateAggStatus(self, name, version, reportHeartbeat = False): - """ - Generate the status which Azure can understand by the status and heartbeat reported by extension - """ - plugin_base_dir = LibDir+'/'+name+'-'+version+'/' - current_seq_no = self.GetCurrentSequenceNumber(plugin_base_dir) - status_file=os.path.join(plugin_base_dir, 'status/', current_seq_no +'.status') - heartbeat_file = os.path.join(plugin_base_dir, 'heartbeat.log') - - handler_state_file = os.path.join(plugin_base_dir, 'config', 'HandlerState') - agg_state = 'NotReady' - handler_state = None - status_obj = None - status_code = None - formatted_message = None - localized_message = None - - if os.path.exists(handler_state_file): - handler_state = GetFileContents(handler_state_file).lower() - if HandlerStatusToAggStatus.has_key(handler_state): - agg_state = HandlerStatusToAggStatus[handler_state] - if reportHeartbeat: - if os.path.exists(heartbeat_file): - d=int(time.time()-os.stat(heartbeat_file).st_mtime) - if d > 600 : # not updated for more than 10 min - agg_state = 'Unresponsive' - else: - try: - heartbeat = json.loads(GetFileContents(heartbeat_file))[0]["heartbeat"] - agg_state = heartbeat.get("status") - status_code = heartbeat.get("code") - formatted_message = heartbeat.get("formattedMessage") - localized_message = heartbeat.get("message") - except: - Error("Incorrect heartbeat file. Ignore it. ") - else: - agg_state = 'Unresponsive' - #get status file reported by extension - if os.path.exists(status_file): - # raw status generated by extension is an array, get the first item and remove the unnecessary element - try: - status_obj = json.loads(GetFileContents(status_file))[0] - del status_obj["version"] - except: - Error("Incorrect status file. Will NOT settingsStatus in settings. ") - agg_status_obj = {"handlerName": name, "handlerVersion": version, "status": agg_state, "runtimeSettingsStatus" : - {"sequenceNumber": current_seq_no}} - if status_obj: - agg_status_obj["runtimeSettingsStatus"]["settingsStatus"] = status_obj - if status_code != None: - agg_status_obj["code"] = status_code - if formatted_message: - agg_status_obj["formattedMessage"] = formatted_message - if localized_message: - agg_status_obj["message"] = localized_message - agg_status_string = json.dumps(agg_status_obj) - LogIfVerbose("Handler Aggregated Status:" + agg_status_string) - return agg_status_string - - - def SetHandlerState(self, handler, state=''): - zip_dir=LibDir+"/" + handler - mfile=None - for root, dirs, files in os.walk(zip_dir): - for f in files: - if f in ('HandlerManifest.json'): - mfile=os.path.join(root,f) - if mfile != None: - break - if mfile == None : - Error('SetHandlerState(): HandlerManifest.json not found, cannot set HandlerState.') - return None - Log("SetHandlerState: "+handler+", "+state) - return SetFileContents(os.path.dirname(mfile)+'/config/HandlerState', state) - - def GetHandlerState(self, handler): - handlerState = GetFileContents(handler+'/config/HandlerState') - if (handlerState): - return handlerState.rstrip('\r\n') - else: - return 'NotInstalled' - - -class HostingEnvironmentConfig(object): - """ - Parse Hosting enviromnet config and store in - HostingEnvironmentConfig.xml - """ - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - def __init__(self): - self.reinitialize() - - def reinitialize(self): - """ - Reset Members. - """ - self.StoredCertificates = None - self.Deployment = None - self.Incarnation = None - self.Role = None - self.HostingEnvironmentSettings = None - self.ApplicationSettings = None - self.Certificates = None - self.ResourceReferences = None - - def Parse(self, xmlText): - """ - Parse and create HostingEnvironmentConfig.xml. - """ - self.reinitialize() - SetFileContents("HostingEnvironmentConfig.xml", xmlText) - dom = xml.dom.minidom.parseString(xmlText) - for a in [ "HostingEnvironmentConfig", "Deployment", "Service", - "ServiceInstance", "Incarnation", "Role", ]: - if not dom.getElementsByTagName(a): - Error("HostingEnvironmentConfig.Parse: Missing " + a) - return None - node = dom.childNodes[0] - if node.localName != "HostingEnvironmentConfig": - Error("HostingEnvironmentConfig.Parse: root not HostingEnvironmentConfig") - return None - self.ApplicationSettings = dom.getElementsByTagName("Setting") - self.Certificates = dom.getElementsByTagName("StoredCertificate") - return self - - def DecryptPassword(self, e): - """ - Return decrypted password. - """ - SetFileContents("password.p7m", - "MIME-Version: 1.0\n" - + "Content-Disposition: attachment; filename=\"password.p7m\"\n" - + "Content-Type: application/x-pkcs7-mime; name=\"password.p7m\"\n" - + "Content-Transfer-Encoding: base64\n\n" - + textwrap.fill(e, 64)) - return RunGetOutput(Openssl + " cms -decrypt -in password.p7m -inkey Certificates.pem -recip Certificates.pem")[1] - - def ActivateResourceDisk(self): - return MyDistro.ActivateResourceDisk() - - def Process(self): - """ - Execute ActivateResourceDisk in separate thread. - Create the user account. - Launch ConfigurationConsumer if specified in the config. - """ - no_thread = False - if DiskActivated == False: - for m in inspect.getmembers(MyDistro): - if 'ActivateResourceDiskNoThread' in m: - no_thread = True - break - if no_thread == True : - MyDistro.ActivateResourceDiskNoThread() - else : - diskThread = threading.Thread(target = self.ActivateResourceDisk) - diskThread.start() - User = None - Pass = None - Expiration = None - Thumbprint = None - for b in self.ApplicationSettings: - sname = b.getAttribute("name") - svalue = b.getAttribute("value") - if User != None and Pass != None: - if User != "root" and User != "" and Pass != "": - CreateAccount(User, Pass, Expiration, Thumbprint) - else: - Error("Not creating user account: " + User) - for c in self.Certificates: - csha1 = c.getAttribute("certificateId").split(':')[1].upper() - if os.path.isfile(csha1 + ".prv"): - Log("Private key with thumbprint: " + csha1 + " was retrieved.") - if os.path.isfile(csha1 + ".crt"): - Log("Public cert with thumbprint: " + csha1 + " was retrieved.") - program = Config.get("Role.ConfigurationConsumer") - if program != None: - try: - Children.append(subprocess.Popen([program, LibDir + "/HostingEnvironmentConfig.xml"])) - except OSError, e : - ErrorWithPrefix('HostingEnvironmentConfig.Process','Exception: '+ str(e) +' occured launching ' + program ) - -class GoalState(Util): - """ - Primary container for all configuration except OvfXml. - Encapsulates http communication with endpoint server. - Initializes and populates: - self.HostingEnvironmentConfig - self.SharedConfig - self.ExtensionsConfig - self.Certificates - """ - # - # - # 2010-12-15 - # 1 - # - # Started - # - # 16001 - # - # - # - # c6d5526c-5ac2-4200-b6e2-56f2b70c5ab2 - # - # - # MachineRole_IN_0 - # Started - # - # http://10.115.153.40:80/machine/c6d5526c-5ac2-4200-b6e2-56f2b70c5ab2/MachineRole%5FIN%5F0?comp=config&type=hostingEnvironmentConfig&incarnation=1 - # http://10.115.153.40:80/machine/c6d5526c-5ac2-4200-b6e2-56f2b70c5ab2/MachineRole%5FIN%5F0?comp=config&type=sharedConfig&incarnation=1 - # http://10.115.153.40:80/machine/c6d5526c-5ac2-4200-b6e2-56f2b70c5ab2/MachineRole%5FIN%5F0?comp=certificates&incarnation=1 - # http://100.67.238.230:80/machine/9c87aa94-3bda-45e3-b2b7-0eb0fca7baff/1552dd64dc254e6884f8d5b8b68aa18f.eg%2Dplug%2Dvm?comp=config&type=extensionsConfig&incarnation=2 - # http://100.67.238.230:80/machine/9c87aa94-3bda-45e3-b2b7-0eb0fca7baff/1552dd64dc254e6884f8d5b8b68aa18f.eg%2Dplug%2Dvm?comp=config&type=fullConfig&incarnation=2 - - # - # - # - # - # - # - # There is only one Role for VM images. - # - # Of primary interest is: - # LBProbePorts -- an http server needs to run here - # We also note Container/ContainerID and RoleInstance/InstanceId to form the health report. - # And of course, Incarnation - # - def __init__(self, Agent): - self.Agent = Agent - self.Endpoint = Agent.Endpoint - self.TransportCert = Agent.TransportCert - self.reinitialize() - - def reinitialize(self): - self.Incarnation = None # integer - self.ExpectedState = None # "Started" - self.HostingEnvironmentConfigUrl = None - self.HostingEnvironmentConfigXml = None - self.HostingEnvironmentConfig = None - self.SharedConfigUrl = None - self.SharedConfigXml = None - self.SharedConfig = None - self.CertificatesUrl = None - self.CertificatesXml = None - self.Certificates = None - self.ExtensionsConfigUrl = None - self.ExtensionsConfigXml = None - self.ExtensionsConfig = None - self.RoleInstanceId = None - self.ContainerId = None - self.LoadBalancerProbePort = None # integer, ?list of integers - - def Parse(self, xmlText): - """ - Request configuration data from endpoint server. - Parse and populate contained configuration objects. - Calls Certificates().Parse() - Calls SharedConfig().Parse - Calls ExtensionsConfig().Parse - Calls HostingEnvironmentConfig().Parse - """ - self.reinitialize() - LogIfVerbose(xmlText) - node = xml.dom.minidom.parseString(xmlText).childNodes[0] - if node.localName != "GoalState": - Error("GoalState.Parse: root not GoalState") - return None - for a in node.childNodes: - if a.nodeType == node.ELEMENT_NODE: - if a.localName == "Incarnation": - self.Incarnation = GetNodeTextData(a) - elif a.localName == "Machine": - for b in a.childNodes: - if b.nodeType == node.ELEMENT_NODE: - if b.localName == "ExpectedState": - self.ExpectedState = GetNodeTextData(b) - Log("ExpectedState: " + self.ExpectedState) - elif b.localName == "LBProbePorts": - for c in b.childNodes: - if c.nodeType == node.ELEMENT_NODE and c.localName == "Port": - self.LoadBalancerProbePort = int(GetNodeTextData(c)) - elif a.localName == "Container": - for b in a.childNodes: - if b.nodeType == node.ELEMENT_NODE: - if b.localName == "ContainerId": - self.ContainerId = GetNodeTextData(b) - Log("ContainerId: " + self.ContainerId) - elif b.localName == "RoleInstanceList": - for c in b.childNodes: - if c.localName == "RoleInstance": - for d in c.childNodes: - if d.nodeType == node.ELEMENT_NODE: - if d.localName == "InstanceId": - self.RoleInstanceId = GetNodeTextData(d) - Log("RoleInstanceId: " + self.RoleInstanceId) - elif d.localName == "State": - pass - elif d.localName == "Configuration": - for e in d.childNodes: - if e.nodeType == node.ELEMENT_NODE: - LogIfVerbose(e.localName) - if e.localName == "HostingEnvironmentConfig": - self.HostingEnvironmentConfigUrl = GetNodeTextData(e) - LogIfVerbose("HostingEnvironmentConfigUrl:" + self.HostingEnvironmentConfigUrl) - self.HostingEnvironmentConfigXml = self.HttpGetWithHeaders(self.HostingEnvironmentConfigUrl) - self.HostingEnvironmentConfig = HostingEnvironmentConfig().Parse(self.HostingEnvironmentConfigXml) - elif e.localName == "SharedConfig": - self.SharedConfigUrl = GetNodeTextData(e) - LogIfVerbose("SharedConfigUrl:" + self.SharedConfigUrl) - self.SharedConfigXml = self.HttpGetWithHeaders(self.SharedConfigUrl) - self.SharedConfig = SharedConfig().Parse(self.SharedConfigXml) - self.SharedConfig.Save() - elif e.localName == "ExtensionsConfig": - self.ExtensionsConfigUrl = GetNodeTextData(e) - LogIfVerbose("ExtensionsConfigUrl:" + self.ExtensionsConfigUrl) - self.ExtensionsConfigXml = self.HttpGetWithHeaders(self.ExtensionsConfigUrl) - elif e.localName == "Certificates": - self.CertificatesUrl = GetNodeTextData(e) - LogIfVerbose("CertificatesUrl:" + self.CertificatesUrl) - self.CertificatesXml = self.HttpSecureGetWithHeaders(self.CertificatesUrl, self.TransportCert) - self.Certificates = Certificates().Parse(self.CertificatesXml) - if self.Incarnation == None: - Error("GoalState.Parse: Incarnation missing") - return None - if self.ExpectedState == None: - Error("GoalState.Parse: ExpectedState missing") - return None - if self.RoleInstanceId == None: - Error("GoalState.Parse: RoleInstanceId missing") - return None - if self.ContainerId == None: - Error("GoalState.Parse: ContainerId missing") - return None - SetFileContents("GoalState." + self.Incarnation + ".xml", xmlText) - return self - - def Process(self): - """ - Calls HostingEnvironmentConfig.Process() - """ - LogIfVerbose("Process goalstate") - self.HostingEnvironmentConfig.Process() - self.SharedConfig.Process() - -class OvfEnv(object): - """ - Read, and process provisioning info from provisioning file OvfEnv.xml - """ - # - # - # - # - # 1.0 - # - # LinuxProvisioningConfiguration - # HostName - # UserName - # UserPassword - # false - # - # - # - # EB0C0AB4B2D5FC35F2F0658D19F44C8283E2DD62 - # $HOME/UserName/.ssh/authorized_keys - # - # - # - # - # EB0C0AB4B2D5FC35F2F0658D19F44C8283E2DD62 - # $HOME/UserName/.ssh/id_rsa - # - # - # - # - # - # - # - def __init__(self): - self.reinitialize() - - def reinitialize(self): - """ - Reset members. - """ - self.WaNs = "http://schemas.microsoft.com/windowsazure" - self.OvfNs = "http://schemas.dmtf.org/ovf/environment/1" - self.MajorVersion = 1 - self.MinorVersion = 0 - self.ComputerName = None - self.AdminPassword = None - self.UserName = None - self.UserPassword = None - self.CustomData = None - self.DisableSshPasswordAuthentication = True - self.SshPublicKeys = [] - self.SshKeyPairs = [] - - def Parse(self, xmlText, isDeprovision = False): - """ - Parse xml tree, retreiving user and ssh key information. - Return self. - """ - self.reinitialize() - LogIfVerbose(re.sub(".*?<", "*<", xmlText)) - dom = xml.dom.minidom.parseString(xmlText) - if len(dom.getElementsByTagNameNS(self.OvfNs, "Environment")) != 1: - Error("Unable to parse OVF XML.") - section = None - newer = False - for p in dom.getElementsByTagNameNS(self.WaNs, "ProvisioningSection"): - for n in p.childNodes: - if n.localName == "Version": - verparts = GetNodeTextData(n).split('.') - major = int(verparts[0]) - minor = int(verparts[1]) - if major > self.MajorVersion: - newer = True - if major != self.MajorVersion: - break - if minor > self.MinorVersion: - newer = True - section = p - if newer == True: - Warn("Newer provisioning configuration detected. Please consider updating waagent.") - if section == None: - Error("Could not find ProvisioningSection with major version=" + str(self.MajorVersion)) - return None - self.ComputerName = GetNodeTextData(section.getElementsByTagNameNS(self.WaNs, "HostName")[0]) - self.UserName = GetNodeTextData(section.getElementsByTagNameNS(self.WaNs, "UserName")[0]) - if isDeprovision == True: - return self - try: - self.UserPassword = GetNodeTextData(section.getElementsByTagNameNS(self.WaNs, "UserPassword")[0]) - except: - pass - CDSection=None - try: - CDSection=section.getElementsByTagNameNS(self.WaNs, "CustomData") - if len(CDSection) > 0 : - self.CustomData=GetNodeTextData(CDSection[0]) - if len(self.CustomData)>0: - SetFileContents(LibDir + '/CustomData', MyDistro.translateCustomData(self.CustomData)) - Log('Wrote ' + LibDir + '/CustomData') - else : - Error(' contains no data!') - except Exception, e: - Error( str(e)+' occured creating ' + LibDir + '/CustomData') - disableSshPass = section.getElementsByTagNameNS(self.WaNs, "DisableSshPasswordAuthentication") - if len(disableSshPass) != 0: - self.DisableSshPasswordAuthentication = (GetNodeTextData(disableSshPass[0]).lower() == "true") - for pkey in section.getElementsByTagNameNS(self.WaNs, "PublicKey"): - LogIfVerbose(repr(pkey)) - fp = None - path = None - for c in pkey.childNodes: - if c.localName == "Fingerprint": - fp = GetNodeTextData(c).upper() - LogIfVerbose(fp) - if c.localName == "Path": - path = GetNodeTextData(c) - LogIfVerbose(path) - self.SshPublicKeys += [[fp, path]] - for keyp in section.getElementsByTagNameNS(self.WaNs, "KeyPair"): - fp = None - path = None - LogIfVerbose(repr(keyp)) - for c in keyp.childNodes: - if c.localName == "Fingerprint": - fp = GetNodeTextData(c).upper() - LogIfVerbose(fp) - if c.localName == "Path": - path = GetNodeTextData(c) - LogIfVerbose(path) - self.SshKeyPairs += [[fp, path]] - return self - - def PrepareDir(self, filepath): - """ - Create home dir for self.UserName - Change owner and return path. - """ - home = MyDistro.GetHome() - # Expand HOME variable if present in path - path = os.path.normpath(filepath.replace("$HOME", home)) - if (path.startswith("/") == False) or (path.endswith("/") == True): - return None - dir = path.rsplit('/', 1)[0] - if dir != "": - CreateDir(dir, "root", 0700) - if path.startswith(os.path.normpath(home + "/" + self.UserName + "/")): - ChangeOwner(dir, self.UserName) - return path - - def NumberToBytes(self, i): - """ - Pack number into bytes. Retun as string. - """ - result = [] - while i: - result.append(chr(i & 0xFF)) - i >>= 8 - result.reverse() - return ''.join(result) - - def BitsToString(self, a): - """ - Return string representation of bits in a. - """ - index=7 - s = "" - c = 0 - for bit in a: - c = c | (bit << index) - index = index - 1 - if index == -1: - s = s + struct.pack('>B', c) - c = 0 - index = 7 - return s - - def OpensslToSsh(self, file): - """ - Return base-64 encoded key appropriate for ssh. - """ - from pyasn1.codec.der import decoder as der_decoder - try: - f = open(file).read().replace('\n','').split("KEY-----")[1].split('-')[0] - k=der_decoder.decode(self.BitsToString(der_decoder.decode(base64.b64decode(f))[0][1]))[0] - n=k[0] - e=k[1] - keydata="" - keydata += struct.pack('>I',len("ssh-rsa")) - keydata += "ssh-rsa" - keydata += struct.pack('>I',len(self.NumberToBytes(e))) - keydata += self.NumberToBytes(e) - keydata += struct.pack('>I',len(self.NumberToBytes(n)) + 1) - keydata += "\0" - keydata += self.NumberToBytes(n) - except Exception, e: - print("OpensslToSsh: Exception " + str(e)) - return None - return "ssh-rsa " + base64.b64encode(keydata) + "\n" - - def Process(self): - """ - Process all certificate and key info. - DisableSshPasswordAuthentication if configured. - CreateAccount(user) - Wait for WaAgent.EnvMonitor.IsHostnamePublished(). - Restart ssh service. - """ - error = None - if self.ComputerName == None : - return "Error: Hostname missing" - error=WaAgent.EnvMonitor.SetHostName(self.ComputerName) - if error: return error - if self.DisableSshPasswordAuthentication: - filepath = "/etc/ssh/sshd_config" - # Disable RFC 4252 and RFC 4256 authentication schemes. - ReplaceFileContentsAtomic(filepath, "\n".join(filter(lambda a: not - (a.startswith("PasswordAuthentication") or a.startswith("ChallengeResponseAuthentication")), - GetFileContents(filepath).split('\n'))) + "\nPasswordAuthentication no\nChallengeResponseAuthentication no\n") - Log("Disabled SSH password-based authentication methods.") - if self.AdminPassword != None: - MyDistro.changePass('root',self.AdminPassword) - if self.UserName != None: - error = MyDistro.CreateAccount(self.UserName, self.UserPassword, None, None) - sel = MyDistro.isSelinuxRunning() - if sel : - MyDistro.setSelinuxEnforce(0) - home = MyDistro.GetHome() - for pkey in self.SshPublicKeys: - Log("Deploy public key:{0}".format(pkey[0])) - if not os.path.isfile(pkey[0] + ".crt"): - Error("PublicKey not found: " + pkey[0]) - error = "Failed to deploy public key (0x09)." - continue - path = self.PrepareDir(pkey[1]) - if path == None: - Error("Invalid path: " + pkey[1] + " for PublicKey: " + pkey[0]) - error = "Invalid path for public key (0x03)." - continue - Run(Openssl + " x509 -in " + pkey[0] + ".crt -noout -pubkey > " + pkey[0] + ".pub") - MyDistro.setSelinuxContext(pkey[0] + '.pub','unconfined_u:object_r:ssh_home_t:s0') - MyDistro.sshDeployPublicKey(pkey[0] + '.pub',path) - MyDistro.setSelinuxContext(path,'unconfined_u:object_r:ssh_home_t:s0') - if path.startswith(os.path.normpath(home + "/" + self.UserName + "/")): - ChangeOwner(path, self.UserName) - for keyp in self.SshKeyPairs: - Log("Deploy key pair:{0}".format(keyp[0])) - if not os.path.isfile(keyp[0] + ".prv"): - Error("KeyPair not found: " + keyp[0]) - error = "Failed to deploy key pair (0x0A)." - continue - path = self.PrepareDir(keyp[1]) - if path == None: - Error("Invalid path: " + keyp[1] + " for KeyPair: " + keyp[0]) - error = "Invalid path for key pair (0x05)." - continue - SetFileContents(path, GetFileContents(keyp[0] + ".prv")) - os.chmod(path, 0600) - Run("ssh-keygen -y -f " + keyp[0] + ".prv > " + path + ".pub") - MyDistro.setSelinuxContext(path,'unconfined_u:object_r:ssh_home_t:s0') - MyDistro.setSelinuxContext(path + '.pub','unconfined_u:object_r:ssh_home_t:s0') - if path.startswith(os.path.normpath(home + "/" + self.UserName + "/")): - ChangeOwner(path, self.UserName) - ChangeOwner(path + ".pub", self.UserName) - if sel : - MyDistro.setSelinuxEnforce(1) - while not WaAgent.EnvMonitor.IsHostnamePublished(): - time.sleep(1) - MyDistro.restartSshService() - return error - - -class WALAEvent(object): - def __init__(self): - - self.providerId="" - self.eventId=1 - - self.OpcodeName="" - self.KeywordName="" - self.TaskName="" - self.TenantName="" - self.RoleName="" - self.RoleInstanceName="" - self.ContainerId="" - self.ExecutionMode="IAAS" - self.OSVersion="" - self.GAVersion="" - self.RAM=0 - self.Processors=0 - - - def ToXml(self): - strEventid=u''.format(self.eventId) - strProviderid=u''.format(self.providerId) - strRecordFormat = u'' - strRecordNoQuoteFormat = u'' - strMtStr=u'mt:wstr' - strMtUInt64=u'mt:uint64' - strMtBool=u'mt:bool' - strMtFloat=u'mt:float64' - strEventsData=u"" - - for attName in self.__dict__: - if attName in ["eventId","filedCount","providerId"]: - continue - - attValue = self.__dict__[attName] - if type(attValue) is int: - strEventsData+=strRecordFormat.format(attName,attValue,strMtUInt64) - continue - if type(attValue) is str: - attValue = xml.sax.saxutils.quoteattr(attValue) - strEventsData+=strRecordNoQuoteFormat.format(attName,attValue,strMtStr) - continue - if str(type(attValue)).count("'unicode'") >0 : - attValue = xml.sax.saxutils.quoteattr(attValue) - strEventsData+=strRecordNoQuoteFormat.format(attName,attValue,strMtStr) - continue - if type(attValue) is bool: - strEventsData+=strRecordFormat.format(attName,attValue,strMtBool) - continue - if type(attValue) is float: - strEventsData+=strRecordFormat.format(attName,attValue,strMtFloat) - continue - - Log("Warning: property "+attName+":"+str(type(attValue))+":type"+str(type(attValue))+"Can't convert to events data:"+":type not supported") - - return u"{0}{1}{2}".format(strProviderid,strEventid,strEventsData) - - def Save(self): - eventfolder = LibDir+"/events" - if not os.path.exists(eventfolder): - os.mkdir(eventfolder) - os.chmod(eventfolder,0700) - if len(os.listdir(eventfolder)) > 1000: - raise Exception("WriteToFolder:Too many file under "+eventfolder+" exit") - - filename = os.path.join(eventfolder,str(int(time.time()*1000000))) - with open(filename+".tmp",'wb+') as hfile: - hfile.write(self.ToXml().encode("utf-8")) - os.rename(filename+".tmp",filename+".tld") - - -class WALAEventOperation: - HeartBeat="HeartBeat" - Provision = "Provision" - Install = "Install" - UnIsntall = "UnInstall" - Disable = "Disable" - Enable = "Enable" - Download = "Download" - Upgrade = "Upgrade" - Update = "Update" - -def AddExtensionEvent(name,op,isSuccess,duration=0,version="1.0",message="",type="",isInternal=False): - event = ExtensionEvent() - event.Name=name - event.Version=version - event.IsInternal=isInternal - event.Operation=op - event.OperationSuccess=isSuccess - event.Message=message - event.Duration=duration - event.ExtensionType=type - try: - event.Save() - except: - Error("Error "+traceback.format_exc()) - - -class ExtensionEvent(WALAEvent): - def __init__(self): - - WALAEvent.__init__(self) - self.eventId=1 - self.providerId="69B669B9-4AF8-4C50-BDC4-6006FA76E975" - self.Name="" - self.Version="" - self.IsInternal=False - self.Operation="" - self.OperationSuccess=True - self.ExtensionType="" - self.Message="" - self.Duration=0 - - -class WALAEventMonitor(WALAEvent): - def __init__(self,postMethod): - WALAEvent.__init__(self) - self.post = postMethod - self.sysInfo={} - self.eventdir = LibDir+"/events" - self.issysteminfoinitilized = False - - def StartEventsLoop(self): - eventThread = threading.Thread(target = self.EventsLoop) - eventThread.setDaemon(True) - eventThread.start() - - def EventsLoop(self): - LastReportHeartBeatTime = datetime.datetime.min - try: - while(True): - if (datetime.datetime.now()-LastReportHeartBeatTime) > datetime.timedelta(hours=12): - LastReportHeartBeatTime = datetime.datetime.now() - AddExtensionEvent(op=WALAEventOperation.HeartBeat,name="WALA",isSuccess=True) - self.postNumbersInOneLoop=0 - self.CollectAndSendWALAEvents() - time.sleep(60) - except: - Error("Exception in events loop:"+traceback.format_exc()) - - def SendEvent(self,providerid,events): - dataFormat = u'{1}'\ - '' - data = dataFormat.format(providerid,events) - self.post("/machine/?comp=telemetrydata", data) - - def CollectAndSendWALAEvents(self): - if not os.path.exists(self.eventdir): - return - #Throtting, can't send more than 3 events in 15 seconds - eventSendNumber=0 - eventFiles = os.listdir(self.eventdir) - events = {} - for file in eventFiles: - if not file.endswith(".tld"): - continue - with open(os.path.join(self.eventdir,file),"rb") as hfile: - #if fail to open or delete the file, throw exception - xmlStr = hfile.read().decode("utf-8",'ignore') - os.remove(os.path.join(self.eventdir,file)) - params="" - eventid="" - providerid="" - #if exception happen during process an event, catch it and continue - try: - xmlStr = self.AddSystemInfo(xmlStr) - for node in xml.dom.minidom.parseString(xmlStr.encode("utf-8")).childNodes[0].childNodes: - if node.tagName == "Param": - params+=node.toxml() - if node.tagName == "Event": - eventid=node.getAttribute("id") - if node.tagName == "Provider": - providerid = node.getAttribute("id") - except: - Error(traceback.format_exc()) - continue - if len(params)==0 or len(eventid)==0 or len(providerid)==0: - Error("Empty filed in params:"+params+" event id:"+eventid+" provider id:"+providerid) - continue - - eventstr = u''.format(eventid,params) - if not events.get(providerid): - events[providerid]="" - if len(events[providerid]) >0 and len(events.get(providerid)+eventstr)>= 63*1024: - eventSendNumber+=1 - self.SendEvent(providerid,events.get(providerid)) - if eventSendNumber %3 ==0: - time.sleep(15) - events[providerid]="" - if len(eventstr) >= 63*1024: - Error("Signle event too large abort "+eventstr[:300]) - continue - - events[providerid]=events.get(providerid)+eventstr - - for key in events.keys(): - if len(events[key]) > 0: - eventSendNumber+=1 - self.SendEvent(key,events[key]) - if eventSendNumber%3 == 0: - time.sleep(15) - - - def AddSystemInfo(self,eventData): - if not self.issysteminfoinitilized: - self.issysteminfoinitilized=True - try: - self.sysInfo["OSVersion"]=platform.system()+":"+"-".join(DistInfo(1))+":"+platform.release() - self.sysInfo["GAVersion"]=GuestAgentVersion - self.sysInfo["RAM"]=MyDistro.getTotalMemory() - self.sysInfo["Processors"]=MyDistro.getProcessorCores() - sharedConfig = xml.dom.minidom.parse("/var/lib/waagent/SharedConfig.xml").childNodes[0] - hostEnvConfig= xml.dom.minidom.parse("/var/lib/waagent/HostingEnvironmentConfig.xml").childNodes[0] - gfiles = RunGetOutput("ls -t /var/lib/waagent/GoalState.*.xml")[1] - goalStateConfi = xml.dom.minidom.parse(gfiles.split("\n")[0]).childNodes[0] - self.sysInfo["TenantName"]=hostEnvConfig.getElementsByTagName("Deployment")[0].getAttribute("name") - self.sysInfo["RoleName"]=hostEnvConfig.getElementsByTagName("Role")[0].getAttribute("name") - self.sysInfo["RoleInstanceName"]=sharedConfig.getElementsByTagName("Instance")[0].getAttribute("id") - self.sysInfo["ContainerId"]=goalStateConfi.getElementsByTagName("ContainerId")[0].childNodes[0].nodeValue - except: - Error(traceback.format_exc()) - - eventObject = xml.dom.minidom.parseString(eventData.encode("utf-8")).childNodes[0] - for node in eventObject.childNodes: - if node.tagName == "Param": - name = node.getAttribute("Name") - if self.sysInfo.get(name): - node.setAttribute("Value",xml.sax.saxutils.escape(str(self.sysInfo[name]))) - - return eventObject.toxml() - - -class Agent(Util): - """ - Primary object container for the provisioning process. - - """ - def __init__(self): - self.GoalState = None - self.Endpoint = None - self.LoadBalancerProbeServer = None - self.HealthReportCounter = 0 - self.TransportCert = "" - self.EnvMonitor = None - self.SendData = None - self.DhcpResponse = None - - def CheckVersions(self): - """ - Query endpoint server for wire protocol version. - Fail if our desired protocol version is not seen. - """ - # - # - # - # 2010-12-15 - # - # - # 2010-12-15 - # 2010-28-10 - # - # - global ProtocolVersion - protocolVersionSeen = False - node = xml.dom.minidom.parseString(self.HttpGetWithoutHeaders("/?comp=versions")).childNodes[0] - if node.localName != "Versions": - Error("CheckVersions: root not Versions") - return False - for a in node.childNodes: - if a.nodeType == node.ELEMENT_NODE and a.localName == "Supported": - for b in a.childNodes: - if b.nodeType == node.ELEMENT_NODE and b.localName == "Version": - v = GetNodeTextData(b) - LogIfVerbose("Fabric supported wire protocol version: " + v) - if v == ProtocolVersion: - protocolVersionSeen = True - if a.nodeType == node.ELEMENT_NODE and a.localName == "Preferred": - v = GetNodeTextData(a.getElementsByTagName("Version")[0]) - Log("Fabric preferred wire protocol version: " + v) - if not protocolVersionSeen: - Warn("Agent supported wire protocol version: " + ProtocolVersion + " was not advertised by Fabric.") - else: - Log("Negotiated wire protocol version: " + ProtocolVersion) - return True - - def Unpack(self, buffer, offset, range): - """ - Unpack bytes into python values. - """ - result = 0 - for i in range: - result = (result << 8) | Ord(buffer[offset + i]) - return result - - def UnpackLittleEndian(self, buffer, offset, length): - """ - Unpack little endian bytes into python values. - """ - return self.Unpack(buffer, offset, list(range(length - 1, -1, -1))) - - def UnpackBigEndian(self, buffer, offset, length): - """ - Unpack big endian bytes into python values. - """ - return self.Unpack(buffer, offset, list(range(0, length))) - - def HexDump3(self, buffer, offset, length): - """ - Dump range of buffer in formatted hex. - """ - return ''.join(['%02X' % Ord(char) for char in buffer[offset:offset + length]]) - - def HexDump2(self, buffer): - """ - Dump buffer in formatted hex. - """ - return self.HexDump3(buffer, 0, len(buffer)) - - def BuildDhcpRequest(self): - """ - Build DHCP request string. - """ - # - # typedef struct _DHCP { - # UINT8 Opcode; /* op: BOOTREQUEST or BOOTREPLY */ - # UINT8 HardwareAddressType; /* htype: ethernet */ - # UINT8 HardwareAddressLength; /* hlen: 6 (48 bit mac address) */ - # UINT8 Hops; /* hops: 0 */ - # UINT8 TransactionID[4]; /* xid: random */ - # UINT8 Seconds[2]; /* secs: 0 */ - # UINT8 Flags[2]; /* flags: 0 or 0x8000 for broadcast */ - # UINT8 ClientIpAddress[4]; /* ciaddr: 0 */ - # UINT8 YourIpAddress[4]; /* yiaddr: 0 */ - # UINT8 ServerIpAddress[4]; /* siaddr: 0 */ - # UINT8 RelayAgentIpAddress[4]; /* giaddr: 0 */ - # UINT8 ClientHardwareAddress[16]; /* chaddr: 6 byte ethernet MAC address */ - # UINT8 ServerName[64]; /* sname: 0 */ - # UINT8 BootFileName[128]; /* file: 0 */ - # UINT8 MagicCookie[4]; /* 99 130 83 99 */ - # /* 0x63 0x82 0x53 0x63 */ - # /* options -- hard code ours */ - # - # UINT8 MessageTypeCode; /* 53 */ - # UINT8 MessageTypeLength; /* 1 */ - # UINT8 MessageType; /* 1 for DISCOVER */ - # UINT8 End; /* 255 */ - # } DHCP; - # - - # tuple of 244 zeros - # (struct.pack_into would be good here, but requires Python 2.5) - sendData = [0] * 244 - - transactionID = os.urandom(4) - macAddress = MyDistro.GetMacAddress() - - # Opcode = 1 - # HardwareAddressType = 1 (ethernet/MAC) - # HardwareAddressLength = 6 (ethernet/MAC/48 bits) - for a in range(0, 3): - sendData[a] = [1, 1, 6][a] - - # fill in transaction id (random number to ensure response matches request) - for a in range(0, 4): - sendData[4 + a] = Ord(transactionID[a]) - - LogIfVerbose("BuildDhcpRequest: transactionId:%s,%04X" % (self.HexDump2(transactionID), self.UnpackBigEndian(sendData, 4, 4))) - - # fill in ClientHardwareAddress - for a in range(0, 6): - sendData[0x1C + a] = Ord(macAddress[a]) - - # DHCP Magic Cookie: 99, 130, 83, 99 - # MessageTypeCode = 53 DHCP Message Type - # MessageTypeLength = 1 - # MessageType = DHCPDISCOVER - # End = 255 DHCP_END - for a in range(0, 8): - sendData[0xEC + a] = [99, 130, 83, 99, 53, 1, 1, 255][a] - return array.array("B", sendData) - - def IntegerToIpAddressV4String(self, a): - """ - Build DHCP request string. - """ - return "%u.%u.%u.%u" % ((a >> 24) & 0xFF, (a >> 16) & 0xFF, (a >> 8) & 0xFF, a & 0xFF) - - def RouteAdd(self, net, mask, gateway): - """ - Add specified route using /sbin/route add -net. - """ - net = self.IntegerToIpAddressV4String(net) - mask = self.IntegerToIpAddressV4String(mask) - gateway = self.IntegerToIpAddressV4String(gateway) - Run("/sbin/route add -net " + net + " netmask " + mask + " gw " + gateway,chk_err=False) - - def HandleDhcpResponse(self, sendData, receiveBuffer): - """ - Parse DHCP response: - Set default gateway. - Set default routes. - Retrieve endpoint server. - Returns endpoint server or None on error. - """ - LogIfVerbose("HandleDhcpResponse") - bytesReceived = len(receiveBuffer) - if bytesReceived < 0xF6: - Error("HandleDhcpResponse: Too few bytes received " + str(bytesReceived)) - return None - - LogIfVerbose("BytesReceived: " + hex(bytesReceived)) - LogWithPrefixIfVerbose("DHCP response:", HexDump(receiveBuffer, bytesReceived)) - - # check transactionId, cookie, MAC address - # cookie should never mismatch - # transactionId and MAC address may mismatch if we see a response meant from another machine - - for offsets in [list(range(4, 4 + 4)), list(range(0x1C, 0x1C + 6)), list(range(0xEC, 0xEC + 4))]: - for offset in offsets: - sentByte = Ord(sendData[offset]) - receivedByte = Ord(receiveBuffer[offset]) - if sentByte != receivedByte: - LogIfVerbose("HandleDhcpResponse: sent cookie:" + self.HexDump3(sendData, 0xEC, 4)) - LogIfVerbose("HandleDhcpResponse: rcvd cookie:" + self.HexDump3(receiveBuffer, 0xEC, 4)) - LogIfVerbose("HandleDhcpResponse: sent transactionID:" + self.HexDump3(sendData, 4, 4)) - LogIfVerbose("HandleDhcpResponse: rcvd transactionID:" + self.HexDump3(receiveBuffer, 4, 4)) - LogIfVerbose("HandleDhcpResponse: sent ClientHardwareAddress:" + self.HexDump3(sendData, 0x1C, 6)) - LogIfVerbose("HandleDhcpResponse: rcvd ClientHardwareAddress:" + self.HexDump3(receiveBuffer, 0x1C, 6)) - LogIfVerbose("HandleDhcpResponse: transactionId, cookie, or MAC address mismatch") - return None - endpoint = None - - # - # Walk all the returned options, parsing out what we need, ignoring the others. - # We need the custom option 245 to find the the endpoint we talk to, - # as well as, to handle some Linux DHCP client incompatibilities, - # options 3 for default gateway and 249 for routes. And 255 is end. - # - - i = 0xF0 # offset to first option - while i < bytesReceived: - option = Ord(receiveBuffer[i]) - length = 0 - if (i + 1) < bytesReceived: - length = Ord(receiveBuffer[i + 1]) - LogIfVerbose("DHCP option " + hex(option) + " at offset:" + hex(i) + " with length:" + hex(length)) - if option == 255: - LogIfVerbose("DHCP packet ended at offset " + hex(i)) - break - elif option == 249: - # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx - LogIfVerbose("Routes at offset:" + hex(i) + " with length:" + hex(length)) - if length < 5: - Error("Data too small for option " + str(option)) - j = i + 2 - while j < (i + length + 2): - maskLengthBits = Ord(receiveBuffer[j]) - maskLengthBytes = (((maskLengthBits + 7) & ~7) >> 3) - mask = 0xFFFFFFFF & (0xFFFFFFFF << (32 - maskLengthBits)) - j += 1 - net = self.UnpackBigEndian(receiveBuffer, j, maskLengthBytes) - net <<= (32 - maskLengthBytes * 8) - net &= mask - j += maskLengthBytes - gateway = self.UnpackBigEndian(receiveBuffer, j, 4) - j += 4 - self.RouteAdd(net, mask, gateway) - if j != (i + length + 2): - Error("HandleDhcpResponse: Unable to parse routes") - elif option == 3 or option == 245: - if i + 5 < bytesReceived: - if length != 4: - Error("HandleDhcpResponse: Endpoint or Default Gateway not 4 bytes") - return None - gateway = self.UnpackBigEndian(receiveBuffer, i + 2, 4) - IpAddress = self.IntegerToIpAddressV4String(gateway) - if option == 3: - self.RouteAdd(0, 0, gateway) - name = "DefaultGateway" - else: - endpoint = IpAddress - name = "Windows Azure wire protocol endpoint" - LogIfVerbose(name + ": " + IpAddress + " at " + hex(i)) - else: - Error("HandleDhcpResponse: Data too small for option " + str(option)) - else: - LogIfVerbose("Skipping DHCP option " + hex(option) + " at " + hex(i) + " with length " + hex(length)) - i += length + 2 - return endpoint - - def DoDhcpWork(self): - """ - Discover the wire server via DHCP option 245. - And workaround incompatibility with Windows Azure DHCP servers. - """ - ShortSleep = False # Sleep 1 second before retrying DHCP queries. - ifname=None - - sleepDurations = [0, 10, 30, 60, 60] - maxRetry = len(sleepDurations) - lastTry = (maxRetry - 1) - for retry in range(0, maxRetry): - try: - #Open DHCP port if iptables is enabled. - Run("iptables -D INPUT -p udp --dport 68 -j ACCEPT",chk_err=False) # We supress error logging on error. - Run("iptables -I INPUT -p udp --dport 68 -j ACCEPT",chk_err=False) # We supress error logging on error. - strRetry = str(retry) - prefix = "DoDhcpWork: try=" + strRetry - LogIfVerbose(prefix) - sendData = self.BuildDhcpRequest() - LogWithPrefixIfVerbose("DHCP request:", HexDump(sendData, len(sendData))) - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - missingDefaultRoute = True - try: - if DistInfo()[0] == 'FreeBSD': - missingDefaultRoute = True - else: - routes = RunGetOutput("route -n")[1] - for line in routes.split('\n'): - if line.startswith("0.0.0.0 ") or line.startswith("default "): - missingDefaultRoute = False - except: - pass - if missingDefaultRoute: - # This is required because sending after binding to 0.0.0.0 fails with - # network unreachable when the default gateway is not set up. - ifname=MyDistro.GetInterfaceName() - Log("DoDhcpWork: Missing default route - adding broadcast route for DHCP.") - if DistInfo()[0] == 'FreeBSD': - Run("route add -net 255.255.255.255 -iface " + ifname,chk_err=False) - else: - Run("route add 255.255.255.255 dev " + ifname,chk_err=False) - if MyDistro.isDHCPEnabled(): - MyDistro.stopDHCP() - sock.bind(("0.0.0.0", 68)) - sock.sendto(sendData, ("", 67)) - sock.settimeout(10) - Log("DoDhcpWork: Setting socket.timeout=10, entering recv") - receiveBuffer = sock.recv(1024) - endpoint = self.HandleDhcpResponse(sendData, receiveBuffer) - if endpoint == None: - LogIfVerbose("DoDhcpWork: No endpoint found") - if endpoint != None or retry == lastTry: - if endpoint != None: - self.SendData = sendData - self.DhcpResponse = receiveBuffer - if retry == lastTry: - LogIfVerbose("DoDhcpWork: try=" + strRetry) - return endpoint - sleepDuration = [sleepDurations[retry % len(sleepDurations)], 1][ShortSleep] - LogIfVerbose("DoDhcpWork: sleep=" + str(sleepDuration)) - time.sleep(sleepDuration) - except Exception, e: - ErrorWithPrefix(prefix, str(e)) - ErrorWithPrefix(prefix, traceback.format_exc()) - finally: - sock.close() - if missingDefaultRoute: - #We added this route - delete it - Log("DoDhcpWork: Removing broadcast route for DHCP.") - if DistInfo()[0] == 'FreeBSD': - Run("route del -net 255.255.255.255 -iface " + ifname,chk_err=False) - else: - Run("route del 255.255.255.255 dev " + ifname,chk_err=False) # We supress error logging on error. - if MyDistro.isDHCPEnabled(): - MyDistro.startDHCP() - return None - - def UpdateAndPublishHostName(self, name): - """ - Set hostname locally and publish to iDNS - """ - Log("Setting host name: " + name) - MyDistro.publishHostname(name) - ethernetInterface = MyDistro.GetInterfaceName() - MyDistro.RestartInterface(ethernetInterface) - self.RestoreRoutes() - - def RestoreRoutes(self): - """ - If there is a DHCP response, then call HandleDhcpResponse. - """ - if self.SendData != None and self.DhcpResponse != None: - self.HandleDhcpResponse(self.SendData, self.DhcpResponse) - - def UpdateGoalState(self): - """ - Retreive goal state information from endpoint server. - Parse xml and initialize Agent.GoalState object. - Return object or None on error. - """ - goalStateXml = None - maxRetry = 9 - log = NoLog - for retry in range(1, maxRetry + 1): - strRetry = str(retry) - log("retry UpdateGoalState,retry=" + strRetry) - goalStateXml = self.HttpGetWithHeaders("/machine/?comp=goalstate") - if goalStateXml != None: - break - log = Log - time.sleep(retry) - if not goalStateXml: - Error("UpdateGoalState failed.") - return - Log("Retrieved GoalState from Windows Azure Fabric.") - self.GoalState = GoalState(self).Parse(goalStateXml) - return self.GoalState - - def ReportReady(self): - """ - Send health report 'Ready' to server. - This signals the fabric that our provosion is completed, - and the host is ready for operation. - """ - counter = (self.HealthReportCounter + 1) % 1000000 - self.HealthReportCounter = counter - healthReport = ("" - + self.GoalState.Incarnation - + "" - + self.GoalState.ContainerId - + "" - + self.GoalState.RoleInstanceId - + "Ready") - a = self.HttpPostWithHeaders("/machine?comp=health", healthReport) - if a != None: - return a.getheader("x-ms-latest-goal-state-incarnation-number") - return None - - def ReportNotReady(self, status, desc): - """ - Send health report 'Provisioning' to server. - This signals the fabric that our provosion is starting. - """ - healthReport = ("" - + self.GoalState.Incarnation - + "" - + self.GoalState.ContainerId - + "" - + self.GoalState.RoleInstanceId - + "NotReady" - + "
" + status + "" + desc + "
" - + "") - a = self.HttpPostWithHeaders("/machine?comp=health", healthReport) - if a != None: - return a.getheader("x-ms-latest-goal-state-incarnation-number") - return None - - def ReportRoleProperties(self, thumbprint): - """ - Send roleProperties and thumbprint to server. - """ - roleProperties = ("" - + "" + self.GoalState.ContainerId + "" - + "" - + "" + self.GoalState.RoleInstanceId + "" - + "" - + "") - a = self.HttpPostWithHeaders("/machine?comp=roleProperties", - roleProperties) - Log("Posted Role Properties. CertificateThumbprint=" + thumbprint) - return a - - def LoadBalancerProbeServer_Shutdown(self): - """ - Shutdown the LoadBalancerProbeServer. - """ - if self.LoadBalancerProbeServer != None: - self.LoadBalancerProbeServer.shutdown() - self.LoadBalancerProbeServer = None - - def GenerateTransportCert(self): - """ - Create ssl certificate for https communication with endpoint server. - """ - Run(Openssl + " req -x509 -nodes -subj /CN=LinuxTransport -days 32768 -newkey rsa:2048 -keyout TransportPrivate.pem -out TransportCert.pem") - cert = "" - for line in GetFileContents("TransportCert.pem").split('\n'): - if not "CERTIFICATE" in line: - cert += line.rstrip() - return cert - - def DoVmmStartup(self): - """ - Spawn the VMM startup script. - """ - Log("Starting Microsoft System Center VMM Initialization Process") - pid = subprocess.Popen(["/bin/bash","/mnt/cdrom/secure/"+VMM_STARTUP_SCRIPT_NAME,"-p /mnt/cdrom/secure/ "]).pid - time.sleep(5) - sys.exit(0) - - def TryUnloadAtapiix(self): - """ - If global modloaded is True, then we loaded the ata_piix kernel module, unload it. - """ - if modloaded: - Run("rmmod ata_piix.ko",chk_err=False) - Log("Unloaded ata_piix.ko driver for ATAPI CD-ROM") - - def TryLoadAtapiix(self): - """ - Load the ata_piix kernel module if it exists. - If successful, set global modloaded to True. - If unable to load module leave modloaded False. - """ - global modloaded - modloaded=False - retcode,krn=RunGetOutput('uname -r') - krn_pth='/lib/modules/'+krn.strip('\n')+'/kernel/drivers/ata/ata_piix.ko' - if Run("lsmod | grep ata_piix",chk_err=False) == 0 : - Log("Module " + krn_pth + " driver for ATAPI CD-ROM is already present.") - return 0 - if retcode: - Error("Unable to provision: Failed to call uname -r") - return "Unable to provision: Failed to call uname" - if os.path.isfile(krn_pth): - retcode,output=RunGetOutput("insmod " + krn_pth,chk_err=False) - else: - Log("Module " + krn_pth + " driver for ATAPI CD-ROM does not exist.") - return 1 - if retcode != 0: - Error('Error calling insmod for '+ krn_pth + ' driver for ATAPI CD-ROM') - return retcode - time.sleep(1) - # check 3 times if the mod is loaded - for i in range(3): - if Run('lsmod | grep ata_piix'): - continue - else : - modloaded=True - break - if not modloaded: - Error('Unable to load '+ krn_pth + ' driver for ATAPI CD-ROM') - return 1 - - Log("Loaded " + krn_pth + " driver for ATAPI CD-ROM") - - # we have succeeded loading the ata_piix mod if it can be done. - - def SearchForVMMStartup(self): - """ - Search for a DVD/CDROM containing VMM's VMM_CONFIG_FILE_NAME. - Call TryLoadAtapiix in case we must load the ata_piix module first. - - If VMM_CONFIG_FILE_NAME is found, call DoVmmStartup. - Else, return to Azure Provisioning process. - """ - self.TryLoadAtapiix() - if os.path.exists('/mnt/cdrom/secure') == False: - CreateDir("/mnt/cdrom/secure", "root", 0700) - mounted=False - for dvds in [re.match(r'(sr[0-9]|hd[c-z]|cdrom[0-9]|cd[0-9]?)',x) for x in os.listdir('/dev/')]: - if dvds == None: - continue - dvd = '/dev/'+dvds.group(0) - if Run("LC_ALL=C fdisk -l " + dvd + " | grep Disk",chk_err=False): - continue # Not mountable - else: - for retry in range(1,6): - retcode,output=RunGetOutput("mount -v " + dvd + " /mnt/cdrom/secure") - Log(output[:-1]) - if retcode == 0: - Log("mount succeeded on attempt #" + str(retry) ) - mounted=True - break - if 'is already mounted on /mnt/cdrom/secure' in output: - Log("Device " + dvd + " is already mounted on /mnt/cdrom/secure." + str(retry) ) - mounted=True - break - Log("mount failed on attempt #" + str(retry) ) - Log("mount loop sleeping 5...") - time.sleep(5) - if not mounted: - # unable to mount - continue - if not os.path.isfile("/mnt/cdrom/secure/"+VMM_CONFIG_FILE_NAME): - #nope - mount the next drive - if mounted: - Run("umount "+dvd,chk_err=False) - mounted=False - continue - else : # it is the vmm startup - self.DoVmmStartup() - - Log("VMM Init script not found. Provisioning for Azure") - return - - def Provision(self): - """ - Responible for: - Regenerate ssh keys, - Mount, read, and parse ovfenv.xml from provisioning dvd rom - Process the ovfenv.xml info - Call ReportRoleProperties - If configured, delete root password. - Return None on success, error string on error. - """ - enabled = Config.get("Provisioning.Enabled") - if enabled != None and enabled.lower().startswith("n"): - return - Log("Provisioning image started.") - type = Config.get("Provisioning.SshHostKeyPairType") - if type == None: - type = "rsa" - regenerateKeys = Config.get("Provisioning.RegenerateSshHostKeyPair") - if regenerateKeys == None or regenerateKeys.lower().startswith("y"): - Run("rm -f /etc/ssh/ssh_host_*key*") - Run("ssh-keygen -N '' -t " + type + " -f /etc/ssh/ssh_host_" + type + "_key") - MyDistro.restartSshService() - #SetFileContents(LibDir + "/provisioned", "") - dvd = None - for dvds in [re.match(r'(sr[0-9]|hd[c-z]|cdrom[0-9]|cd[0-9]?)',x) for x in os.listdir('/dev/')]: - if dvds == None : - continue - dvd = '/dev/'+dvds.group(0) - if dvd == None: - # No DVD device detected - Error("No DVD device detected, unable to provision.") - return "No DVD device detected, unable to provision." - if MyDistro.mediaHasFilesystem(dvd) is False : - out=MyDistro.load_ata_piix() - if out: - return out - for i in range(10): # we may have to wait - if os.path.exists(dvd): - break - Log("Waiting for DVD - sleeping 1 - "+str(i+1)+" try...") - time.sleep(1) - if os.path.exists('/mnt/cdrom/secure') == False: - CreateDir("/mnt/cdrom/secure", "root", 0700) - #begin mount loop - 5 tries - 5 sec wait between - for retry in range(1,6): - location='/mnt/cdrom/secure' - retcode,output=MyDistro.mountDVD(dvd,location) - Log(output[:-1]) - if retcode == 0: - Log("mount succeeded on attempt #" + str(retry) ) - break - if 'is already mounted on /mnt/cdrom/secure' in output: - Log("Device " + dvd + " is already mounted on /mnt/cdrom/secure." + str(retry) ) - break - Log("mount failed on attempt #" + str(retry) ) - Log("mount loop sleeping 5...") - time.sleep(5) - if not os.path.isfile("/mnt/cdrom/secure/ovf-env.xml"): - Error("Unable to provision: Missing ovf-env.xml on DVD.") - return "Failed to retrieve provisioning data (0x02)." - ovfxml = (GetFileContents(u"/mnt/cdrom/secure/ovf-env.xml",asbin=False)) # use unicode here to ensure correct codec gets used. - if ord(ovfxml[0]) > 128 and ord(ovfxml[1]) > 128 and ord(ovfxml[2]) > 128 : - ovfxml = ovfxml[3:] # BOM is not stripped. First three bytes are > 128 and not unicode chars so we ignore them. - ovfxml=ovfxml.strip(chr(0x00)) # we may have NULLs. - ovfxml=ovfxml[ovfxml.find('.*?<", "*<", ovfxml)) - Run("umount " + dvd,chk_err=False) - MyDistro.unload_ata_piix() - error = None - if ovfxml != None: - Log("Provisioning image using OVF settings in the DVD.") - ovfobj = OvfEnv().Parse(ovfxml) - if ovfobj != None: - error = ovfobj.Process() - if error : - Error ("Provisioning image FAILED " + error) - return ("Provisioning image FAILED " + error) - Log("Ovf XML process finished") - # This is done here because regenerated SSH host key pairs may be potentially overwritten when processing the ovfxml - fingerprint = RunGetOutput("ssh-keygen -lf /etc/ssh/ssh_host_" + type + "_key.pub")[1].rstrip().split()[1].replace(':','') - self.ReportRoleProperties(fingerprint) - delRootPass = Config.get("Provisioning.DeleteRootPassword") - if delRootPass != None and delRootPass.lower().startswith("y"): - MyDistro.deleteRootPassword() - Log("Provisioning image completed.") - return error - - def Run(self): - """ - Called by 'waagent -daemon.' - Main loop to process the goal state. State is posted every 25 seconds - when provisioning has been completed. - - Search for VMM enviroment, start VMM script if found. - Perform DHCP and endpoint server discovery by calling DoDhcpWork(). - Check wire protocol versions. - Set SCSI timeout on root device. - Call GenerateTransportCert() to create ssl certs for server communication. - Call UpdateGoalState(). - If not provisioned, call ReportNotReady("Provisioning", "Starting") - Call Provision(), set global provisioned = True if successful. - Call goalState.Process() - Start LBProbeServer if indicated in waagent.conf. - Start the StateConsumer if indicated in waagent.conf. - ReportReady if provisioning is complete. - If provisioning failed, call ReportNotReady("ProvisioningFailed", provisionError) - """ - SetFileContents("/var/run/waagent.pid", str(os.getpid()) + "\n") - - # Determine if we are in VMM. Spawn VMM_STARTUP_SCRIPT_NAME if found. - self.SearchForVMMStartup() - ipv4='' - while ipv4 == '' or ipv4 == '0.0.0.0' : - ipv4=MyDistro.GetIpv4Address() - if ipv4 == '' or ipv4 == '0.0.0.0' : - Log("Waiting for network.") - time.sleep(10) - - Log("IPv4 address: " + ipv4) - mac='' - mac=MyDistro.GetMacAddress() - if len(mac)>0 : - Log("MAC address: " + ":".join(["%02X" % Ord(a) for a in mac])) - - # Consume Entropy in ACPI table provided by Hyper-V - try: - SetFileContents("/dev/random", GetFileContents("/sys/firmware/acpi/tables/OEM0")) - except: - pass - - Log("Probing for Windows Azure environment.") - self.Endpoint = self.DoDhcpWork() - - if self.Endpoint == None: - Log("Windows Azure environment not detected.") - while True: - time.sleep(60) - - Log("Discovered Windows Azure endpoint: " + self.Endpoint) - if not self.CheckVersions(): - Error("Agent.CheckVersions failed") - sys.exit(1) - - self.EnvMonitor = EnvMonitor() - - # Set SCSI timeout on SCSI disks - MyDistro.initScsiDiskTimeout() - global provisioned - global provisionError - - global Openssl - Openssl = Config.get("OS.OpensslPath") - if Openssl == None: - Openssl = "openssl" - - self.TransportCert = self.GenerateTransportCert() - - eventMonitor = None - incarnation = None # goalStateIncarnationFromHealthReport - currentPort = None # loadBalancerProbePort - goalState = None # self.GoalState, instance of GoalState - provisioned = os.path.exists(LibDir + "/provisioned") - program = Config.get("Role.StateConsumer") - provisionError = None - lbProbeResponder = True - setting = Config.get("LBProbeResponder") - if setting != None and setting.lower().startswith("n"): - lbProbeResponder = False - while True: - if (goalState == None) or (incarnation == None) or (goalState.Incarnation != incarnation): - try: - goalState = self.UpdateGoalState() - except HttpResourceGoneError as e: - Warn("Incarnation is out of date:{0}".format(e)) - incarnation = None - continue - - if goalState == None : - Warn("Failed to fetch goalstate") - continue - - if provisioned == False: - self.ReportNotReady("Provisioning", "Starting") - - goalState.Process() - - if provisioned == False: - provisionError = self.Provision() - if provisionError == None : - provisioned = True - SetFileContents(LibDir + "/provisioned", "") - lastCtime = "NOTFIND" - try: - walaConfigFile = MyDistro.getConfigurationPath() - lastCtime = time.ctime(os.path.getctime(walaConfigFile)) - except: - pass - #Get Ctime of wala config, can help identify the base image of this VM - AddExtensionEvent(name="WALA",op=WALAEventOperation.Provision,isSuccess=True, - message="WALA Config Ctime:"+lastCtime) - - executeCustomData = Config.get("Provisioning.ExecuteCustomData") - if executeCustomData != None and executeCustomData.lower().startswith("y"): - if os.path.exists(LibDir + '/CustomData'): - Run('chmod +x ' + LibDir + '/CustomData') - Run(LibDir + '/CustomData') - else: - Error(LibDir + '/CustomData does not exist.') - - # - # only one port supported - # restart server if new port is different than old port - # stop server if no longer a port - # - goalPort = goalState.LoadBalancerProbePort - if currentPort != goalPort: - try: - self.LoadBalancerProbeServer_Shutdown() - currentPort = goalPort - if currentPort != None and lbProbeResponder == True: - self.LoadBalancerProbeServer = LoadBalancerProbeServer(currentPort) - if self.LoadBalancerProbeServer == None : - lbProbeResponder = False - Log("Unable to create LBProbeResponder.") - except Exception, e: - Error("Failed to launch LBProbeResponder: {0}".format(e)) - currentPort = None - - # Report SSH key fingerprint - type = Config.get("Provisioning.SshHostKeyPairType") - if type == None: - type = "rsa" - - host_key_path = "/etc/ssh/ssh_host_" + type + "_key.pub" - if(MyDistro.waitForSshHostKey(host_key_path)): - fingerprint = RunGetOutput("ssh-keygen -lf /etc/ssh/ssh_host_" + type + "_key.pub")[1].rstrip().split()[1].replace(':','') - self.ReportRoleProperties(fingerprint) - - if program != None and DiskActivated == True: - try: - Children.append(subprocess.Popen([program, "Ready"])) - except OSError, e : - ErrorWithPrefix('SharedConfig.Parse','Exception: '+ str(e) +' occured launching ' + program ) - program = None - - sleepToReduceAccessDenied = 3 - time.sleep(sleepToReduceAccessDenied) - if provisionError != None: - incarnation = self.ReportNotReady("ProvisioningFailed", provisionError) - else: - incarnation = self.ReportReady() - # Process our extensions. - if goalState.ExtensionsConfig == None and goalState.ExtensionsConfigXml != None : - goalState.ExtensionsConfig = ExtensionsConfig().Parse(goalState.ExtensionsConfigXml) - - # report the status/heartbeat results of extension processing - if goalState.ExtensionsConfig != None : - goalState.ExtensionsConfig.ReportHandlerStatus() - - if not eventMonitor: - eventMonitor = WALAEventMonitor(self.HttpPostWithHeaders) - eventMonitor.StartEventsLoop() - - time.sleep(25 - sleepToReduceAccessDenied) - - -WaagentLogrotate = """\ -/var/log/waagent.log { - monthly - rotate 6 - notifempty - missingok -} -""" - -def GetMountPoint(mountlist, device): - """ - Example of mountlist: - /dev/sda1 on / type ext4 (rw) - proc on /proc type proc (rw) - sysfs on /sys type sysfs (rw) - devpts on /dev/pts type devpts (rw,gid=5,mode=620) - tmpfs on /dev/shm type tmpfs (rw,rootcontext="system_u:object_r:tmpfs_t:s0") - none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw) - /dev/sdb1 on /mnt/resource type ext4 (rw) - """ - if (mountlist and device): - for entry in mountlist.split('\n'): - if(re.search(device, entry)): - tokens = entry.split() - #Return the 3rd column of this line - return tokens[2] if len(tokens) > 2 else None - return None - -def FindInLinuxKernelCmdline(option): - """ - Return match object if 'option' is present in the kernel boot options - of the grub configuration. - """ - m=None - matchs=r'^.*?'+MyDistro.grubKernelBootOptionsLine+r'.*?'+option+r'.*$' - try: - m=FindStringInFile(MyDistro.grubKernelBootOptionsFile,matchs) - except IOError, e: - Error('FindInLinuxKernelCmdline: Exception opening ' + MyDistro.grubKernelBootOptionsFile + 'Exception:' + str(e)) - - return m - -def AppendToLinuxKernelCmdline(option): - """ - Add 'option' to the kernel boot options of the grub configuration. - """ - if not FindInLinuxKernelCmdline(option): - src=r'^(.*?'+MyDistro.grubKernelBootOptionsLine+r')(.*?)("?)$' - rep=r'\1\2 '+ option + r'\3' - try: - ReplaceStringInFile(MyDistro.grubKernelBootOptionsFile,src,rep) - except IOError, e : - Error('AppendToLinuxKernelCmdline: Exception opening ' + MyDistro.grubKernelBootOptionsFile + 'Exception:' + str(e)) - return 1 - Run("update-grub",chk_err=False) - return 0 - -def RemoveFromLinuxKernelCmdline(option): - """ - Remove 'option' to the kernel boot options of the grub configuration. - """ - if FindInLinuxKernelCmdline(option): - src=r'^(.*?'+MyDistro.grubKernelBootOptionsLine+r'.*?)('+option+r')(.*?)("?)$' - rep=r'\1\3\4' - try: - ReplaceStringInFile(MyDistro.grubKernelBootOptionsFile,src,rep) - except IOError, e : - Error('RemoveFromLinuxKernelCmdline: Exception opening ' + MyDistro.grubKernelBootOptionsFile + 'Exception:' + str(e)) - return 1 - Run("update-grub",chk_err=False) - return 0 - -def FindStringInFile(fname,matchs): - """ - Return match object if found in file. - """ - try: - ms=re.compile(matchs) - for l in (open(fname,'r')).readlines(): - m=re.search(ms,l) - if m: - return m - except: - raise - - return None - -def ReplaceStringInFile(fname,src,repl): - """ - Replace 'src' with 'repl' in file. - """ - try: - sr=re.compile(src) - if FindStringInFile(fname,src): - updated='' - for l in (open(fname,'r')).readlines(): - n=re.sub(sr,repl,l) - updated+=n - ReplaceFileContentsAtomic(fname,updated) - except : - raise - return - -def ApplyVNUMAWorkaround(): - """ - If kernel version has NUMA bug, add 'numa=off' to - kernel boot options. - """ - VersionParts = platform.release().replace('-', '.').split('.') - if int(VersionParts[0]) > 2: - return - if int(VersionParts[1]) > 6: - return - if int(VersionParts[2]) > 37: - return - if AppendToLinuxKernelCmdline("numa=off") == 0 : - Log("Your kernel version " + platform.release() + " has a NUMA-related bug: NUMA has been disabled.") - else : - "Error adding 'numa=off'. NUMA has not been disabled." - -def RevertVNUMAWorkaround(): - """ - Remove 'numa=off' from kernel boot options. - """ - if RemoveFromLinuxKernelCmdline("numa=off") == 0 : - Log('NUMA has been re-enabled') - else : - Log('NUMA has not been re-enabled') - -def Install(): - """ - Install the agent service. - Check dependencies. - Create /etc/waagent.conf and move old version to - /etc/waagent.conf.old - Copy RulesFiles to /var/lib/waagent - Create /etc/logrotate.d/waagent - Set /etc/ssh/sshd_config ClientAliveInterval to 180 - Call ApplyVNUMAWorkaround() - """ - if MyDistro.checkDependencies(): - return 1 - os.chmod(sys.argv[0], 0755) - SwitchCwd() - for a in RulesFiles: - if os.path.isfile(a): - if os.path.isfile(GetLastPathElement(a)): - os.remove(GetLastPathElement(a)) - shutil.move(a, ".") - Warn("Moved " + a + " -> " + LibDir + "/" + GetLastPathElement(a) ) - MyDistro.registerAgentService() - if os.path.isfile("/etc/waagent.conf"): - try: - os.remove("/etc/waagent.conf.old") - except: - pass - try: - os.rename("/etc/waagent.conf", "/etc/waagent.conf.old") - Warn("Existing /etc/waagent.conf has been renamed to /etc/waagent.conf.old") - except: - pass - SetFileContents("/etc/waagent.conf", MyDistro.waagent_conf_file) - SetFileContents("/etc/logrotate.d/waagent", WaagentLogrotate) - filepath = "/etc/ssh/sshd_config" - ReplaceFileContentsAtomic(filepath, "\n".join(filter(lambda a: not - a.startswith("ClientAliveInterval"), - GetFileContents(filepath).split('\n'))) + "\nClientAliveInterval 180\n") - Log("Configured SSH client probing to keep connections alive.") - ApplyVNUMAWorkaround() - return 0 - -def GetMyDistro(dist_class_name=''): - """ - Return MyDistro object. - NOTE: Logging is not initialized at this point. - """ - if dist_class_name == '': - if 'Linux' in platform.system(): - Distro=DistInfo()[0] - else : # I know this is not Linux! - if 'FreeBSD' in platform.system(): - Distro=platform.system() - Distro=Distro.strip('"') - Distro=Distro.strip(' ') - dist_class_name=Distro+'Distro' - else: - Distro=dist_class_name - if not globals().has_key(dist_class_name): - print Distro+' is not a supported distribution.' - return None - return globals()[dist_class_name]() # the distro class inside this module. - -def DistInfo(fullname=0): - if 'FreeBSD' in platform.system(): - release = re.sub('\-.*\Z', '', str(platform.release())) - distinfo = ['FreeBSD', release] - return distinfo - - if 'linux_distribution' in dir(platform): - distinfo = list(platform.linux_distribution(full_distribution_name=fullname)) - distinfo[0] = distinfo[0].strip() # remove trailing whitespace in distro name - return distinfo - else: - return platform.dist() - -def PackagedInstall(buildroot): - """ - Called from setup.py for use by RPM. - Generic implementation Creates directories and - files /etc/waagent.conf, /etc/init.d/waagent, /usr/sbin/waagent, - /etc/logrotate.d/waagent, /etc/sudoers.d/waagent under buildroot. - Copies generated files waagent.conf, into place and exits. - """ - MyDistro=GetMyDistro() - if MyDistro == None : - sys.exit(1) - MyDistro.packagedInstall(buildroot) - -def LibraryInstall(buildroot): - pass - -def Uninstall(): - """ - Uninstall the agent service. - Copy RulesFiles back to original locations. - Delete agent-related files. - Call RevertVNUMAWorkaround(). - """ - SwitchCwd() - for a in RulesFiles: - if os.path.isfile(GetLastPathElement(a)): - try: - shutil.move(GetLastPathElement(a), a) - Warn("Moved " + LibDir + "/" + GetLastPathElement(a) + " -> " + a ) - except: - pass - MyDistro.unregisterAgentService() - MyDistro.uninstallDeleteFiles() - RevertVNUMAWorkaround() - return 0 - -def Deprovision(force, deluser): - """ - Remove user accounts created by provisioning. - Disables root password if Provisioning.DeleteRootPassword = 'y' - Stop agent service. - Remove SSH host keys if they were generated by the provision. - Set hostname to 'localhost.localdomain'. - Delete cached system configuration files in /var/lib and /var/lib/waagent. - """ - - #Append blank line at the end of file, so the ctime of this file is changed every time - Run("echo ''>>"+ MyDistro.getConfigurationPath()) - - SwitchCwd() - ovfxml = GetFileContents(LibDir+"/ovf-env.xml") - ovfobj = None - if ovfxml != None: - ovfobj = OvfEnv().Parse(ovfxml, True) - - print("WARNING! The waagent service will be stopped.") - print("WARNING! All SSH host key pairs will be deleted.") - print("WARNING! Cached DHCP leases will be deleted.") - MyDistro.deprovisionWarnUser() - delRootPass = Config.get("Provisioning.DeleteRootPassword") - if delRootPass != None and delRootPass.lower().startswith("y"): - print("WARNING! root password will be disabled. You will not be able to login as root.") - - if ovfobj != None and deluser == True: - print("WARNING! " + ovfobj.UserName + " account and entire home directory will be deleted.") - - if force == False and not raw_input('Do you want to proceed (y/n)? ').startswith('y'): - return 1 - - MyDistro.stopAgentService() - - # Remove SSH host keys - regenerateKeys = Config.get("Provisioning.RegenerateSshHostKeyPair") - if regenerateKeys == None or regenerateKeys.lower().startswith("y"): - Run("rm -f /etc/ssh/ssh_host_*key*") - - # Remove root password - if delRootPass != None and delRootPass.lower().startswith("y"): - MyDistro.deleteRootPassword() - # Remove distribution specific networking configuration - - MyDistro.publishHostname('localhost.localdomain') - MyDistro.deprovisionDeleteFiles() - if deluser == True: - MyDistro.DeleteAccount(ovfobj.UserName) - return 0 - -def SwitchCwd(): - """ - Switch to cwd to /var/lib/waagent. - Create if not present. - """ - CreateDir(LibDir, "root", 0700) - os.chdir(LibDir) - -def Usage(): - """ - Print the arguments to waagent. - """ - print("usage: " + sys.argv[0] + " [-verbose] [-force] [-help|-install|-uninstall|-deprovision[+user]|-version|-serialconsole|-daemon]") - return 0 - - - -def main(): - """ - Instantiate MyDistro, exit if distro class is not defined. - Parse command-line arguments, exit with usage() on error. - Instantiate ConfigurationProvider. - Call appropriate non-daemon methods and exit. - If daemon mode, enter Agent.Run() loop. - """ - if GuestAgentVersion == "": - print("WARNING! This is a non-standard agent that does not include a valid version string.") - - if len(sys.argv) == 1: - sys.exit(Usage()) - - LoggerInit('/var/log/waagent.log','/dev/console') - global LinuxDistro - LinuxDistro=DistInfo()[0] - - #The platform.py lib has issue with detecting oracle linux distribution. - #Merge the following patch provided by oracle as a temparory fix. - if os.path.exists("/etc/oracle-release"): - LinuxDistro="Oracle Linux" - - global MyDistro - MyDistro=GetMyDistro() - if MyDistro == None : - sys.exit(1) - args = [] - conf_file = None - global force - force = False - for a in sys.argv[1:]: - if re.match("^([-/]*)(help|usage|\?)", a): - sys.exit(Usage()) - elif re.match("^([-/]*)version", a): - print(GuestAgentVersion + " running on " + LinuxDistro) - sys.exit(0) - elif re.match("^([-/]*)verbose", a): - myLogger.verbose = True - elif re.match("^([-/]*)force", a): - force = True - elif re.match("^(?:[-/]*)conf=.+", a): - conf_file = re.match("^(?:[-/]*)conf=(.+)", a).groups()[0] - elif re.match("^([-/]*)(setup|install)", a): - sys.exit(MyDistro.Install()) - elif re.match("^([-/]*)(uninstall)", a): - sys.exit(Uninstall()) - else: - args.append(a) - global Config - Config = ConfigurationProvider(conf_file) - - logfile = Config.get("Logs.File") - if logfile is not None: - myLogger.file_path = logfile - logconsole = Config.get("Logs.Console") - if logconsole is not None and logconsole.lower().startswith("n"): - myLogger.con_path = None - verbose = Config.get("Logs.Verbose") - if verbose != None and verbose.lower().startswith("y"): - myLogger.verbose=True - global daemon - daemon = False - for a in args: - if re.match("^([-/]*)deprovision\+user", a): - sys.exit(Deprovision(force, True)) - elif re.match("^([-/]*)deprovision", a): - sys.exit(Deprovision(force, False)) - elif re.match("^([-/]*)daemon", a): - daemon = True - elif re.match("^([-/]*)serialconsole", a): - AppendToLinuxKernelCmdline("console=ttyS0 earlyprintk=ttyS0") - Log("Configured kernel to use ttyS0 as the boot console.") - sys.exit(0) - else: - print("Invalid command line parameter:" + a) - sys.exit(1) - - if daemon == False: - sys.exit(Usage()) - global modloaded - modloaded = False - try: - SwitchCwd() - Log(GuestAgentLongName + " Version: " + GuestAgentVersion) - if IsLinux(): - Log("Linux Distribution Detected : " + LinuxDistro) - global WaAgent - WaAgent = Agent() - WaAgent.Run() - except Exception, e: - Error(traceback.format_exc()) - Error("Exception: " + str(e)) - sys.exit(1) - -if __name__ == '__main__' : - main() diff --git a/bin/waagent2 b/bin/waagent2 new file mode 100755 index 0000000..92486f2 --- /dev/null +++ b/bin/waagent2 @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# +# Windows Azure Linux Agent +# +# Copyright 2015 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.6+ and Openssl 1.0+ +# +# Implements parts of RFC 2131, 1541, 1497 and +# http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx +# http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx +# + +if __name__ == '__main__' : + import sys + import azurelinuxagent.agent as agent + agent.main() + sys.exit() + diff --git a/bin/waagent3 b/bin/waagent3 new file mode 100755 index 0000000..211a2bb --- /dev/null +++ b/bin/waagent3 @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# +# Windows Azure Linux Agent +# +# Copyright 2015 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.6+ and Openssl 1.0+ +# +# Implements parts of RFC 2131, 1541, 1497 and +# http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx +# http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx +# + +if __name__ == '__main__' : + import sys + import azurelinuxagent.agent as agent + agent.main() + sys.exit() + diff --git a/setup.py b/setup.py index 90777b6..65abd8e 100755 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ def get_data_files(name, version, fullname): #Script file script_dest = '/usr/sbin' - script_src = ['bin/waagent'] + script_src = ['bin/waagent', 'bin/waagent2', 'bin/waagent3'] if name == 'coreos': script_dest = '/usr/share/oem/bin' data_files.append((script_dest, script_src)) diff --git a/tests/test_agent.py b/tests/test_agent.py index 8e14f14..0236ffc 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env import tests.tools as tools import uuid import unittest diff --git a/tests/test_certificates.py b/tests/test_certificates.py index 6aae434..ba81fd3 100644 --- a/tests/test_certificates.py +++ b/tests/test_certificates.py @@ -18,8 +18,8 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env -from tools import * +from . import env +from .tools import * import uuid import unittest import os diff --git a/tests/test_conf.py b/tests/test_conf.py index 9ace231..54d50f4 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -19,7 +19,7 @@ # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx import os -import env +from . import env import uuid import unittest import tests.tools as tools diff --git a/tests/test_deprovision.py b/tests/test_deprovision.py index 8cd4104..7824660 100644 --- a/tests/test_deprovision.py +++ b/tests/test_deprovision.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env from tests.tools import * import unittest import azurelinuxagent.distro.default.deprovision as deprovision_handler diff --git a/tests/test_dhcp.py b/tests/test_dhcp.py index c2c86de..0850fbb 100644 --- a/tests/test_dhcp.py +++ b/tests/test_dhcp.py @@ -18,8 +18,8 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env -from tools import * +from . import env +from .tools import * import uuid import unittest import os @@ -28,7 +28,7 @@ import azurelinuxagent.utils.fileutil as fileutil import azurelinuxagent.distro.default.dhcp as dhcp_handler SampleDhcpResponse = None -with open(os.path.join(env.test_root, "dhcp")) as F: +with open(os.path.join(env.test_root, "dhcp"), 'rb') as F: SampleDhcpResponse = F.read() mock_socket_send = MockFunc('socket_send', SampleDhcpResponse) diff --git a/tests/test_distroLoader.py b/tests/test_distroLoader.py index 2601a7c..fdf700b 100644 --- a/tests/test_distroLoader.py +++ b/tests/test_distroLoader.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env from tests.tools import * import unittest from azurelinuxagent.utils.osutil import OSUTIL, OSUtilError diff --git a/tests/test_envmon.py b/tests/test_envmon.py index f4ef511..fc1240b 100644 --- a/tests/test_envmon.py +++ b/tests/test_envmon.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env from tests.tools import * import unittest import time diff --git a/tests/test_event.py b/tests/test_event.py index f8503a9..2f75526 100644 --- a/tests/test_event.py +++ b/tests/test_event.py @@ -18,8 +18,8 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env -from tools import * +from . import env +from .tools import * import uuid import unittest import os diff --git a/tests/test_ext.py b/tests/test_ext.py index 2ee8d25..a4e6908 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env from tests.tools import * import uuid import unittest diff --git a/tests/test_extensionsconfig.py b/tests/test_extensionsconfig.py index be97e6e..9ca7416 100644 --- a/tests/test_extensionsconfig.py +++ b/tests/test_extensionsconfig.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env import tests.tools as tools import uuid import unittest diff --git a/tests/test_file_util.py b/tests/test_file_util.py index 7d3ac3d..d95937d 100644 --- a/tests/test_file_util.py +++ b/tests/test_file_util.py @@ -18,22 +18,36 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +import tests.env import tests.tools as tools import uuid import unittest import os +import sys import azurelinuxagent.utils.fileutil as fileutil -import test class TestFileOperations(unittest.TestCase): - def test_get_set_file_contents(self): + def test_read_write_file(self): test_file='/tmp/test_file' content = str(uuid.uuid4()) fileutil.write_file(test_file, content) self.assertTrue(tools.simple_file_grep(test_file, content)) - self.assertEquals(content, fileutil.read_file('/tmp/test_file')) + + content_read = fileutil.read_file('/tmp/test_file') + print(type(content_read)) + self.assertEquals(content, content_read) os.remove(test_file) + + def test_rw_utf8_file(self): + test_file='/tmp/test_file3' + content = "\u6211" + fileutil.write_file(test_file, content) + self.assertTrue(tools.simple_file_grep(test_file, content)) + + content_read = fileutil.read_file('/tmp/test_file3') + self.assertEquals(content, content_read) + os.remove(test_file) + def test_append_file(self): test_file='/tmp/test_file2' @@ -42,17 +56,6 @@ class TestFileOperations(unittest.TestCase): self.assertTrue(tools.simple_file_grep(test_file, content)) os.remove(test_file) - def test_replace_file(self): - test_file='/tmp/test_file3' - old_content = str(uuid.uuid4()) - content = str(uuid.uuid4()) - with open(test_file, "a+") as F: - F.write(old_content) - fileutil.replace_file(test_file, content) - self.assertFalse(tools.simple_file_grep(test_file, old_content)) - self.assertTrue(tools.simple_file_grep(test_file, content)) - os.remove(test_file) - def test_get_last_path_element(self): filepath = '/tmp/abc.def' filename = fileutil.base_name(filepath) diff --git a/tests/test_goalstate.py b/tests/test_goalstate.py index f82fb96..66cc78e 100644 --- a/tests/test_goalstate.py +++ b/tests/test_goalstate.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env import tests.tools as tools import uuid import unittest diff --git a/tests/test_hostingenv.py b/tests/test_hostingenv.py index 2e2cf04..03651f6 100644 --- a/tests/test_hostingenv.py +++ b/tests/test_hostingenv.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env import tests.tools as tools import uuid import unittest diff --git a/tests/test_logger.py b/tests/test_logger.py index dc37bbe..66a8935 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env import tests.tools as tools import uuid import unittest @@ -45,8 +45,8 @@ class TestLogger(unittest.TestCase): _logger.info("{0} {1}", 0, 1) _logger.warn("{0} {1}", 0, 1) _logger.error("{0} {1}", 0, 1) - _logger.info("this is a unicode {0}", u'\u6211') - _logger.info("this is a utf-8 {0}", u'\u6211'.encode('utf-8')) + _logger.info("this is a unicode {0}", '\u6211') + _logger.info("this is a utf-8 {0}", '\u6211'.encode('utf-8')) _logger.info("this is a gbk {0}", 0xff ) def test_file_appender(self): @@ -63,10 +63,25 @@ class TestLogger(unittest.TestCase): _logger.verbose("Verbose should not be logged: {0}", msg) self.assertFalse(tools.simple_file_grep('/tmp/testlog', msg)) - _logger.info("this is a unicode {0}", u'\u6211') - _logger.info("this is a utf-8 {0}", u'\u6211'.encode('utf-8')) + _logger.info("this is a unicode {0}", '\u6211') + _logger.info("this is a utf-8 {0}", '\u6211'.encode('utf-8')) _logger.info("this is a gbk {0}", 0xff) + def test_concole_appender(self): + _logger = logger.Logger() + _logger.add_appender(logger.AppenderType.CONSOLE, + logger.LogLevel.VERBOSE, + '/tmp/testlog') + + msg = str(uuid.uuid4()) + _logger.info("Test logger: {0}", msg) + self.assertTrue(tools.simple_file_grep('/tmp/testlog', msg)) + + msg = str(uuid.uuid4()) + _logger.verbose("Test logger: {0}", msg) + self.assertFalse(tools.simple_file_grep('/tmp/testlog', msg)) + + def test_log_to_non_exists_dev(self): _logger = logger.Logger() _logger.add_appender(logger.AppenderType.CONSOLE, diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 12bb671..a850030 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env from tests.tools import * import unittest from azurelinuxagent.metadata import AGENT_NAME, AGENT_VERSION, \ diff --git a/tests/test_osutil.py b/tests/test_osutil.py index 2ea7b89..daadd0b 100644 --- a/tests/test_osutil.py +++ b/tests/test_osutil.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env from tests.tools import * import uuid import unittest diff --git a/tests/test_ovfxml.py b/tests/test_ovfxml.py index 8235913..ac173e3 100644 --- a/tests/test_ovfxml.py +++ b/tests/test_ovfxml.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env import tests.tools as tools import uuid import unittest diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 113388c..674ad15 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -18,15 +18,15 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env import tests.tools as tools -from tools import * +from .tools import * import uuid import unittest import os import time import json -import httplib +import http.client from azurelinuxagent.protocol.common import * extensionDataStr = """ diff --git a/tests/test_protocolFactory.py b/tests/test_protocolFactory.py index 0db8d4a..7ffba73 100644 --- a/tests/test_protocolFactory.py +++ b/tests/test_protocolFactory.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env from tests.tools import * import uuid import unittest diff --git a/tests/test_redhat.py b/tests/test_redhat.py index 2ad39de..91555f2 100644 --- a/tests/test_redhat.py +++ b/tests/test_redhat.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env from tests.tools import * import unittest from azurelinuxagent.distro.redhat.osutil import RedhatOSUtil diff --git a/tests/test_resourcedisk.py b/tests/test_resourcedisk.py index c8a903c..804593c 100644 --- a/tests/test_resourcedisk.py +++ b/tests/test_resourcedisk.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env from tests.tools import * import unittest import azurelinuxagent.distro.default.resourceDisk as rdh diff --git a/tests/test_rest_util.py b/tests/test_rest_util.py index 89add19..c079f96 100644 --- a/tests/test_rest_util.py +++ b/tests/test_rest_util.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env from tests.tools import * import uuid import unittest diff --git a/tests/test_sharedconfig.py b/tests/test_sharedconfig.py index af220c2..f9bb3e1 100644 --- a/tests/test_sharedconfig.py +++ b/tests/test_sharedconfig.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env import tests.tools as tools import uuid import unittest diff --git a/tests/test_shell_util.py b/tests/test_shell_util.py index d2058d3..1be966a 100644 --- a/tests/test_shell_util.py +++ b/tests/test_shell_util.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env import tests.tools as tools import uuid import unittest diff --git a/tests/test_text_util.py b/tests/test_text_util.py index ebe0973..972d1db 100644 --- a/tests/test_text_util.py +++ b/tests/test_text_util.py @@ -18,8 +18,8 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env -from tools import * +from . import env +from .tools import * import uuid import unittest import os diff --git a/tests/test_v1.py b/tests/test_v1.py index a526df2..cfe0650 100644 --- a/tests/test_v1.py +++ b/tests/test_v1.py @@ -18,22 +18,22 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env import tests.tools as tools -from tools import * +from .tools import * import uuid import unittest import os import time -import httplib +import http.client import azurelinuxagent.logger as logger import azurelinuxagent.protocol.v1 as v1 -from test_version import VersionInfoSample -from test_goalstate import goal_state_sample -from test_hostingenv import hosting_env_sample -from test_sharedconfig import shared_config_sample -from test_certificates import certs_sample, transport_cert -from test_extensionsconfig import ext_conf_sample, manifest_sample +from .test_version import VersionInfoSample +from .test_goalstate import goal_state_sample +from .test_hostingenv import hosting_env_sample +from .test_sharedconfig import shared_config_sample +from .test_certificates import certs_sample, transport_cert +from .test_extensionsconfig import ext_conf_sample, manifest_sample #logger.LoggerInit("/dev/stdout", "/dev/null", verbose=True) #logger.LoggerInit("/dev/stdout", "/dev/null", verbose=False) @@ -130,12 +130,12 @@ class TestStatusBlob(unittest.TestCase): status_blob = v1.StatusBlob(vm_status) self.assertNotEquals(None, status_blob.to_json()) - @mock(v1.restutil, 'http_put', MockFunc(retval=MockResp(httplib.CREATED))) - @mock(v1.restutil, 'http_head', MockFunc(retval=MockResp(httplib.OK))) + @mock(v1.restutil, 'http_put', MockFunc(retval=MockResp(http.client.CREATED))) + @mock(v1.restutil, 'http_head', MockFunc(retval=MockResp(http.client.OK))) def test_put_page_blob(self): vm_status = v1.VMStatus() status_blob = v1.StatusBlob(vm_status) - data = ['a'] * 100 + data = 'a' * 100 status_blob.put_page_blob("http://foo.bar", data) class TestConvert(unittest.TestCase): diff --git a/tests/test_version.py b/tests/test_version.py index 6a4dc7c..057feae 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -18,7 +18,7 @@ # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -import env +from . import env import tests.tools as tools import uuid import unittest