Fix unit test

test/test_extensionsconfig.py

@@ -64,7 +64,7 @@ ExtensionsConfigSample="""\
   </GAFamilies>
 </GuestAgentExtension>
 <Plugins>
-  <Plugin name="OSTCExtensions.ExampleHandlerLinux" version="1.4" location="http://rdfepirv2hknprdstr03.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_CustomScriptForLinuxTest_asiaeast_manifest.xml" config="" state="enabled" autoUpgrade="false" failoverlocation="http://rdfepirv2hknprdstr04.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_CustomScriptForLinuxTest_asiaeast_manifest.xml" runAsStartupTask="false" isJson="true" />
+  <Plugin name="OSTCExtensions.ExampleHandlerLinux" version="1.4" location="http://rdfepirv2hknprdstr03.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_CustomScriptForLinuxTest_asiaeast_manifest.xml" config="" state="enabled" autoUpgrade="true" failoverlocation="http://rdfepirv2hknprdstr04.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_CustomScriptForLinuxTest_asiaeast_manifest.xml" runAsStartupTask="false" isJson="true" />
 </Plugins>
 <PluginSettings>
   <Plugin name="OSTCExtensions.ExampleHandlerLinux" version="1.4">
@@ -84,6 +84,12 @@ ManifestSample="""\
 <Uri>http://blahblah</Uri>
 </Uris>
 </Plugin>
+<Plugin>
+<Version>1.1</Version>
+<Uris>
+<Uri>http://blahblah</Uri>
+</Uris>
+</Plugin>
 </Plugins>
 <InternalPlugins />
 </PluginVersionManifest>
@@ -100,12 +106,14 @@ class TestExtensionsConfig(unittest.TestCase):
         self.assertEquals("OSTCExtensions.ExampleHandlerLinux", 
                           extension.getName())
         self.assertEquals("1.4", extension.getVersion())
-        self.assertEquals(None, extension.getUpgradePolicy())
+        self.assertEquals('auto', extension.getUpgradePolicy())
         self.assertEquals("enabled", extension.getState())
         self.assertEquals("4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", 
                           extension.getCertificateThumbprint())
         self.assertEquals("MIICWgYJK", extension.getProtectedSettings())
-        self.assertEquals(json.loads('{"foo":"bar"}'), extension.getPublicSettings())
+        self.assertEquals(json.loads('{"foo":"bar"}'), 
+                          extension.getPublicSettings())
+
    
 if __name__ == '__main__':
     unittest.main()

test/test_logger.py

@@ -68,7 +68,7 @@ class TestLogger(unittest.TestCase):
     def test_logger_init(self):
         _logger = logger.Logger()
         logger.LoggerInit('/tmp/testlog1', '/tmp/testconsole', logger = _logger)
-        self.assertEquals(2, len(_logger.appenders))
+        self.assertEquals(3, len(_logger.appenders))
 
         msg = str(uuid.uuid4())
         _logger.info("Test logger: {0}", msg)

test/test_rest_util.py

@@ -51,16 +51,16 @@ class TestHttpOperations(unittest.TestCase):
         self.assertEquals(True, secure)
 
     def test_http_get(self):
-        resp = restutil.HttpGet("http://httpbin.org/get")
+        resp = restutil.HttpGet("http://httpbin.org/get").read()
         self.assertNotEquals(None, resp)
        
         msg = str(uuid.uuid4())
-        resp = restutil.HttpGet("http://httpbin.org/get", {"x-abc":msg})
+        resp = restutil.HttpGet("http://httpbin.org/get", {"x-abc":msg}).read()
         self.assertNotEquals(None, resp)
         self.assertTrue(msg in resp)
 
     def test_https_get(self):
-        resp = restutil.HttpGet("https://httpbin.org/get")
+        resp = restutil.HttpGet("https://httpbin.org/get").read()
         self.assertNotEquals(None, resp)
 
 if __name__ == '__main__':

test/test_v1.py

@@ -36,22 +36,30 @@ from test_certificates import CertificatesSample
 from test_extensionsconfig import ExtensionsConfigSample, ManifestSample
 
 def MockHttpGet(url, headers=None, maxRetry=1):
+    content = None
     if "versions" in url:
-        return VersionInfoSample
+        content = VersionInfoSample
     elif "goalstate" in url:
-        return GoalStateSample
+        content = GoalStateSample
     elif "hostingenvuri" in url:
-        return HostingEnvSample
+        content = HostingEnvSample
     elif "sharedconfiguri" in url:
-        return SharedConfigSample
+        content = SharedConfigSample
     elif "certificatesuri" in url:
-        return CertificatesSample
+        content = CertificatesSample
     elif "extensionsconfiguri" in url:
-        return ExtensionsConfigSample
+        content = ExtensionsConfigSample
     elif "manifest.xml" in url:
-        return ManifestSample
+        content = ManifestSample
     else:
         raise Exception("Bad url {0}".format(url))
+    return MockResp(content)
+
+class MockResp(object):
+    def __init__(self, content):
+        self.content = content
+    def read(self):
+        return self.content
 
 MockHttpPut = MockFunc('HttpPut')
 MockHttpPost = MockFunc('HttpPost')
@@ -113,6 +121,8 @@ class TestProtocolV1(unittest.TestCase):
         self.assertNotEquals(None, certs)
         extensions = p.getExtensions()
         self.assertNotEquals(None, extensions)
+        ext = extensions[0]
+        self.assertEquals('1.1', ext.getTargetVersion('1.4')['version'])
    
     @Mockup(v1.restutil, 'HttpPost', MockHttpPost)
     def test_report_provision_status(self):

walinuxagent/agent.py

@@ -21,6 +21,8 @@ import os
 import sys
 import re
 import shutil
+import time
+import traceback
 import walinuxagent.logger as logger
 import walinuxagent.conf as conf
 from walinuxagent.utils.osutil import CurrOS, CurrOSInfo
@@ -30,6 +32,7 @@ import walinuxagent.protocol.detection as proto
 import walinuxagent.dhcphandler as dhcp
 import walinuxagent.envmonitor as envmon
 import walinuxagent.extension as ext
+import walinuxagent.provision as provision
 
 GuestAgentName = "WALinuxAgent"
 GuestAgentLongName = "Microsoft Azure Linux Agent"
@@ -75,14 +78,14 @@ class Agent():
 
         provisoned = os.path.join(CurrOS.GetLibDir(), "provisioned")
         if(not os.path.isfile(provisoned)):
-            provisionHandler = ProvisionHandler(self.config, 
+            provisionHandler = provision.ProvisionHandler(self.config, 
                                                 self.protocol, 
                                                 self.envmonitor)
             try:
                 provisionHandler.provision()
-                fileutil.SetFileContents(provisoned)
+                fileutil.SetFileContents(provisoned, "")
             except Exception, e:
-                protocol.reportAgentStatus(GuestAgentVersion, 
+                self.protocol.reportAgentStatus(GuestAgentVersion, 
                                            "NotReady", 
                                            "ProvisioningFailed")
                 raise e
@@ -102,10 +105,12 @@ class Agent():
             agentStatusDetail = "Guest Agent is running"
             #Handle extensions
             try:
-                exthandler = ext.ExtensionHandler()
+                exthandler = ext.ExtensionHandler(self.config, self.protocol)
                 exthandler.process()
             except Exception, e:
-                logger.Error("Failed to handle extensions: {0}", e)
+                logger.Error("Failed to handle extensions: {0} {1}", 
+                             e,
+                             traceback.format_exc())
             self.protocol.reportAgentStatus(GuestAgentVersion, 
                                             agentStatus,
                                             agentStatusDetail)
@@ -224,11 +229,14 @@ def Main():
     elif command == "deprovision":
         Deprovision(force=force, deluser=False)
     elif command == "daemon":
-        logger.LoggerInit('/var/log/waagent.log', '/dev/console')
-        fileutil.CreateDir(CurrOS.GetLibDir(), 'root', '0700')
-        os.chdir(CurrOS.GetLibDir())
         configPath = CurrOS.GetConfigurationPath()
         config = conf.LoadConfiguration(configPath)
+        verbose = config.getSwitch("Logs.Verbose", False)
+        logger.LoggerInit('/var/log/waagent.log', 
+                          '/dev/console',
+                          verbose=verbose)
+        fileutil.CreateDir(CurrOS.GetLibDir(), 'root', '0700')
+        os.chdir(CurrOS.GetLibDir())
         Agent(config).run()
     elif command == "serialconsole":
         #TODO

walinuxagent/extension.py

@@ -16,11 +16,20 @@
 #
 # Requires Python 2.4+ and Openssl 1.0+
 #
+import os
+import traceback
+import zipfile
+import json
+import subprocess
+import walinuxagent.logger as logger
+import walinuxagent.utils.fileutil as fileutil
+import walinuxagent.utils.restutil as restutil
 from walinuxagent.utils.osutil import CurrOSInfo, CurrOS
 
 class ExtensionHandler(object):
-    def __init__(self, config):
+    def __init__(self, config, protocol):
         self.config = config
+        self.protocol = protocol
 
     def process(self):
         extSettings = self.protocol.getExtensions()
@@ -31,10 +40,17 @@ class ExtensionHandler(object):
             try:
                 ext.handle()
             except Exception, e:
-                logger.Error("Failed to handle extension: {0}-{1}, {2}", 
+                logger.Error("Failed to handle extension: {0}-{1}, {2}, {3}", 
                              setting.getName(),
                              setting.getVersion(),
-                             e)
+                             e,
+                             traceback.format_exc())
+
+def ParseExtensionDirName(dirName):
+    seprator = dirName.rfind('-')
+    if seprator < 0:
+        raise Exception("Invalid extenation dir name")
+    return dirName[0:seprator], dirName[seprator + 1:]
 
 def LoadExtensionInstance(setting):
     """
@@ -42,7 +58,7 @@ def LoadExtensionInstance(setting):
     """
     targetName = setting.getName()
     for dirName in os.listdir(CurrOS.GetLibDir()):
-        if dirName.startswith(targetName):
+        if os.path.isdir(dirName) and dirName.startswith(targetName):
             name, version = ParseExtensionDirName(dirName)
             #Here we need to ensure names are exactly the same.
             if name == targetName:
@@ -65,6 +81,9 @@ class ExtensionInstance(object):
         }))
  
     def handle(self):
+        self.logger.info("Process extension:{0} {1}", 
+                    self.setting.getName(),
+                    self.setting.getVersion())
         state = self.setting.getState()
         if state == 'enabled':
             self.handleEnable()
@@ -113,64 +132,82 @@ class ExtensionInstance(object):
         new.enable()
 
     def download(self):
-        uris = self.getPackageUris()
+        uris = self.setting.getPackageUris()
         package = None
         for uri in uris:
             try:
-                package = restutil.HttpGet(uri)
-                break
+                resp = restutil.HttpGet(uri)
+                if resp is not None:
+                    package = resp.read()
+                    break
             except Exception, e:
-                logger.Warn("Unable to download extension from: {0}", uri)
+                self.logger.warn("Unable to download extension from: {0}", uri)
         if package is None:
             raise Exception("Download extension failed")
 
         #Unpack the package
         packageFile = os.path.join(CurrOS.GetLibDir(),
                                    os.path.basename(uri) + ".zip")
-        fileutil.SetFileContents(packageFile, package)
+        fileutil.SetFileContents(packageFile, bytearray(package))
         baseDir = self.setting.getBaseDir()
         zipfile.ZipFile(packageFile).extractall(baseDir)
         
         #Save manifest
-        manFile = fileutil.SearchFor(baseDir, 'HandlerManifest.json')
+        manFile = fileutil.SearchForFile(baseDir, 'HandlerManifest.json')
         man = fileutil.GetFileContents(manFile, removeBom=True)
         fileutil.SetFileContents(self.setting.getManifestFile(), man)    
 
         #Create status and config dir
         statusDir = self.setting.getStatusDir() 
         fileutil.CreateDir(statusDir, 'root', 0700)
-        configDir = self.self.getConfigDir()
+        configDir = self.setting.getConfigDir()
         fileutil.CreateDir(configDir, 'root', 0700)
 
     def enable(self):
         man = self.loadManifest()
         self.updateHandlerEnvironment()
+        self.updateSetting()
         self.launchCommand(man.getEnableCommand())
+        fileutil.SetFileContents(self.setting.getHandlerStateFile(),
+                                 "Enabled")
 
     def disable(self):
         man = self.loadManifest()
         self.updateHandlerEnvironment()
+        self.updateSetting()
         self.launchCommand(man.getDisableCommand())
+        fileutil.SetFileContents(self.setting.getHandlerStateFile(),
+                                 "Disabled")
 
     def install(self):
         man = self.loadManifest()
         self.updateHandlerEnvironment()
+        self.updateSetting()
         self.launchCommand(man.getInstallCommand())
+        fileutil.SetFileContents(self.setting.getHandlerStateFile(),
+                                 "Installed")
 
     def uninstall(self):
         self.loadManifest()
         self.updateHandlerEnvironment()
+        self.updateSetting()
         self.launchCommand(man.getUninstallCommand())
+        fileutil.SetFileContents(self.setting.getHandlerStateFile(),
+                                 "Uninstalled")
 
     def update(self):
         self.loadManifest()
         self.updateHandlerEnvironment()
+        self.updateSetting()
         self.launchCommand(man.getUpdateCommand())
+        fileutil.SetFileContents(self.setting.getHandlerStateFile(),
+                                 "Installed")
 
     def launchCommand(self, cmd):
-        baseDir = self.getBaseDir() 
+        baseDir = self.setting.getBaseDir() 
         cmd = os.path.join(baseDir, cmd)
         cmd = "{0} {1}".format(cmd, baseDir)
+        fileutil.RChangeMod(baseDir, 0700)
         try:
             devnull = open(os.devnull, 'w')
             child = subprocess.Popen(cmd, shell=True, cwd=baseDir, stdout=devnull)
@@ -180,21 +217,28 @@ class ExtensionInstance(object):
                 time.sleep(5)
                 retry -= 1
             if retry == 0:
-                self.logger.Error("Process exceeded timeout of {0} seconds" 
+                self.logger.error("Process exceeded timeout of {0} seconds" 
                                   "Terminating process", timeout)
                 os.kill(child.pid, 9)
             ret = child.wait()
-            if code == None or code != 0:
-                self.logger.Error("Process {0} returned non-zero exit code"
-                                  " ({1})", cmd, code)
+            if ret == None or ret != 0:
+                self.logger.error("Process {0} returned non-zero exit code"
+                                  " ({1})", cmd, ret)
         except Exception, e:
-            self.logger.Error('Exception launching {0}, {1}', cmd, e)
+            self.logger.error('Exception launching {0}, {1}', cmd, e)
+            raise e
     
     def loadManifest(self):
         manFile = self.setting.getManifestFile()
-        return json.loads(fileutil.GetFileContents(manFile))
+        data = json.loads(fileutil.GetFileContents(manFile))
+        if data is not None and len(data) > 0:
+            return HandlerManifest(data[0])
+        raise Exception('Failed to load manifest file.')
+
+    def updateSetting(self):
+        fileutil.SetFileContents(self.setting.getSettingsFile(),
+                                 json.dumps(self.setting.getSettings()))
 
-    
     def updateHandlerEnvironment(self):
         env = [{
             "name" : self.setting.getName(),
@@ -246,27 +290,27 @@ class HandlerManifest(object):
         return self.data["version"]
 
     def getInstallCommand(self):
-        return self.data["installCommand"]
+        return self.data['handlerManifest']["installCommand"]
 
     def getUninstallCommand(self):
-        return self.data["uninstallCommand"]
+        return self.data['handlerManifest']["uninstallCommand"]
 
     def getUpdateCommand(self):
-        return self.data["updateCommand"]
+        return self.data['handlerManifest']["updateCommand"]
 
     def getEnableCommand(self):
-        return self.data["enableCommand"]
+        return self.data['handlerManifest']["enableCommand"]
 
     def getDisableCommand(self):
-        return self.data["disableCommand"]
+        return self.data['handlerManifest']["disableCommand"]
 
     def getRebootAfterInstall(self):
-        return self.data["rebootAfterInstall"].lower() == "true"
+        return self.data['handlerManifest']["rebootAfterInstall"].lower() == "true"
 
     def getReportHeartbeat(self):
-        return self.data["reportHeartbeat"].lower() == "true"
+        return self.data['handlerManifest']["reportHeartbeat"].lower() == "true"
 
     def getUpdateWithInstall(self):
         if "updateMode" in self.data:
-            return self.data["updateMode"].lower() == "updatewithinstall"
+            return self.data['handlerManifest']["updateMode"].lower() == "updatewithinstall"
         return False

walinuxagent/protocol/common.py

@@ -17,6 +17,7 @@
 # Requires Python 2.4+ and Openssl 1.0+
 #
 import os
+import copy
 import xml.dom.minidom
 import walinuxagent.logger as logger
 from walinuxagent.utils.osutil import CurrOS
@@ -105,20 +106,25 @@ class ExtensionInfo():
         return self.data["properties"]["state"]
 
     def getSeqNo(self):
-        settings = self.data["properties"]["runtimeSettings"]["handlerSettings"]
-        return settings["sequenceNumber"]
+        settings = self.data["properties"]["runtimeSettings"][0]
+        return settings["handlerSettings"]["sequenceNumber"]
+
+    def getSettings(self):
+        return {
+            'runtimeSettings' : self.data["properties"]['runtimeSettings']
+        }
 
     def getPublicSettings(self):
-        settings = self.data["properties"]["runtimeSettings"]["handlerSettings"]
-        return settings["publicSettings"]
+        settings = self.data["properties"]["runtimeSettings"][0]
+        return settings["handlerSettings"]["publicSettings"]
 
     def getProtectedSettings(self):
-        settings = self.data["properties"]["runtimeSettings"]["handlerSettings"]
-        return settings["privateSettings"]
+        settings = self.data["properties"]["runtimeSettings"][0]
+        return settings["handlerSettings"]["privateSettings"]
 
     def getCertificateThumbprint(self):
-        settings = self.data["properties"]["runtimeSettings"]["handlerSettings"]
-        return settings["certificateThumbprint"]
+        settings = self.data["properties"]["runtimeSettings"][0]
+        return settings["handlerSettings"]["certificateThumbprint"]
 
     def getTargetVersion(self, currVersion):
         if self.getUpgradePolicy().lower() != 'auto':
@@ -131,8 +137,11 @@ class ExtensionInfo():
 
         versionUris = self.getVersionUris()
         if major is not None:
-            versionUris = filter(lambda x : x["version"].startswith(major + "."))
-        versionUris.sort(lambda x, y : cmp(x["version"], y["version"]))
+            versionUris = filter(lambda x : x["version"].startswith(major + "."), 
+                                 versionUris)
+        versionUris = sorted(versionUris, 
+                             key=lambda x: x["version"], 
+                             reverse=True)
         if len(versionUris) > 0:
             return versionUris[0]
         else:
@@ -142,8 +151,8 @@ class ExtensionInfo():
         versionUris = self.getVersionUris()
         version = self.getVersion()
         for versionUri in versionUris:
-            if versionUri.version == version:
-                return versionUri.uris
+            if versionUri['version']== version:
+                return versionUri['uris']
         return None
 
     def copy(self, version):

walinuxagent/protocol/detection.py

@@ -17,6 +17,7 @@
 # Requires Python 2.4+ and Openssl 1.0+
 #
 import os
+import traceback
 import walinuxagent.logger as logger
 from walinuxagent.utils.osutil import CurrOS, CurrOSInfo
 import walinuxagent.utils.fileutil as fileutil
@@ -44,11 +45,14 @@ def DetectAvailableProtocols(protocols=__Protocols):
         try:
             detected = protocol.Detect()
             fileutil.SetFileContents(protocolFilePath, '')
-            logger.Info("Detect protocol:{0}", protocol.__name__)
+            logger.Info("Detect protocol: {0}", protocol.__name__)
             availableProtocols.append(detected)
             break
         except Exception, e:
-            logger.Warn("Probe {0} failed:{1}", protocol.__name__, e)
+            logger.Warn("Probe {0} failed: {1} {2}", 
+                        protocol.__name__, 
+                        e,
+                        traceback.format_exc())
             if os.path.isfile(protocolFilePath):
                 os.remove(protocolFilePath)
     return availableProtocols

walinuxagent/protocol/v1.py

@@ -20,6 +20,7 @@ import os
 import json
 import re
 import time
+import traceback
 import xml.etree.ElementTree as ET
 import walinuxagent.logger as logger
 import walinuxagent.utils.restutil as restutil
@@ -85,9 +86,8 @@ class ProtocolV1(Protocol):
         self.certificates = None
         self.extensions = None
  
-    @logger.LogError("check protocol version")
     def checkProtocolVersion(self):
-        versionInfoXml = restutil.HttpGet(VersionInfoUri.format(self.endpoint))
+        versionInfoXml = restutil.HttpGet(VersionInfoUri.format(self.endpoint)).read()
         self.versionInfo = VersionInfo(versionInfoXml)
         fileutil.SetFileContents(VersionInfoFile, versionInfoXml)
 
@@ -104,13 +104,35 @@ class ProtocolV1(Protocol):
             logger.Warn("Agent supported wire protocol version: {0} was not "
                         "advised by Fabric.", ProtocolVersion)
             raise Exception("Wire protocol version not supported")
-    
+
+    def getHeader(self):
+        return {
+            "x-ms-agent-name":"WALinuxAgent",
+            "x-ms-version":ProtocolVersion
+        }
+
+    def getHeaderWithContentTypeXml(self):
+        return {
+            "x-ms-agent-name":"WALinuxAgent",
+            "x-ms-version":ProtocolVersion,
+            "Content-Type":"text/xml;charset=utf-8"
+        }
+
+    def getHearderWithCert(self):
+        cert = ""
+        for line in fileutil.GetFileContents(TransportCertFile).split('\n'):
+            if "CERTIFICATE" not in line:
+                cert += line.rstrip()
+        return {
+            "x-ms-agent-name":"WALinuxAgent",
+            "x-ms-version":ProtocolVersion,
+            "x-ms-cipher-name": "DES_EDE3_CBC",
+            "x-ms-guest-agent-public-x509-cert":cert
+        }
+
     def updateGoalState(self):
         goalStateXml = restutil.HttpGet(GoalStateUri.format(self.endpoint),
-                                        headers={
-                                            "x-ms-agent-name":"WALinuxAgent",
-                                            "x-ms-version":ProtocolVersion
-                                        })
+                                        headers=self.getHeader()).read()
         if goalStateXml is None:
             raise Exception("Failed update goalstate")
         self.goalState = GoalState(goalStateXml)
@@ -122,10 +144,7 @@ class ProtocolV1(Protocol):
 
     def updateHostingEnv(self):
         hostingEnvXml = restutil.HttpGet(self.goalState.getHostingEnvUri(),
-                                         headers={
-                                            "x-ms-agent-name":"WALinuxAgent",
-                                            "x-ms-version":ProtocolVersion
-                                         })
+                                         headers=self.getHeader()).read()
         if hostingEnvXml is None:
             raise Exception("Failed to update hosting environment config")
         self.hostingEnv = HostingEnv(hostingEnvXml)
@@ -133,19 +152,16 @@ class ProtocolV1(Protocol):
 
     def updateSharedConfig(self):
         sharedConfigXml = restutil.HttpGet(self.goalState.getSharedConfigUri(),
-                                           headers={
-                                                "x-ms-agent-name":"WALinuxAgent",
-                                                "x-ms-version":ProtocolVersion
-                                           })
+                                           headers=self.getHeader()).read()
         self.sharedConfig = SharedConfig(sharedConfigXml)
         fileutil.SetFileContents(SharedConfigFile, sharedConfigXml)
 
     def updateCertificates(self):
+        certificatesUri = self.goalState.getCertificatesUri()
+        if certificatesUri is None:
+            return
         certificatesXml = restutil.HttpGet(self.goalState.getCertificatesUri(),
-                                           headers={
-                                                "x-ms-agent-name":"WALinuxAgent",
-                                                "x-ms-version":ProtocolVersion
-                                           })
+                                           headers=self.getHearderWithCert()).read()
         if certificatesXml is None:
             raise Exception("Failed to update certificates")
         fileutil.SetFileContents(CertificatesFile, certificatesXml)
@@ -155,12 +171,7 @@ class ProtocolV1(Protocol):
     
     def updateExtensionConfig(self):
         extentionsXml = restutil.HttpGet(self.goalState.getExtensionsUri(),
-                                         headers={
-                                            "x-ms-agent-name":"WALinuxAgent",
-                                            "x-ms-version":ProtocolVersion,
-                                            "x-ms-cipher-name": "DES_EDE3_CBC",
-                                            "x-ms-guest-agent-public-x509-cert":self.getTransportCert()
-                                         })
+                                         headers=self.getHeader()).read()
         if extentionsXml is None:
             raise Exception("Failed to update extensions config")
         self.extensions = ExtensionsConfig(extentionsXml)
@@ -171,7 +182,7 @@ class ProtocolV1(Protocol):
             manifestUri = self.extensions.getManifestUri(ext.getName())
             for uri in manifestUri:
                 try:
-                    manifestXml = restutil.HttpGet(uri)
+                    manifestXml = restutil.HttpGet(uri).read()
                     manifestXml = RemoveBom(manifestXml)
                     manifestFile = ManifestFile.format(ext.getName(), 
                                                        self.incarnation)
@@ -180,16 +191,11 @@ class ProtocolV1(Protocol):
                     break
                 except Exception, e:
                     #Download manifest failed, will retry with failover location
-                    logger.Warn("Download manifest for {0} failed: uri={1}",
+                    logger.Warn("Download manifest for {0} failed: {1} {2} {3}",
                                 ext.getName(),
-                                uri)
-
-    def getTransportCert(self):
-        cert = ""
-        for line in fileutil.GetFileContents(TransportCertFile).split('\n'):
-            if "CERTIFICATE" not in line:
-                cert += line.rstrip()
-        return cert
+                                uri,
+                                e,
+                                traceback.format_exc())
 
     def refreshCache(self):
         """
@@ -247,19 +253,28 @@ class ProtocolV1(Protocol):
                 ExtensionManifest(manifestXml).update(ext)
         return self.extensions
 
-    def reportProvisionStatus(self, status=None, subStatus="", 
-                              description="", thumbprint=None):
+    def reportProvisionStatus(self, status=None, subStatus=None, 
+                              description='', thumbprint=None):
         if status is not None:
             healthReport = self._buildHealthReport(status, 
                                                    subStatus, 
                                                    description)
             healthReportUri = HealthReportUri.format(self.endpoint)
-            ret = restutil.HttpPost(healthReportUri, healthReport)
+            headers=self.getHeaderWithContentTypeXml()
+            resp = restutil.HttpPost(healthReportUri, 
+                                     healthReport,
+                                     headers=headers)
+            if resp is not None:
+                latest = resp.getheader("x-ms-latest-goal-state-incarnation-number")
+                if latest is not None:
+                    self.incarnation = latest
 
         if thumbprint is not None:
             roleProp = self._buildRoleProperties(thumbprint)
             rolePropUri = RolePropUri.format(self.endpoint)
-            ret = restutil.HttpPost(rolePropUri, roleProp)
+            ret = restutil.HttpPost(rolePropUri, 
+                                    roleProp,
+                                    headers=self.getHeaderWithContentTypeXml())
 
     def _buildRoleProperties(self, thumbprint):
         return (u"<?xml version=\"1.0\" encoding=\"utf-8\"?>"
@@ -281,8 +296,14 @@ class ProtocolV1(Protocol):
                            thumbprint)
 
     def _buildHealthReport(self, status, subStatus, description):
+        detail = None
+        if subStatus is not None:
+            detail = ("<Details>"
+                      "<SubStatus>{0}</SubStatus>"
+                      "<Description>{1}</Description>"
+                      "</Details>").format(subStatus, description)
         return (u"<?xml version=\"1.0\" encoding=\"utf-8\"?>"
-                "<Health"
+                "<Health "
                     "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\""
                 " xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\">"
                 "<GoalStateIncarnation>{0}</GoalStateIncarnation>"
@@ -293,10 +314,7 @@ class ProtocolV1(Protocol):
                 "<InstanceId>{2}</InstanceId>"
                 "<Health>"
                 "<State>{3}</State>"
-                "<Details>"
-                "<SubStatus>{4}</SubStatus>"
-                "<Description>{5}</Description>"
-                "</Details>"
+                "{4}"
                 "</Health>"
                 "</Role>"
                 "</RoleInstanceList>"
@@ -306,8 +324,7 @@ class ProtocolV1(Protocol):
                            self.goalState.getContainerId(),
                            self.goalState.getRoleInstanceId(),
                            status, 
-                           subStatus, 
-                           description)
+                           detail if detail is not None else '')
 
     def reportAgentStatus(self, version, status, message):
         tstamp=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
@@ -490,7 +507,8 @@ class GoalState():
         self.hostingEnvUri = (FindFirstNode(xmlDoc, 
                                             ".//HostingEnvironmentConfig")).text
         self.sharedConfigUri = (FindFirstNode(xmlDoc, ".//SharedConfig")).text
-        self.certificatesUri = (FindFirstNode(xmlDoc, ".//Certificates")).text
+        node = (FindFirstNode(xmlDoc, ".//Certificates"))
+        self.certificatesUri = node.text if node is not None else None
         self.extensionsUri = (FindFirstNode(xmlDoc, ".//ExtensionsConfig")).text
         self.roleInstanceId = (FindFirstNode(xmlDoc, 
                                              ".//RoleInstance/InstanceId")).text
@@ -710,7 +728,7 @@ class ExtensionsConfig(object):
             location = extension.attrib["location"]
             failoverLocation = extension.attrib["failoverlocation"]
             autoUpgrade = extension.attrib["autoUpgrade"]
-            upgradePolicy = "auto" if autoUpgrade == "true" else None
+            upgradePolicy = "auto" if autoUpgrade == "true" else "manual"
             state = extension.attrib["state"]
             setting = filter(lambda x: x.attrib["name"] == name 
                              and x.attrib["version"] == version,
@@ -736,7 +754,7 @@ class ExtensionsConfig(object):
             handlerSettings["certificateThumbprint"] = thumbprint
 
             runtimeSettings["handlerSettings"] = handlerSettings
-            properties["runtimeSettings"] = runtimeSettings
+            properties["runtimeSettings"] = [runtimeSettings]
             ext["properties"] = properties
             self.extensions.append(ExtensionInfo(ext))
             self.manifestUris[name] = (location, failoverLocation)
@@ -755,7 +773,8 @@ class ExtensionManifest(object):
         packages = FindAllNodes(xmlDoc, ".//Plugins/Plugin")
         for package in packages:
             version = FindFirstNode(package, "Version").text
-            uris = filter(lambda x : x.text, FindAllNodes(package, "Uri"))
+            uris = FindAllNodes(package, "Uris/Uri")
+            uris = map(lambda x : x.text, uris)
             self.versionUris.append({
                 "version":version,
                 "uris":uris

walinuxagent/provision.py

@@ -18,6 +18,7 @@
 #
 
 import os
+import traceback
 import walinuxagent.logger as logger
 from walinuxagent.utils.osutil import CurrOS, CurrOSInfo
 import walinuxagent.utils.shellutil as shellutil
@@ -38,14 +39,15 @@ class ProvisionHandler(object):
                 #In some distro like Ubuntu, cloud init does the provision
                 #In this case, we need to wait the cloud init to complete 
                 #provision work and generate ssh host key
-                CurrOS.WaitForSshHostKey()
+                keyPairType = self.config.get("Provisioning.SshHostKeyPairType", "rsa")
+                CurrOS.WaitForSshHostKey(keyPairType)
 
-            keyPairType = config.get("Provisioning.SshHostKeyPairType", "rsa")
+            keyPairType = self.config.get("Provisioning.SshHostKeyPairType", "rsa")
             thumbprint = CurrOS.GetSshHostKeyThumbprint(keyPairType)
             self.protocol.reportProvisionStatus(status="Ready",
                                                 thumbprint = thumbprint)
         except Exception, e:
-            logger.Error("Provision failed: {0}", e)
+            logger.Error("Provision failed: {0} {1}", e, traceback.format_exc())
             self.protocol.reportProvisionStatus(status="NotReady",
                                                 subStatus="Provisioning Failed")
             raise e

walinuxagent/utils/fileutil.py

@@ -37,6 +37,8 @@ def GetFileContents(filepath, asbin=False, removeBom=False):
     if asbin:
         mode+='b'
     try:
+        if not os.path.isfile(filepath):
+            return None
         with open(filepath, mode) as F :
             c=F.read()
         if (not asbin) and removeBom:
@@ -44,7 +46,7 @@ def GetFileContents(filepath, asbin=False, removeBom=False):
         return c
     except IOError, e:
         logger.Error('Reading from file {0} Exception is {1}', filepath, e)
-        return None        
+        raise e
 
 def SetFileContents(filepath, contents, append=False):
     """
@@ -161,3 +163,14 @@ def UpdateConfigFile(path, lineStart, val, chk_err=False):
     config.append(val)
     fileutil.ReplaceFileContentsAtomic(path, '\n'.join(config))
 
+def SearchForFile(dirName, fileName):
+    for root, dirs, files in os.walk(dirName):
+        for f in files:
+            if f == 'HandlerManifest.json':
+                return os.path.join(root, f)
+    return None
+
+def RChangeMod(path, mode):
+    for root, dirs, files in os.walk(path):
+        for f in files:
+            os.chmod(os.path.join(root, f), mode)

walinuxagent/utils/osutil.py

@@ -43,7 +43,7 @@ Define distro specific behavior. DefaultDistro class defines default behavior
 for all distros. Each concrete distro classes could overwrite default behavior
 if needed.
 """
-class DefaultDistro():
+class DefaultDistro(object):
     def __init__(self):
         self.libDir = "/var/lib/waagent"
         self.dvdMountPoint = "/mnt/cdrom/secure"
@@ -160,7 +160,10 @@ class DefaultDistro():
         fileutil.ChangeMod('/etc/sudoers.d/waagent', 0440)
 
     def DeleteRootPassword(self):
-        passwd = fileutil.GetFileContents(self.passwdPath).split("\n")
+        passwdContent = fileutil.GetFileContents(self.passwdPath)
+        if passwdContent is None:
+            raise Exception("Failed to delete root password.")
+        passwd = passwdContent.split('\n')
         newPasswd = filter(lambda x : not x.startswith("root:"), passwd)
         newPasswd.insert(0, "root:*LOCK*:14600::::::")
         fileutil.ReplaceFileContentsAtomic(self.passwdPath, "\n".join(newPasswd))
@@ -310,12 +313,12 @@ class DefaultDistro():
         else:
             return None
 
-    def WaitForSshHostKey(keyPairType, maxRetry=6):
+    def WaitForSshHostKey(self, keyPairType, maxRetry=6):
         path = '/etc/ssh/ssh_host_{0}_key'.format(keyPairType)
         for retry in range(0, maxRetry):
             if os.path.isfile(path):
                 return
-            logger.Info("Wait for ssh host key be generated.")
+            logger.Info("Wait for ssh host key be generated: {0}", path)
             time.sleep(1)
         raise Exception("Can't find ssh host key.")
 
@@ -689,6 +692,7 @@ class DefaultDistro():
 
 class DebianDistro(DefaultDistro):
     def __init__(self):
+        super(DebianDistro, self).__init__()
         self.dhcpClientConfigFile = '/etc/dhcp/dhclient.conf'
 
     def RestartSshService(self):

walinuxagent/utils/restutil.py

@@ -68,23 +68,31 @@ def HttpRequest(method, url, data, headers=None, maxRetry=1):
     On error, sleep 10 and maxRetry times.
     Return the output buffer or None.
     """
+    def isValidResponse(resp):
+        if resp is None:
+            return False
+        if resp.status in {httplib.OK, httplib.CREATED, httplib.ACCEPTED}:
+            return True
+        return False
+
     logger.Verbose("{0} {1}", method, url)
+    logger.Verbose("Data={0}", data)
+    logger.Verbose("Header={0}", headers)
     host, action, secure = _ParseUrl(url)
     resp = _HttpRequest(method, host, action, data, secure, headers)
     for retry in range(0, maxRetry):
-        if resp is not None and resp.status == httplib.OK:
+        if isValidResponse(resp):
             break;
         elif resp is None:
             logger.Error("Retry={0}, response is empty.", retry)
         else:
-            logger.Error("Retry={0}, Status={1}, {2} {3}", retry, 
-                         resp.status, method, url)
+            logger.Error("Retry={0}, Status={1}, Message={2}, {3}, {4}", retry, 
+                         resp.status, resp.reason, method, url)
         time.sleep(__RetryWaitingInterval)
         resp = _HttpRequest(method, host, action, data, secure, headers)
 
-    if (resp is not None 
-            and (resp.status == httplib.OK or resp.status == httplib.ACCEPTED)):
-        return resp.read()
+    if isValidResponse(resp):
+        return resp
     else:
         return None
 

walinuxagent/utils/textutil.py

@@ -177,6 +177,6 @@ def SetSshConfig(config, name, val):
     return config
 
 def RemoveBom(c):
-    if ord(c[0]) > 128 and ord(c[1]) > 128 and ord(c[2] > 128):
+    if ord(c[0]) > 128 and ord(c[1]) > 128 and ord(c[2]) > 128:
         c = c[3:]
     return c