Merge pull request #451 from hglkrijger/proxy

fix proxy port type

azurelinuxagent/common/conf.py

@@ -24,10 +24,12 @@ import os
 import azurelinuxagent.common.utils.fileutil as fileutil
 from azurelinuxagent.common.exception import AgentConfigError
 
+
 class ConfigurationProvider(object):
     """
     Parse amd store key:values in /etc/waagent.conf.
     """
+
     def __init__(self):
         self.values = dict()
 
@@ -66,6 +68,7 @@ class ConfigurationProvider(object):
 
 __conf__ = ConfigurationProvider()
 
+
 def load_conf_from_file(conf_file_path, conf=__conf__):
     """
     Load conf file from: conf_file_path
@@ -80,105 +83,138 @@ def load_conf_from_file(conf_file_path, conf=__conf__):
         raise AgentConfigError(("Failed to load conf file:{0}, {1}"
                                 "").format(conf_file_path, err))
 
+
 def enable_rdma(conf=__conf__):
     return conf.get_switch("OS.EnableRDMA", False)
 
+
 def get_logs_verbose(conf=__conf__):
     return conf.get_switch("Logs.Verbose", False)
 
+
 def get_lib_dir(conf=__conf__):
     return conf.get("Lib.Dir", "/var/lib/waagent")
 
+
 def get_dvd_mount_point(conf=__conf__):
     return conf.get("DVD.MountPoint", "/mnt/cdrom/secure")
 
+
 def get_agent_pid_file_path(conf=__conf__):
     return conf.get("Pid.File", "/var/run/waagent.pid")
 
+
 def get_ext_log_dir(conf=__conf__):
     return conf.get("Extension.LogDir", "/var/log/azure")
 
+
 def get_openssl_cmd(conf=__conf__):
     return conf.get("OS.OpensslPath", "/usr/bin/openssl")
 
+
 def get_home_dir(conf=__conf__):
     return conf.get("OS.HomeDir", "/home")
 
+
 def get_passwd_file_path(conf=__conf__):
     return conf.get("OS.PasswordPath", "/etc/shadow")
 
+
 def get_sudoers_dir(conf=__conf__):
     return conf.get("OS.SudoersDir", "/etc/sudoers.d")
 
+
 def get_sshd_conf_file_path(conf=__conf__):
     return conf.get("OS.SshdConfigPath", "/etc/ssh/sshd_config")
 
+
 def get_root_device_scsi_timeout(conf=__conf__):
     return conf.get("OS.RootDeviceScsiTimeout", None)
 
+
 def get_ssh_host_keypair_type(conf=__conf__):
     return conf.get("Provisioning.SshHostKeyPairType", "rsa")
 
+
 def get_provision_enabled(conf=__conf__):
     return conf.get_switch("Provisioning.Enabled", True)
 
+
 def get_allow_reset_sys_user(conf=__conf__):
     return conf.get_switch("Provisioning.AllowResetSysUser", False)
 
+
 def get_regenerate_ssh_host_key(conf=__conf__):
     return conf.get_switch("Provisioning.RegenerateSshHostKeyPair", False)
 
+
 def get_delete_root_password(conf=__conf__):
     return conf.get_switch("Provisioning.DeleteRootPassword", False)
 
+
 def get_decode_customdata(conf=__conf__):
     return conf.get_switch("Provisioning.DecodeCustomData", False)
 
+
 def get_execute_customdata(conf=__conf__):
     return conf.get_switch("Provisioning.ExecuteCustomData", False)
 
+
 def get_password_cryptid(conf=__conf__):
     return conf.get("Provisioning.PasswordCryptId", "6")
 
+
 def get_password_crypt_salt_len(conf=__conf__):
     return conf.get_int("Provisioning.PasswordCryptSaltLength", 10)
 
+
 def get_monitor_hostname(conf=__conf__):
     return conf.get_switch("Provisioning.MonitorHostName", False)
 
+
 def get_httpproxy_host(conf=__conf__):
     return conf.get("HttpProxy.Host", None)
 
+
 def get_httpproxy_port(conf=__conf__):
-    return conf.get("HttpProxy.Port", None)
+    return conf.get_int("HttpProxy.Port", None)
+
 
 def get_detect_scvmm_env(conf=__conf__):
     return conf.get_switch("DetectScvmmEnv", False)
 
+
 def get_resourcedisk_format(conf=__conf__):
     return conf.get_switch("ResourceDisk.Format", False)
 
+
 def get_resourcedisk_enable_swap(conf=__conf__):
     return conf.get_switch("ResourceDisk.EnableSwap", False)
 
+
 def get_resourcedisk_mountpoint(conf=__conf__):
     return conf.get("ResourceDisk.MountPoint", "/mnt/resource")
 
+
 def get_resourcedisk_mountoptions(conf=__conf__):
     return conf.get("ResourceDisk.MountOptions", None)
 
+
 def get_resourcedisk_filesystem(conf=__conf__):
     return conf.get("ResourceDisk.Filesystem", "ext3")
 
+
 def get_resourcedisk_swap_size_mb(conf=__conf__):
     return conf.get_int("ResourceDisk.SwapSizeMB", 0)
 
+
 def get_autoupdate_gafamily(conf=__conf__):
     return conf.get("AutoUpdate.GAFamily", "Prod")
 
+
 def get_autoupdate_enabled(conf=__conf__):
     return conf.get_switch("AutoUpdate.Enabled", True)
 
+
 def get_autoupdate_frequency(conf=__conf__):
     return conf.get_int("Autoupdate.Frequency", 3600)
-

azurelinuxagent/common/protocol/wire.py

@@ -380,7 +380,7 @@ class StatusBlob(object):
                 url,
                 {
                     "x-ms-date": timestamp,
-                    'x-ms-version': self.__class__.__storage_version__
+                    "x-ms-version": self.__class__.__storage_version__
                 })
         except HttpError as e:
             raise ProtocolError((u"Failed to get status blob type: {0}"

azurelinuxagent/common/utils/restutil.py

@@ -60,28 +60,39 @@ def _http_request(method, host, rel_uri, port=None, data=None, secure=False,
     if secure:
         port = 443 if port is None else port
         if proxy_host is not None and proxy_port is not None:
-            conn = httpclient.HTTPSConnection(proxy_host, proxy_port,
+            conn = httpclient.HTTPSConnection(proxy_host,
+                                              proxy_port,
                                               timeout=10)
             conn.set_tunnel(host, port)
             # If proxy is used, full url is needed.
             url = "https://{0}:{1}{2}".format(host, port, rel_uri)
         else:
-            conn = httpclient.HTTPSConnection(host, port, timeout=10)
+            conn = httpclient.HTTPSConnection(host,
+                                              port,
+                                              timeout=10)
             url = rel_uri
     else:
         port = 80 if port is None else port
         if proxy_host is not None and proxy_port is not None:
-            conn = httpclient.HTTPConnection(proxy_host, proxy_port,
+            conn = httpclient.HTTPConnection(proxy_host,
+                                             proxy_port,
                                              timeout=10)
             # If proxy is used, full url is needed.
             url = "http://{0}:{1}{2}".format(host, port, rel_uri)
         else:
-            conn = httpclient.HTTPConnection(host, port, timeout=10)
+            conn = httpclient.HTTPConnection(host,
+                                             port,
+                                             timeout=10)
             url = rel_uri
-    if headers is None:
-        conn.request(method, url, data)
-    else:
-        conn.request(method, url, data, headers)
+
+    logger.verbose("HTTPConnection [{0}] [{1}] [{2}] [{3}]",
+                   method,
+                   url,
+                   data,
+                   headers)
+
+    headers = {} if headers is None else headers
+    conn.request(method=method, url=url, body=data, headers=headers)
     resp = conn.getresponse()
     return resp
 
@@ -92,9 +103,6 @@ def http_request(method, url, data, headers=None, max_retry=3,
     Sending http request to server
     On error, sleep 10 and retry max_retry times.
     """
-    logger.verbose("HTTP Req: {0} {1}", method, url)
-    logger.verbose("    Data={0}", data)
-    logger.verbose("    Header={0}", headers)
     host, port, secure, rel_uri = _parse_url(url)
 
     # Check proxy
@@ -114,28 +122,44 @@ def http_request(method, url, data, headers=None, max_retry=3,
                     "(new in python 2.7)")
         secure = False
 
+    logger.verbose("HTTP method: [{0}]", method)
+    logger.verbose("HTTP host: [{0}]", host)
+    logger.verbose("HTTP uri: [{0}]", rel_uri)
+    logger.verbose("HTTP port: [{0}]", port)
+    logger.verbose("HTTP data: [{0}]", data)
+    logger.verbose("HTTP secure: [{0}]", secure)
+    logger.verbose("HTTP headers: [{0}]", headers)
+    logger.verbose("HTTP proxy: [{0}:{1}]", proxy_host, proxy_port)
+
     for retry in range(0, max_retry):
         try:
-            resp = _http_request(method, host, rel_uri, port=port, data=data,
-                                 secure=secure, headers=headers,
-                                 proxy_host=proxy_host, proxy_port=proxy_port)
-            logger.verbose("HTTP Resp: Status={0}", resp.status)
-            logger.verbose("    Header={0}", resp.getheaders())
+            resp = _http_request(method,
+                                 host,
+                                 rel_uri,
+                                 port=port,
+                                 data=data,
+                                 secure=secure,
+                                 headers=headers,
+                                 proxy_host=proxy_host,
+                                 proxy_port=proxy_port)
+            logger.verbose("HTTP response status: [{0}]", resp.status)
             return resp
         except httpclient.HTTPException as e:
-            logger.warn('HTTPException {0}, args:{1}', e, repr(e.args))
+            logger.warn('HTTPException: [{0}]', e)
         except IOError as e:
-            logger.warn('Socket IOError {0}, args:{1}', e, repr(e.args))
+            logger.warn('IOError: [{0}]', e)
 
         if retry < max_retry - 1:
-            logger.info("Retry={0}, {1} {2}", retry, method, url)
+            logger.info("Retry {0}", retry)
             time.sleep(RETRY_WAITING_INTERVAL)
+        else:
+            logger.error("All retries failed")
 
     if url is not None and len(url) > 100:
         url_log = url[0: 100]  # In case the url is too long
     else:
         url_log = url
-    raise HttpError("HTTP Err: {0} {1}".format(method, url_log))
+    raise HttpError("HTTPError: {0} {1}".format(method, url_log))
 
 
 def http_get(url, headers=None, max_retry=3, chk_proxy=False):

azurelinuxagent/ga/exthandlers.py

@@ -293,17 +293,14 @@ class ExtHandlersHandler(object):
                         message=ustr(e))
         
         logger.verbose("Report vm agent status")
-        
         try:
             self.protocol.report_vm_status(vm_status)
+            if self.log_report:
+                logger.verbose("Successfully reported vm agent status")
         except ProtocolError as e:
             message = "Failed to report vm agent status: {0}".format(e)
             add_event(AGENT_NAME, version=CURRENT_VERSION, is_success=False, message=message)
 
-        if self.log_report:
-            logger.verbose("Successfully reported vm agent status")
-
-
     def report_ext_handler_status(self, vm_status, ext_handler):
         ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
         

tests/utils/test_rest_util.py

@@ -15,16 +15,13 @@
 # Requires Python 2.4+ and Openssl 1.0+
 #
 
-from tests.tools import AgentTestCase, patch, Mock, MagicMock
-import uuid
 import unittest
-import os
 import azurelinuxagent.common.utils.restutil as restutil
-from azurelinuxagent.common.future import ustr, httpclient
-import azurelinuxagent.common.logger as logger
+from azurelinuxagent.common.future import httpclient
+from tests.tools import AgentTestCase, patch, Mock, MagicMock
+
 
 class TestHttpOperations(AgentTestCase):
-
     def test_parse_url(self):
         test_uri = "http://abc.def/ghi#hash?jkl=mn"
         host, port, secure, rel_uri = restutil._parse_url(test_uri)
@@ -36,11 +33,11 @@ class TestHttpOperations(AgentTestCase):
         self.assertEquals("abc.def", host)
         self.assertEquals("/", rel_uri)
         self.assertEquals(False, secure)
-        
+
         test_uri = "https://abc.def/ghi?jkl=mn"
         host, port, secure, rel_uri = restutil._parse_url(test_uri)
         self.assertEquals(True, secure)
-    
+
         test_uri = "http://abc.def:80/"
         host, port, secure, rel_uri = restutil._parse_url(test_uri)
         self.assertEquals("abc.def", host)
@@ -53,71 +50,71 @@ class TestHttpOperations(AgentTestCase):
         self.assertEquals(None, host)
         self.assertEquals(rel_uri, "None")
 
-
     @patch("azurelinuxagent.common.future.httpclient.HTTPSConnection")
     @patch("azurelinuxagent.common.future.httpclient.HTTPConnection")
     def test_http_request(self, HTTPConnection, HTTPSConnection):
-        mock_httpconn = MagicMock()
-        mock_httpresp = MagicMock()
-        mock_httpconn.getresponse = Mock(return_value=mock_httpresp)
-        HTTPConnection.return_value = mock_httpconn
-        HTTPSConnection.return_value = mock_httpconn
-       
-        mock_httpresp.read = Mock(return_value="_(:3| <)_")
+        mock_http_conn = MagicMock()
+        mock_http_resp = MagicMock()
+        mock_http_conn.getresponse = Mock(return_value=mock_http_resp)
+        HTTPConnection.return_value = mock_http_conn
+        HTTPSConnection.return_value = mock_http_conn
 
-        #Test http get
+        mock_http_resp.read = Mock(return_value="_(:3| <)_")
+
+        # Test http get
         resp = restutil._http_request("GET", "foo", "bar")
         self.assertNotEquals(None, resp)
         self.assertEquals("_(:3| <)_", resp.read())
-    
-        #Test https get
+
+        # Test https get
         resp = restutil._http_request("GET", "foo", "bar", secure=True)
         self.assertNotEquals(None, resp)
         self.assertEquals("_(:3| <)_", resp.read())
-        
-        #Test http get with proxy
-        mock_httpresp.read = Mock(return_value="_(:3| <)_")
+
+        # Test http get with proxy
+        mock_http_resp.read = Mock(return_value="_(:3| <)_")
         resp = restutil._http_request("GET", "foo", "bar", proxy_host="foo.bar",
                                       proxy_port=23333)
         self.assertNotEquals(None, resp)
         self.assertEquals("_(:3| <)_", resp.read())
-    
-        #Test https get
+
+        # Test https get
         resp = restutil._http_request("GET", "foo", "bar", secure=True)
         self.assertNotEquals(None, resp)
         self.assertEquals("_(:3| <)_", resp.read())
-        
-        #Test https get with proxy
-        mock_httpresp.read = Mock(return_value="_(:3| <)_")
+
+        # Test https get with proxy
+        mock_http_resp.read = Mock(return_value="_(:3| <)_")
         resp = restutil._http_request("GET", "foo", "bar", proxy_host="foo.bar",
                                       proxy_port=23333, secure=True)
         self.assertNotEquals(None, resp)
         self.assertEquals("_(:3| <)_", resp.read())
-    
+
     @patch("time.sleep")
     @patch("azurelinuxagent.common.utils.restutil._http_request")
     def test_http_request_with_retry(self, _http_request, sleep):
-        mock_httpresp = MagicMock()
-        mock_httpresp.read = Mock(return_value="hehe")
-        _http_request.return_value = mock_httpresp
+        mock_http_resp = MagicMock()
+        mock_http_resp.read = Mock(return_value="hehe")
+        _http_request.return_value = mock_http_resp
 
-        #Test http get
-        resp = restutil.http_get("http://foo.bar") 
+        # Test http get
+        resp = restutil.http_get("http://foo.bar")
         self.assertEquals("hehe", resp.read())
 
-        #Test https get
-        resp = restutil.http_get("https://foo.bar") 
+        # Test https get
+        resp = restutil.http_get("https://foo.bar")
         self.assertEquals("hehe", resp.read())
-        
-        #Test http failure
+
+        # Test http failure
         _http_request.side_effect = httpclient.HTTPException("Http failure")
-        self.assertRaises(restutil.HttpError, restutil.http_get, "http://foo.bar")
+        self.assertRaises(restutil.HttpError, restutil.http_get,
+                          "http://foo.bar")
 
-        #Test http failure
+        # Test http failure
         _http_request.side_effect = IOError("IO failure")
-        self.assertRaises(restutil.HttpError, restutil.http_get, "http://foo.bar")
-    
-if __name__ == '__main__':
-    unittest.main()
+        self.assertRaises(restutil.HttpError, restutil.http_get,
+                          "http://foo.bar")
+
+
 if __name__ == '__main__':
     unittest.main()