commit db9a86eeeacac1bce757c81b622f7237187c65f5
author Richard Jankowski <rjankows@ciena.com> Mon Sep 17 13:33:29 2018 -0400
committer khenaidoo Nursimulu <knursimu@ciena.com> Mon Sep 17 19:01:52 2018 +0000
tree e07eb6f6a8097f6191a55e64eab58f7149ab040c
parent 4ca78b58ebeb388c02068806dd074bc8b509c779

VOL-1183: OFagent cannot reach vcore after vcore moved to different node in cluster

Two gRPC calls made by OFagent's ConnectionManager, Subscribe and ListLogicalDevices, actually
run in the Twisted reactor thread. On rare occasions, either call never returns when the vcore
connected to the OFagent dies. Because the blocking gRPC runs in the reactor thread, the entire
application is blocked and cannot detect comms errors with the deceased vcore. The handling  of
these gRPCs has been moved to the OFagent's internal GrpcClient, which wraps all gRPC calls in
the twisted.internet.threads.deferToThread function.

Change-Id: Ifa3479a3cbca7ae8d7f5e2ca0372e22507d5e4b9

tests/utests/ofagent/test_connection_mgr.py

diff --git a/tests/utests/ofagent/test_connection_mgr.py b/tests/utests/ofagent/test_connection_mgr.py
index 4ca022c..fe7bcdf 100644
--- a/tests/utests/ofagent/test_connection_mgr.py
+++ b/tests/utests/ofagent/test_connection_mgr.py
@@ -17,6 +17,9 @@
 
 class TestConection_mgr(TestCase):
 
+    # Set a default for the gRPC timeout (seconds)
+    grpc_timeout = 10
+
     def gen_endpoints(self):
         consul_endpoint = "localhost:8500"
         voltha_endpoint= "localhost:8880"
@@ -45,7 +48,8 @@
     def test_connection_mgr_init(self):
         consul_endpoint,voltha_endpoint,controller_endpoints  = self.gen_endpoints()
         my_name = self.gen_container_name()
-        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, controller_endpoints, my_name)
+        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, self.grpc_timeout,
+                                                 controller_endpoints, my_name)
         self.assertEqual(test_connection_init.consul_endpoint,consul_endpoint)
         self.assertEqual(test_connection_init.vcore_endpoint, voltha_endpoint)
         self.assertEqual(test_connection_init.controller_endpoints, controller_endpoints)
@@ -53,7 +57,8 @@
     def test_resolve_endpoint(self):
         consul_endpoint, voltha_endpoint, controller_endpoints = self.gen_endpoints()
         my_name = self.gen_container_name()
-        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, controller_endpoints, my_name)
+        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, self.grpc_timeout,
+                                                 controller_endpoints, my_name)
         host,port = test_connection_init.resolve_endpoint(endpoint=consul_endpoint)
         assert isinstance(port, int)
         assert isinstance(host, basestring)
@@ -62,21 +67,24 @@
         consul_endpoint, voltha_endpoint, controller_endpoints = self.gen_endpoints()
         my_name = self.gen_container_name()
         devices,device = self.gen_devices()
-        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, controller_endpoints, my_name)
+        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, self.grpc_timeout,
+                                                 controller_endpoints, my_name)
         test_connection_init.refresh_agent_connections(devices)
 
     def test_create_agent(self):
         consul_endpoint, voltha_endpoint, controller_endpoints = self.gen_endpoints()
         my_name = self.gen_container_name()
         devices,device = self.gen_devices()
-        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, controller_endpoints, my_name)
+        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, self.grpc_timeout,
+                                                 controller_endpoints, my_name)
         test_connection_init.create_agent(device)
 
     def test_delete_agent(self):
         consul_endpoint, voltha_endpoint, controller_endpoints = self.gen_endpoints()
         my_name = self.gen_container_name()
         devices,device = self.gen_devices()
-        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, controller_endpoints, my_name)
+        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, self.grpc_timeout,
+                                                 controller_endpoints, my_name)
         test_connection_init.create_agent(device)
         with self.assertRaises(Exception) as context:
             test_connection_init.delete_agent(device.datapath_id)
@@ -88,7 +96,8 @@
         my_name = self.gen_container_name()
         devices,device = self.gen_devices()
         packet_in = self.gen_packet_in()
-        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, controller_endpoints, my_name)
+        test_connection_init = ConnectionManager(consul_endpoint, voltha_endpoint, self.grpc_timeout,
+                                                 controller_endpoints, my_name)
         test_connection_init.create_agent(device)
         test_connection_init.forward_packet_in(device.id, packet_in)