(VOL-4953)enabling retry codes in open-olt-adaptor

Enabling retry codes in open-olt-adaptor grpc-client so that it retries the grpc requests to the voltha-core
in case the core is unavailable or the request times out
(retry for pipeline)
Change-Id: Iae93e445ed6363b5fd654bdc1cbe3e53778b2a67
diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go
index 9993adb..002c459 100644
--- a/internal/pkg/config/config.go
+++ b/internal/pkg/config/config.go
@@ -31,6 +31,7 @@
 	defaultKvstoretype          = EtcdStoreName
 	defaultKvstoretimeout       = 5 * time.Second
 	defaultRPCTimeout           = 10 * time.Second
+	defaultPerRPCRetryTimeout   = 2 * time.Second
 	defaultKvstoreaddress       = "127.0.0.1:2379" // Port: Consul = 8500; Etcd = 2379
 	defaultLoglevel             = "WARN"
 	defaultBanner               = false
@@ -60,6 +61,7 @@
 	defaultMaxBackoffRetryDelay               = 10 * time.Second
 	defaultAdapterEndpoint                    = "adapter-open-olt"
 	defaultCheckOnuDevExistenceAtOnuDiscovery = false
+	defaultMaxRetries                         = 10
 )
 
 // AdapterFlags represents the set of configurations used by the read-write adaptercore service
@@ -72,6 +74,7 @@
 	KVStoreTimeout                     time.Duration
 	KVStoreAddress                     string
 	RPCTimeout                         time.Duration
+	PerRPCRetryTimeout                 time.Duration
 	EventTopic                         string
 	LogLevel                           string
 	OnuNumber                          int
@@ -97,6 +100,7 @@
 	MaxBackoffRetryDelay               time.Duration
 	AdapterEndpoint                    string
 	CheckOnuDevExistenceAtOnuDiscovery bool
+	MaxRetries                         uint
 }
 
 // NewAdapterFlags returns a new RWCore config
@@ -127,9 +131,11 @@
 		EnableONUStats:                     defaultEnableONUStats,
 		EnableGEMStats:                     defaultEnableGEMStats,
 		RPCTimeout:                         defaultRPCTimeout,
+		PerRPCRetryTimeout:                 defaultPerRPCRetryTimeout,
 		MinBackoffRetryDelay:               defaultMinBackoffRetryDelay,
 		MaxBackoffRetryDelay:               defaultMaxBackoffRetryDelay,
 		CheckOnuDevExistenceAtOnuDiscovery: defaultCheckOnuDevExistenceAtOnuDiscovery,
+		MaxRetries:                         defaultMaxRetries,
 	}
 	return &adapterFlags
 }
@@ -277,11 +283,21 @@
 		defaultMinBackoffRetryDelay,
 		"The minimum number of milliseconds to delay before a connection retry attempt")
 
+	flag.DurationVar(&(so.PerRPCRetryTimeout),
+		"per_rpc_retry_timeout",
+		defaultPerRPCRetryTimeout,
+		"The default timeout per RPC retry")
+
 	flag.DurationVar(&(so.MaxBackoffRetryDelay),
 		"max_retry_delay",
 		defaultMaxBackoffRetryDelay,
 		"The maximum number of milliseconds to delay before a connection retry attempt")
 
+	flag.UintVar(&(so.MaxRetries),
+		"max_grpc_client_retry",
+		defaultMaxRetries,
+		"The maximum number of times olt adaptor will retry in case grpc request timeouts")
+
 	flag.BoolVar(&(so.CheckOnuDevExistenceAtOnuDiscovery),
 		"check_onu_exist_on_discovery",
 		defaultCheckOnuDevExistenceAtOnuDiscovery,