[VOL-5529] Transition to deviceInit when gRPC is disconnected

Change-Id: If77eb62071390790afab644ddf3eea55bc521d5d
Signed-off-by: bseeniva <balaji.seenivasan@radisys.com>
diff --git a/internal/pkg/core/device_handler.go b/internal/pkg/core/device_handler.go
index 7ef9639..ee582a6 100755
--- a/internal/pkg/core/device_handler.go
+++ b/internal/pkg/core/device_handler.go
@@ -1215,7 +1215,7 @@
 		}
 	}
 
-	logger.Debugw(ctx, "Dailing grpc", log.Fields{"device-id": dh.device.Id})
+	logger.Debugw(ctx, "Dialing grpc", log.Fields{"device-id": dh.device.Id})
 	grpc_prometheus.EnableClientHandlingTimeHistogram()
 	// Use Interceptors to automatically inject and publish Open Tracing Spans by this GRPC client
 	dh.clientCon, err = grpc.Dial(dh.device.GetHostAndPort(),
@@ -1263,7 +1263,8 @@
 // postInit create olt client instance to invoke RPC on the olt device
 func (dh *DeviceHandler) postInit(ctx context.Context) error {
 	dh.Client = oop.NewOpenoltClient(dh.clientCon)
-	dh.transitionMap.Handle(ctx, GrpcConnected)
+	// decoupling it to avoid blocking state machine transition
+	go dh.transitionMap.Handle(ctx, GrpcConnected)
 	return nil
 }
 
@@ -1315,7 +1316,22 @@
 	}
 
 	if err := dh.initializeDeviceHandlerModules(ctx); err != nil {
-		return olterrors.NewErrAdapter("device-handler-initialization-failed", log.Fields{"device-id": dh.device.Id}, err).LogAt(log.ErrorLevel)
+		logger.Error(ctx, "device-handler-initialization-failed", log.Fields{"device-id": dh.device.Id, "error": err})
+		cloned := proto.Clone(device).(*voltha.Device)
+		cloned.ConnectStatus = voltha.ConnectStatus_UNREACHABLE
+		cloned.OperStatus = voltha.OperStatus_UNKNOWN
+		dh.device = cloned
+
+		if err = dh.updateDeviceStateInCore(ctx, &ca.DeviceStateFilter{
+			DeviceId:   cloned.Id,
+			OperStatus: cloned.OperStatus,
+			ConnStatus: cloned.ConnectStatus,
+		}); err != nil {
+			logger.Error(ctx, "device-state-update-failed", log.Fields{"device-id": dh.device.Id, "error": err})
+		}
+		// decoupling it to avoid blocking state machine transition
+		go dh.transitionMap.Handle(ctx, GrpcDisconnected)
+		return nil
 	}
 
 	go dh.updateLocalDevice(ctx)
@@ -1399,7 +1415,7 @@
 	cancel()
 
 	if err != nil {
-		return nil, olterrors.NewErrPersistence("get", "device", 0, nil, err)
+		return nil, olterrors.NewErrCommunication("get-device-info-failed", log.Fields{"device-id": dh.device.Id}, err).Log()
 	}
 	if deviceInfo == nil {
 		return nil, olterrors.NewErrInvalidValue(log.Fields{"device": nil}, nil)
diff --git a/internal/pkg/core/olt_state_transitions.go b/internal/pkg/core/olt_state_transitions.go
index b012a64..7691079 100644
--- a/internal/pkg/core/olt_state_transitions.go
+++ b/internal/pkg/core/olt_state_transitions.go
@@ -21,6 +21,7 @@
 	"context"
 	"reflect"
 	"runtime"
+	"sync"
 
 	"github.com/opencord/voltha-lib-go/v7/pkg/log"
 )
@@ -72,6 +73,7 @@
 type TransitionMap struct {
 	transitions        map[Trigger]Transition
 	currentDeviceState DeviceState
+	oltTransitionMutex sync.Mutex
 }
 
 //    OpenoltDevice state machine:
@@ -147,6 +149,8 @@
 // after handlers if the transition is a valid transition
 func (tMap *TransitionMap) Handle(ctx context.Context, trigger Trigger) {
 	// Check whether the transtion is valid from current state
+	tMap.oltTransitionMutex.Lock()
+	defer tMap.oltTransitionMutex.Unlock()
 	if !tMap.isValidTransition(trigger) {
 		logger.Errorw(ctx, "invalid-transition-triggered",
 			log.Fields{