/*
 * Copyright 2018-2024 Open Networking Foundation (ONF) and the ONF Contributors

 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at

 * http://www.apache.org/licenses/LICENSE-2.0

 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package device

import (
	"context"
	"encoding/hex"
	"errors"
	"fmt"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/opencord/voltha-protos/v5/go/adapter_service"
	"github.com/opencord/voltha-protos/v5/go/core"
	"github.com/opencord/voltha-protos/v5/go/omci"

	"github.com/cenkalti/backoff/v3"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/emptypb"

	"github.com/opencord/voltha-go/rw_core/config"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"github.com/opencord/voltha-go/db/model"
	"github.com/opencord/voltha-go/rw_core/core/adapter"
	"github.com/opencord/voltha-go/rw_core/core/device/flow"
	"github.com/opencord/voltha-go/rw_core/core/device/group"
	"github.com/opencord/voltha-go/rw_core/core/device/port"
	"github.com/opencord/voltha-go/rw_core/core/device/transientstate"
	coreutils "github.com/opencord/voltha-go/rw_core/utils"
	"github.com/opencord/voltha-lib-go/v7/pkg/log"
	"github.com/opencord/voltha-protos/v5/go/common"
	ca "github.com/opencord/voltha-protos/v5/go/core_adapter"
	"github.com/opencord/voltha-protos/v5/go/extension"
	ofp "github.com/opencord/voltha-protos/v5/go/openflow_13"
	"github.com/opencord/voltha-protos/v5/go/voltha"
)

var errReconcileAborted = errors.New("reconcile aborted")
var errContextExpired = errors.New("context expired")
var errNoConnection = errors.New("no connection")
var errNoAddrChange = errors.New("no change in device address")
var errAddrDuplicate = errors.New("ip address already in use")
var errInvalidAddr = errors.New("invalid address type")

// Agent represents device agent attributes
type Agent struct {
	adapterMgr      *adapter.Manager
	deviceMgr       *Manager
	dbProxy         *model.Proxy
	exitChannel     chan int
	device          *voltha.Device
	requestQueue    *coreutils.RequestQueue
	stopReconciling chan int
	config          *config.RWCoreFlags

	flowCache            *flow.Cache
	groupCache           *group.Cache
	portLoader           *port.Loader
	transientStateLoader *transientstate.Loader
	deviceID             string
	parentID             string
	deviceType           string
	adapterEndpoint      string
	internalTimeout      time.Duration
	rpcTimeout           time.Duration
	flowTimeout          time.Duration
	stopReconcilingMutex sync.RWMutex
	startOnce            sync.Once
	stopOnce             sync.Once
	isRootDevice         bool
	stopped              bool
}

// newAgent creates a new device agent. The device will be initialized when start() is called.
func newAgent(device *voltha.Device, deviceMgr *Manager, dbPath *model.Path, deviceProxy *model.Proxy, internalTimeout, rpcTimeout, flowTimeout time.Duration) *Agent {
	deviceID := device.Id
	if deviceID == "" {
		deviceID = coreutils.CreateDeviceID()
	}

	return &Agent{
		deviceID:             deviceID,
		isRootDevice:         device.Root,
		parentID:             device.ParentId,
		deviceType:           device.Type,
		adapterEndpoint:      device.AdapterEndpoint,
		deviceMgr:            deviceMgr,
		adapterMgr:           deviceMgr.adapterMgr,
		exitChannel:          make(chan int, 1),
		dbProxy:              deviceProxy,
		internalTimeout:      internalTimeout,
		rpcTimeout:           rpcTimeout,
		flowTimeout:          flowTimeout,
		device:               proto.Clone(device).(*voltha.Device),
		requestQueue:         coreutils.NewRequestQueue(),
		config:               deviceMgr.config,
		flowCache:            flow.NewCache(),
		groupCache:           group.NewCache(),
		portLoader:           port.NewLoader(dbPath.SubPath("ports").Proxy(deviceID)),
		transientStateLoader: transientstate.NewLoader(dbPath.SubPath("core").Proxy("transientstate"), deviceID),
	}
}

// start() saves the device to the data model and registers for callbacks on that device if deviceToCreate!=nil.
// Otherwise, it will load the data from the dB and setup the necessary callbacks and proxies. Returns the device that
// was started.
func (agent *Agent) start(ctx context.Context, deviceExist bool, deviceToCreate *voltha.Device) (*voltha.Device, error) {
	needToStart := false
	if agent.startOnce.Do(func() { needToStart = true }); !needToStart {
		return agent.getDeviceReadOnly(ctx)
	}
	var startSucceeded bool
	defer func() {
		if !startSucceeded {
			if err := agent.stop(ctx); err != nil {
				logger.Errorw(ctx, "failed-to-cleanup-after-unsuccessful-start", log.Fields{"device-id": agent.deviceID, "error": err})
			}
		}
	}()
	if deviceExist {
		device := deviceToCreate
		if device == nil {
			// Load from dB
			device = &voltha.Device{}
			have, err := agent.dbProxy.Get(ctx, agent.deviceID, device)
			if err != nil {
				return nil, err
			} else if !have {
				return nil, status.Errorf(codes.NotFound, "device-%s", agent.deviceID)
			}
			logger.Infow(ctx, "device-loaded-from-db", log.Fields{"device-id": agent.deviceID, "adapter-endpoint": device.AdapterEndpoint, "type": device.Type})
		}
		agent.deviceType = device.Type
		agent.adapterEndpoint = device.AdapterEndpoint
		agent.device = proto.Clone(device).(*voltha.Device)
		// load the ports from KV to cache
		agent.portLoader.Load(ctx)
		agent.transientStateLoader.Load(ctx)
	} else {
		// Create a new device
		var desc string
		var err error
		prevState := common.AdminState_UNKNOWN
		currState := common.AdminState_UNKNOWN
		requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}

		defer func() { agent.logDeviceUpdate(ctx, &prevState, &currState, requestStatus, err, desc) }()

		// Assumption is that AdminState, FlowGroups, and Flows are uninitialized since this
		// is a new device, so populate them here before passing the device to ldProxy.Set.
		// agent.deviceId will also have been set during newAgent().
		device := (proto.Clone(deviceToCreate)).(*voltha.Device)
		device.Id = agent.deviceID
		device.AdminState = voltha.AdminState_PREPROVISIONED
		currState = device.AdminState
		if !deviceToCreate.GetRoot() && deviceToCreate.ProxyAddress != nil {
			// Set the default vlan ID to the one specified by the parent adapter.  It can be
			// overwritten by the child adapter during a device update request
			device.Vlan = deviceToCreate.ProxyAddress.ChannelId
		}

		// Save the device to the model
		if err = agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
			err = status.Errorf(codes.Aborted, "failed-adding-device-%s: %s", agent.deviceID, err)
			return nil, err
		}
		_ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, device.OperStatus, device.ConnectStatus, prevState, device, time.Now().Unix())
		requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
		agent.device = device
	}
	startSucceeded = true
	log.EnrichSpan(ctx, log.Fields{"device-id": agent.deviceID})
	logger.Debugw(ctx, "device-agent-started", log.Fields{"device-id": agent.deviceID})

	return agent.getDeviceReadOnly(ctx)
}

// stop stops the device agent.  Not much to do for now
func (agent *Agent) stop(ctx context.Context) error {
	needToStop := false
	if agent.stopOnce.Do(func() { needToStop = true }); !needToStop {
		return nil
	}
	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}
	defer agent.requestQueue.RequestComplete()

	logger.Debugw(ctx, "stopping-device-agent", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})
	// Remove the device transient loader
	if err := agent.deleteTransientState(ctx); err != nil {
		return err
	}
	// Remove the device from the KV store
	if err := agent.dbProxy.Remove(ctx, agent.deviceID); err != nil {
		return err
	}
	var parentDevice *voltha.Device
	if !agent.device.Root {
		// Get Parent Serial number for sending event
		var err error
		parentDevice, err = agent.deviceMgr.getParentDevice(ctx, agent.device)
		if err != nil {
			logger.Errorw(ctx, "getParentDevice deviceMgr map query error", log.Fields{"device-id": agent.deviceID, "parent-id": parentDevice, "error": err})
		}
	}
	// Send the device event to the message bus
	_ = agent.deviceMgr.Agent.SendDeviceDeletedEvent(ctx, agent.device, parentDevice, time.Now().Unix())

	close(agent.exitChannel)

	agent.stopped = true

	logger.Infow(ctx, "device-agent-stopped", log.Fields{"device-id": agent.deviceID, "parent-id": agent.parentID})

	return nil
}

// Load the most recent state from the KVStore for the device.
func (agent *Agent) reconcileWithKVStore(ctx context.Context) {
	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "error": err})
		return
	}
	defer agent.requestQueue.RequestComplete()
	logger.Debug(ctx, "reconciling-device-agent-devicetype")
	// TODO: context timeout
	device := &voltha.Device{}
	if have, err := agent.dbProxy.Get(ctx, agent.deviceID, device); err != nil {
		logger.Errorw(ctx, "kv-get-failed", log.Fields{"device-id": agent.deviceID, "error": err})
		return
	} else if !have {
		return // not found in kv
	}

	agent.deviceType = device.Type
	agent.device = device
	agent.adapterEndpoint = device.AdapterEndpoint
	agent.portLoader.Load(ctx)
	agent.transientStateLoader.Load(ctx)

	logger.Debugw(ctx, "reconciled-device-agent-devicetype", log.Fields{"device-id": agent.deviceID, "type": agent.deviceType})
}

// onSuccess is a common callback for scenarios where we receive a nil response following a request to an adapter
// nolint: unparam
func (agent *Agent) onSuccess(ctx context.Context, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
	if deviceUpdateLog {
		requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
		desc := "adapter-response"
		agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
		return
	}
	logger.Debugw(ctx, "successful-operation", log.Fields{"device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
}

// onFailure is a common callback for scenarios where we receive an error response following a request to an adapter
// and the only action required is to publish the failed result on kafka
// nolint: unparam
func (agent *Agent) onFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types, deviceUpdateLog bool) {
	// Send an event on kafka
	rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
	go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
		voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())

	// Log the device update event
	if deviceUpdateLog {
		requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
		desc := "adapter-response"
		agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
		return
	}
	logger.Errorw(ctx, "failed-operation", log.Fields{"error": err, "device-id": agent.deviceID, "rpc": coreutils.GetRPCMetadataFromContext(ctx)})
}

// onForceDeleteResponse is invoked following a force delete request to an adapter.
// nolint: unparam
func (agent *Agent) onForceDeleteResponse(ctx context.Context, prevState, currState *common.AdminState_Types, dErr error) {
	// Log the status
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
	if dErr != nil {
		requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	}
	agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, dErr, "adapter-force-delete-response")

	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		logger.Errorw(ctx, "failed-getting-device-request-lock", log.Fields{"device-id": agent.deviceID, "error": err})
	}
	previousDeviceTransientState := agent.getTransientState()
	newDevice := agent.cloneDeviceWithoutLock()

	// Even on a delete error response, cleaup the device in the core
	requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
	err := agent.updateDeviceWithTransientStateAndReleaseLock(ctx, newDevice,
		core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE, previousDeviceTransientState)
	if err != nil {
		requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	}
	agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, "transient-state-update")
}

// onDeleteSuccess is a common callback for scenarios where we receive a nil response following a delete request
// to an adapter.
func (agent *Agent) onDeleteSuccess(ctx context.Context, prevState, currState *common.AdminState_Types) error {
	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
		return err
	}
	previousDeviceTransientState := agent.getTransientState()
	newDevice := agent.cloneDeviceWithoutLock()
	if err := agent.updateDeviceWithTransientStateAndReleaseLock(ctx, newDevice,
		core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE, previousDeviceTransientState); err != nil {
		ctx1, cancel1 := context.WithTimeout(context.Background(), agent.rpcTimeout)               // incase of ctx cancellation, updatetranscientstate will fail , so creating a new context for updating
		if err1 := agent.updateTransientState(ctx1, core.DeviceTransientState_NONE); err1 != nil { // reset the device transient state if the transition handlers fail, so the next retry can go through
			logger.Errorf(ctx, "failed-to-reset-transient-state-to-none: %s", err1)
		}
		cancel1()
		logger.Errorw(ctx, "delete-device-failure", log.Fields{"device-id": agent.deviceID, "error": err})
		return err
	}
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
	desc := "adapter-response"
	agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, nil, desc)
	return nil
}

// onDeleteFailure is a common callback for scenarios where we receive an error response following a delete request
//
//	to an adapter and the only action required is to return the error response.
func (agent *Agent) onDeleteFailure(ctx context.Context, err error, prevState, currState *common.AdminState_Types) {
	logger.Errorw(ctx, "rpc-failed", log.Fields{"rpc": coreutils.GetRPCMetadataFromContext(ctx), "device-id": agent.deviceID, "error": err})

	// Only updating of transient state is required, no transition.
	if er := agent.updateTransientState(ctx, core.DeviceTransientState_DELETE_FAILED); er != nil {
		logger.Errorw(ctx, "failed-to-update-transient-state-as-delete-failed", log.Fields{"device-id": agent.deviceID, "error": er})
	}
	rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
	go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
		voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())

	// Log the device update event
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	desc := "adapter-response"
	agent.logDeviceUpdate(ctx, prevState, currState, requestStatus, err, desc)
}

// getDeviceReadOnly returns a device which MUST NOT be modified, but is safe to keep forever.
func (agent *Agent) getDeviceReadOnly(ctx context.Context) (*voltha.Device, error) {
	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return nil, err
	}
	defer agent.requestQueue.RequestComplete()
	return agent.device, nil
}

// getDeviceReadOnlyWithoutLock returns a device which MUST NOT be modified, but is safe to keep forever.  This is very efficient.
// The device lock MUST be held by the caller.
func (agent *Agent) getDeviceReadOnlyWithoutLock() *voltha.Device {
	return agent.device
}

// cloneDeviceWithoutLock returns a copy of the device which is safe to modify.
// The device lock MUST be held by the caller.
func (agent *Agent) cloneDeviceWithoutLock() *voltha.Device {
	return proto.Clone(agent.device).(*voltha.Device)
}

func (agent *Agent) updateDeviceTypeAndEndpoint(ctx context.Context) error {
	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}
	changed := false
	cloned := agent.cloneDeviceWithoutLock()
	if cloned.Type == "" {
		adapterType, err := agent.adapterMgr.GetAdapterType(cloned.Type)
		if err != nil {
			agent.requestQueue.RequestComplete()
			return err
		}
		cloned.Type = adapterType
		changed = true
	}

	if cloned.AdapterEndpoint == "" {
		var err error
		if cloned.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, cloned.Id, cloned.Type); err != nil {
			agent.requestQueue.RequestComplete()
			return err
		}
		agent.adapterEndpoint = cloned.AdapterEndpoint
		changed = true
	}

	if changed {
		return agent.updateDeviceAndReleaseLock(ctx, cloned)
	}
	agent.requestQueue.RequestComplete()
	return nil
}

// enableDevice activates a preprovisioned or a disable device
func (agent *Agent) enableDevice(ctx context.Context) error {
	// To preserve and use oldDevice state as prev state in new device
	var err error
	var desc string
	var prevAdminState, currAdminState common.AdminState_Types
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}

	defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}
	logger.Debugw(ctx, "enable-device", log.Fields{"device-id": agent.deviceID})

	oldDevice := agent.getDeviceReadOnlyWithoutLock()
	prevAdminState = oldDevice.AdminState

	if !agent.proceedWithRequest(oldDevice) {
		agent.requestQueue.RequestComplete()
		err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
		return err
	}
	// vol-4275 TST meeting 08/04/2021: Let EnableDevice to be called again if device is in FAILED operational state,
	// even the admin state is ENABLED.
	if oldDevice.AdminState == voltha.AdminState_ENABLED && oldDevice.OperStatus != voltha.OperStatus_FAILED {
		logger.Warnw(ctx, "device-already-enabled", log.Fields{"device-id": agent.deviceID})
		agent.requestQueue.RequestComplete()
		err = status.Errorf(codes.FailedPrecondition, "cannot-enable-an-already-enabled-device: %s", oldDevice.Id)
		return err
	}

	// Verify whether there is a device type that supports this device type
	_, err = agent.adapterMgr.GetAdapterType(oldDevice.Type)
	if err != nil {
		agent.requestQueue.RequestComplete()
		return err
	}

	// Update device adapter endpoint if not set.  This is set once by the Core and use as is by the adapters.  E.g if this is a
	// child device then the parent adapter will use this device's adapter endpoint (set here) to communicate with it.
	newDevice := agent.cloneDeviceWithoutLock()
	if newDevice.AdapterEndpoint == "" {
		if newDevice.AdapterEndpoint, err = agent.adapterMgr.GetAdapterEndpoint(ctx, newDevice.Id, newDevice.Type); err != nil {
			agent.requestQueue.RequestComplete()
			return err
		}
		agent.adapterEndpoint = newDevice.AdapterEndpoint
	}

	// Update the Admin State and set the operational state to activating before sending the request to the Adapters
	newDevice.AdminState = voltha.AdminState_ENABLED
	newDevice.OperStatus = voltha.OperStatus_ACTIVATING

	// Adopt the device if it was in pre-provision state.  In all other cases, try to re-enable it.
	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": newDevice.AdapterEndpoint,
			})
		agent.requestQueue.RequestComplete()
		return err
	}
	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
	requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
	go func() {
		defer cancel()
		if oldDevice.AdminState == voltha.AdminState_PREPROVISIONED {
			_, err = client.AdoptDevice(subCtx, newDevice)
		} else {
			_, err = client.ReEnableDevice(subCtx, newDevice)
		}
		if err == nil {
			agent.onSuccess(subCtx, nil, nil, true)
		} else {
			agent.onFailure(subCtx, err, nil, nil, true)
		}
	}()

	// Update device
	if err = agent.updateDeviceAndReleaseLock(ctx, newDevice); err != nil {
		return err
	}
	currAdminState = newDevice.AdminState
	return nil
}

// addFlowsAndGroups adds the "newFlows" and "newGroups" from the existing flows/groups and sends the update to the
// adapters
func (agent *Agent) addFlowsAndGroups(ctx context.Context, newFlows []*ofp.OfpFlowStats, newGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
	var flwResponse, grpResponse coreutils.Response
	var err error
	// if new flow list is empty then the called function returns quickly
	if flwResponse, err = agent.addFlowsToAdapter(ctx, newFlows, flowMetadata); err != nil {
		return err
	}
	// if new group list is empty then the called function returns quickly
	if grpResponse, err = agent.addGroupsToAdapter(ctx, newGroups, flowMetadata); err != nil {
		return err
	}
	if errs := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); errs != nil {
		logger.Warnw(ctx, "adapter-response", log.Fields{"device-id": agent.deviceID, "result": errs})
		return status.Errorf(codes.Aborted, "flow-failure-device-%s", agent.deviceID)
	}
	return nil
}

// deleteFlowsAndGroups removes the "flowsToDel" and "groupsToDel" from the existing flows/groups and sends the update to the
// adapters
func (agent *Agent) deleteFlowsAndGroups(ctx context.Context, flowsToDel []*ofp.OfpFlowStats, groupsToDel []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
	var flwResponse, grpResponse coreutils.Response
	var err error
	if flwResponse, err = agent.deleteFlowsFromAdapter(ctx, flowsToDel, flowMetadata); err != nil {
		return err
	}
	if grpResponse, err = agent.deleteGroupsFromAdapter(ctx, groupsToDel, flowMetadata); err != nil {
		return err
	}

	if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
		return status.Errorf(codes.Aborted, "errors-%s", res)
	}
	return nil
}

// updateFlowsAndGroups replaces the existing flows and groups with "updatedFlows" and "updatedGroups" respectively. It
// also sends the updates to the adapters
func (agent *Agent) updateFlowsAndGroups(ctx context.Context, updatedFlows []*ofp.OfpFlowStats, updatedGroups []*ofp.OfpGroupEntry, flowMetadata *ofp.FlowMetadata) error {
	var flwResponse, grpResponse coreutils.Response
	var err error
	if flwResponse, err = agent.updateFlowsToAdapter(ctx, updatedFlows, flowMetadata); err != nil {
		return err
	}
	if grpResponse, err = agent.updateGroupsToAdapter(ctx, updatedGroups, flowMetadata); err != nil {
		return err
	}

	if res := coreutils.WaitForNilOrErrorResponses(agent.flowTimeout, flwResponse, grpResponse); res != nil {
		return status.Errorf(codes.Aborted, "errors-%s", res)
	}
	return nil
}

// disableDevice disable a device
func (agent *Agent) disableDevice(ctx context.Context) error {
	var err error
	var desc string
	var prevAdminState, currAdminState common.AdminState_Types
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}
	logger.Debugw(ctx, "disable-device", log.Fields{"device-id": agent.deviceID})

	cloned := agent.cloneDeviceWithoutLock()
	prevAdminState = agent.device.AdminState

	if !agent.proceedWithRequest(cloned) {
		err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress/failed: %s", agent.deviceID)
		agent.requestQueue.RequestComplete()
		return err
	}

	if cloned.AdminState == voltha.AdminState_DISABLED {
		desc = "device-already-disabled"
		agent.requestQueue.RequestComplete()
		return nil
	}
	if cloned.AdminState == voltha.AdminState_PREPROVISIONED {
		agent.requestQueue.RequestComplete()
		err = status.Errorf(codes.FailedPrecondition, "deviceId:%s, invalid-admin-state:%s", agent.deviceID, cloned.AdminState)
		return err
	}

	// Update the Admin State and operational state before sending the request out
	cloned.AdminState = voltha.AdminState_DISABLED
	cloned.OperStatus = voltha.OperStatus_UNKNOWN

	var client adapter_service.AdapterServiceClient
	client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": cloned.AdapterEndpoint,
			})
		agent.requestQueue.RequestComplete()
		return err
	}
	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
	requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
	go func() {
		defer cancel()
		_, err = client.DisableDevice(subCtx, cloned)
		if err == nil {
			agent.onSuccess(subCtx, nil, nil, true)
		} else {
			agent.onFailure(subCtx, err, nil, nil, true)
		}
	}()

	// Update device
	if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
		return err
	}
	currAdminState = cloned.AdminState

	return nil
}

func (agent *Agent) rebootDevice(ctx context.Context) error {
	var desc string
	var err error
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		desc = err.Error()
		return err
	}
	defer agent.requestQueue.RequestComplete()
	logger.Debugw(ctx, "reboot-device", log.Fields{"device-id": agent.deviceID})

	device := agent.getDeviceReadOnlyWithoutLock()

	if device.ConnectStatus != voltha.ConnectStatus_REACHABLE {
		err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device :%s is in operstatus:%s ,connect-status:%s", agent.deviceID, device.OperStatus, device.ConnectStatus)
		return err
	}

	if !agent.proceedWithRequest(device) {
		err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed:%s", agent.deviceID)
		return err
	}

	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": device.AdapterEndpoint,
			})
		return err
	}
	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
	requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
	_, err = client.RebootDevice(subCtx, device)
	if err == nil {
		agent.onSuccess(subCtx, nil, nil, true)
	} else {
		agent.onFailure(subCtx, err, nil, nil, true)
	}
	cancel()
	return err
}

func (agent *Agent) deleteDeviceForce(ctx context.Context) error {
	logger.Debugw(ctx, "delete-device-force", log.Fields{"device-id": agent.deviceID})

	var desc string
	var err error
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}
	// Get the device Transient state, return err if it is DELETING
	previousDeviceTransientState := agent.getTransientState()
	device := agent.cloneDeviceWithoutLock()
	if !agent.isForceDeletingAllowed(previousDeviceTransientState) {
		agent.requestQueue.RequestComplete()
		err = status.Error(codes.FailedPrecondition, fmt.Sprintf("deviceId:%s, force deletion is in progress", agent.deviceID))
		return err
	}

	previousAdminState := device.AdminState
	if previousAdminState != common.AdminState_PREPROVISIONED {
		var client adapter_service.AdapterServiceClient
		client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
		if err != nil {
			logger.Errorw(ctx, "grpc-client-nil",
				log.Fields{
					"error":            err,
					"device-id":        agent.deviceID,
					"device-type":      agent.deviceType,
					"adapter-endpoint": device.AdapterEndpoint,
				})
			agent.requestQueue.RequestComplete()
			return fmt.Errorf("remote-not-reachable %w", errNoConnection)
		}
		// Use the incoming context deadline if it's longer than the configured rpcTimeout,
		// otherwise use rpcTimeout. This allows clients to specify longer timeouts for
		// operations like device delete that may take significant time.
		timeout := agent.rpcTimeout
		if deadline, ok := ctx.Deadline(); ok {
			if ctxTimeout := time.Until(deadline); ctxTimeout > timeout {
				timeout = ctxTimeout
			}
		}
		subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), timeout)
		requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
		go func() {
			defer cancel()
			_, err = client.DeleteDevice(subCtx, device)
			if err == nil {
				agent.onSuccess(subCtx, nil, nil, true)
			} else {
				agent.onFailure(subCtx, err, nil, nil, true)
			}
		}()
	}

	// Update device
	if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
		core.DeviceTransientState_FORCE_DELETING, previousDeviceTransientState); err != nil {
		ctx1, cancel1 := context.WithTimeout(context.Background(), agent.rpcTimeout)               // incase of ctx cancellation, updatetranscientstate will fail , so creating a new context for updating
		if err1 := agent.updateTransientState(ctx1, core.DeviceTransientState_NONE); err1 != nil { // reset the device transient state if the transition handlers fail, so the next retry can go through
			logger.Errorf(ctx, "failed-to-reset-transient-state-to-none: %s", err1)
		}
		cancel1()
		return err
	}
	return nil
}

func (agent *Agent) deleteDevice(ctx context.Context) error {
	logger.Debugw(ctx, "delete-device", log.Fields{"device-id": agent.deviceID})

	var desc string
	var err error
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		logger.Errorw(ctx, "can't obtain device agent lock during delete device", log.Fields{"device-id": agent.deviceID, "error": err})
		desc = err.Error()
		return err
	}

	device := agent.cloneDeviceWithoutLock()

	if !agent.proceedWithRequest(device) {
		agent.requestQueue.RequestComplete()
		err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
		return err
	}

	// Get the device Transient state, return err if it is DELETING
	previousDeviceTransientState := agent.getTransientState()

	previousAdminState := device.AdminState
	// Change the device transient state to DELETING_FROM_ADAPTER  state till the device is removed from adapters.
	currentDeviceTransientState := core.DeviceTransientState_DELETING_FROM_ADAPTER

	if previousAdminState == common.AdminState_PREPROVISIONED {
		// Change the state to DELETING POST ADAPTER RESPONSE directly as adapters have no info of the device.
		currentDeviceTransientState = core.DeviceTransientState_DELETING_POST_ADAPTER_RESPONSE
	}
	// Update device and release lock
	if err = agent.updateDeviceWithTransientStateAndReleaseLock(ctx, device,
		currentDeviceTransientState, previousDeviceTransientState); err != nil {
		ctx1, cancel1 := context.WithTimeout(context.Background(), agent.rpcTimeout)               // incase of ctx cancellation, updatetranscientstate will fail , so creating a new context for updating
		if err1 := agent.updateTransientState(ctx1, core.DeviceTransientState_NONE); err1 != nil { // reset the device transient state if the transition handlers fail, so the next retry can go through
			logger.Errorf(ctx, "failed-to-reset-transient-state-to-none: %s", err1)
		}
		cancel1()

		desc = err.Error()
		return err
	}
	// If the device was in pre-prov state (only parent device are in that state) then do not send the request to the
	// adapter
	if previousAdminState != common.AdminState_PREPROVISIONED {
		var client adapter_service.AdapterServiceClient
		client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
		if err != nil {
			logger.Errorw(ctx, "grpc-client-nil",
				log.Fields{
					"error":            err,
					"device-id":        agent.deviceID,
					"device-type":      agent.deviceType,
					"adapter-endpoint": device.AdapterEndpoint,
				})
			return status.Errorf(codes.Internal, "error-getting-grpc-client-for-adapter: %s", err)
		}
		// Use the incoming context deadline if it's longer than the configured rpcTimeout,
		// otherwise use rpcTimeout. This allows clients to specify longer timeouts for
		// operations like device delete that may take significant time.
		timeout := agent.rpcTimeout
		if deadline, ok := ctx.Deadline(); ok {
			if ctxTimeout := time.Until(deadline); ctxTimeout > timeout {
				timeout = ctxTimeout
			}
		}
		subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), timeout)
		requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
		_, err = client.DeleteDevice(subCtx, device)
		if (err == nil) || (status.Code(err) == codes.NotFound) {
			err = agent.onDeleteSuccess(subCtx, &previousAdminState, &agent.device.AdminState) // return error is the device transition update fails , so that northbound can retry

		} else {
			agent.onDeleteFailure(subCtx, err, &previousAdminState, &agent.device.AdminState)
		}
		cancel()
	}

	return err
}

func (agent *Agent) setParentID(ctx context.Context, device *voltha.Device, parentID string) error {
	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}
	logger.Debugw(ctx, "set-parent-id", log.Fields{"device-id": device.Id, "parent-id": parentID})

	cloned := agent.cloneDeviceWithoutLock()
	cloned.ParentId = parentID
	return agent.updateDeviceAndReleaseLock(ctx, cloned)
}

// getSwitchCapability retrieves the switch capability of a parent device
func (agent *Agent) getSwitchCapability(ctx context.Context) (*ca.SwitchCapability, error) {
	logger.Debugw(ctx, "get-switch-capability", log.Fields{"device-id": agent.deviceID})

	device, err := agent.getDeviceReadOnly(ctx)
	if err != nil {
		return nil, err
	}

	// Get the gRPC client
	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		return nil, err
	}

	return client.GetOfpDeviceInfo(ctx, device)
}

func (agent *Agent) onPacketFailure(ctx context.Context, err error, packet *ofp.OfpPacketOut) {
	logger.Errorw(ctx, "packet-out-error", log.Fields{
		"device-id": agent.deviceID,
		"error":     err.Error(),
		"packet":    hex.EncodeToString(packet.Data),
	})
	rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
	go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce,
		voltha.EventCategory_COMMUNICATION, nil, time.Now().Unix())
}

func (agent *Agent) packetOut(ctx context.Context, outPort uint32, packet *ofp.OfpPacketOut) error {
	if agent.deviceType == "" {
		agent.reconcileWithKVStore(ctx)
	}
	// Send packet to adapter
	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":       err,
				"device-id":   agent.deviceID,
				"device-type": agent.deviceType,
			})
		return err
	}
	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
	go func() {
		defer cancel()
		_, err := client.SendPacketOut(subCtx, &ca.PacketOut{
			DeviceId:     agent.deviceID,
			EgressPortNo: outPort,
			Packet:       packet,
		})
		if err == nil {
			agent.onSuccess(subCtx, nil, nil, false)
		} else {
			agent.onPacketFailure(subCtx, err, packet)
		}
	}()
	return nil
}

func (agent *Agent) updateDeviceUsingAdapterData(ctx context.Context, device *voltha.Device) error {
	var err error
	var desc string
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}
	logger.Debugw(ctx, "update-device-using-adapter-data", log.Fields{"device-id": device.Id})

	cloned := agent.cloneDeviceWithoutLock()
	cloned.Root = device.Root
	cloned.Vendor = device.Vendor
	cloned.Model = device.Model
	cloned.SerialNumber = device.SerialNumber
	cloned.MacAddress = device.MacAddress
	cloned.Vlan = device.Vlan
	cloned.Reason = device.Reason
	cloned.ImageDownloads = device.ImageDownloads
	cloned.OperStatus = device.OperStatus
	cloned.ConnectStatus = device.ConnectStatus
	if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
		requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
	}
	return err
}

func (agent *Agent) updateDeviceStatus(ctx context.Context, operStatus voltha.OperStatus_Types, connStatus voltha.ConnectStatus_Types) error {
	var err error
	var desc string
	opStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, opStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}

	cloned := agent.cloneDeviceWithoutLock()
	// Ensure the enums passed in are valid - they will be invalid if they are not set when this function is invoked
	if s, ok := voltha.ConnectStatus_Types_name[int32(connStatus)]; ok {
		logger.Debugw(ctx, "update-device-conn-status", log.Fields{"ok": ok, "val": s})
		cloned.ConnectStatus = connStatus
	}
	if s, ok := voltha.OperStatus_Types_name[int32(operStatus)]; ok {
		logger.Debugw(ctx, "update-device-oper-status", log.Fields{"ok": ok, "val": s})
		cloned.OperStatus = operStatus
	}
	logger.Debugw(ctx, "update-device-status", log.Fields{"device-id": cloned.Id, "oper-status": cloned.OperStatus, "connect-status": cloned.ConnectStatus})
	// Store the device
	if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
		opStatus.Code = common.OperationResp_OPERATION_SUCCESS
	}
	return err
}

// TODO: A generic device update by attribute
func (agent *Agent) updateDeviceAttribute(ctx context.Context, name string, value interface{}) {
	if value == nil {
		return
	}

	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		logger.Warnw(ctx, "request-aborted", log.Fields{"device-id": agent.deviceID, "name": name, "error": err})
		return
	}

	cloned := agent.cloneDeviceWithoutLock()
	updated := false
	s := reflect.ValueOf(cloned).Elem()
	if s.Kind() == reflect.Struct {
		// exported field
		f := s.FieldByName(name)
		if f.IsValid() && f.CanSet() {
			switch f.Kind() {
			case reflect.String:
				f.SetString(value.(string))
				updated = true
			case reflect.Uint32:
				f.SetUint(uint64(value.(uint32)))
				updated = true
			case reflect.Bool:
				f.SetBool(value.(bool))
				updated = true
			default:
				// Fallback for enums and other user-defined types
				v := reflect.ValueOf(value)
				if v.IsValid() && v.Type().AssignableTo(f.Type()) {
					f.Set(v)
					updated = true
				}

			}
		}
	}
	logger.Debugw(ctx, "update-field-status", log.Fields{"device-id": cloned.Id, "name": name, "updated": updated})
	//	Save the data

	if err := agent.updateDeviceAndReleaseLock(ctx, cloned); err != nil {
		logger.Warnw(ctx, "attribute-update-failed", log.Fields{"attribute": name, "value": value})
	}
}

func (agent *Agent) simulateAlarm(ctx context.Context, simulateReq *voltha.SimulateAlarmRequest) error {
	var err error
	var desc string
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}
	defer agent.requestQueue.RequestComplete()
	logger.Debugw(ctx, "simulate-alarm", log.Fields{"device-id": agent.deviceID})

	device := agent.getDeviceReadOnlyWithoutLock()

	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": device.AdapterEndpoint,
			})
		return err
	}
	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
	requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
	go func() {
		defer cancel()
		_, err := client.SimulateAlarm(subCtx, &ca.SimulateAlarmMessage{Device: device, Request: simulateReq})
		if err == nil {
			agent.onSuccess(subCtx, nil, nil, false)
		} else {
			agent.onFailure(subCtx, err, nil, nil, false)
		}
	}()
	return nil
}

// This function updates the device in the DB, releases the device lock, and runs any state transitions.
// The calling function MUST hold the device lock.  The caller MUST NOT modify the device after this is called.
func (agent *Agent) updateDeviceAndReleaseLock(ctx context.Context, device *voltha.Device) error {
	// fail early if this agent is no longer valid
	if agent.stopped {
		agent.requestQueue.RequestComplete()
		return errors.New("device-agent-stopped")
	}

	// update in db
	if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
		agent.requestQueue.RequestComplete()
		return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
	}
	logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})

	prevDevice := agent.device
	// update the device
	agent.device = device
	// If any of the states has chenged, send the change event.
	if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
		_ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
	}
	deviceTransientState := agent.getTransientState()
	prevDeviceTransientState := deviceTransientState
	// Update the transient state to olt reboot in progress if the device is root and oper status is rebooted
	if device.Root && device.OperStatus == common.OperStatus_REBOOTED && prevDevice.OperStatus != common.OperStatus_REBOOTED {
		if err := agent.updateTransientState(ctx, core.DeviceTransientState_REBOOT_IN_PROGRESS); err != nil {
			return err
		}
		deviceTransientState = core.DeviceTransientState_REBOOT_IN_PROGRESS
	}
	// release lock before processing transition
	agent.requestQueue.RequestComplete()
	subCtx := coreutils.WithSpanAndRPCMetadataFromContext(ctx)

	if err := agent.deviceMgr.stateTransitions.ProcessTransition(subCtx,
		device, prevDevice, deviceTransientState, prevDeviceTransientState); err != nil {
		logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
		// Sending RPC EVENT here
		rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
		agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
			nil, time.Now().Unix())

	}
	return nil
}

// This function updates the device transient in the DB through loader, releases the device lock, and runs any state transitions.
// The calling function MUST hold the device lock.  The caller MUST NOT modify the device after this is called.
func (agent *Agent) updateDeviceWithTransientStateAndReleaseLock(ctx context.Context, device *voltha.Device,
	transientState, prevTransientState core.DeviceTransientState_Types) error {
	// fail early if this agent is no longer valid
	if agent.stopped {
		agent.requestQueue.RequestComplete()
		return errors.New("device-agent-stopped")
	}
	// update device TransientState
	if err := agent.updateTransientState(ctx, transientState); err != nil {
		agent.requestQueue.RequestComplete()
		return err
	}
	// update in db
	if err := agent.dbProxy.Set(ctx, agent.deviceID, device); err != nil {
		// Reverting TransientState update
		if errTransient := agent.updateTransientState(ctx, prevTransientState); errTransient != nil {
			logger.Errorw(ctx, "failed-to-revert-transient-state-update-on-error", log.Fields{"device-id": device.Id,
				"previous-transient-state": prevTransientState, "current-transient-state": transientState, "error": errTransient})
		}
		agent.requestQueue.RequestComplete()
		return status.Errorf(codes.Internal, "failed-update-device:%s: %s", agent.deviceID, err)
	}

	logger.Debugw(ctx, "updated-device-in-store", log.Fields{"device-id: ": agent.deviceID})

	prevDevice := agent.device
	// update the device
	agent.device = device
	// If any of the states has chenged, send the change event.
	if prevDevice.OperStatus != device.OperStatus || prevDevice.ConnectStatus != device.ConnectStatus || prevDevice.AdminState != device.AdminState {
		_ = agent.deviceMgr.Agent.SendDeviceStateChangeEvent(ctx, prevDevice.OperStatus, prevDevice.ConnectStatus, prevDevice.AdminState, device, time.Now().Unix())
	}

	// release lock before processing transition
	agent.requestQueue.RequestComplete()

	if err := agent.deviceMgr.stateTransitions.ProcessTransition(ctx,
		device, prevDevice, transientState, prevTransientState); err != nil {
		logger.Errorw(ctx, "failed-process-transition", log.Fields{"device-id": device.Id, "previous-admin-state": prevDevice.AdminState, "current-admin-state": device.AdminState})
		// Sending RPC EVENT here
		rpce := agent.deviceMgr.NewRPCEvent(ctx, agent.deviceID, err.Error(), nil)
		go agent.deviceMgr.SendRPCEvent(ctx, "RPC_ERROR_RAISE_EVENT", rpce, voltha.EventCategory_COMMUNICATION,
			nil, time.Now().Unix())
		return err
	}

	return nil
}
func (agent *Agent) updateDeviceReason(ctx context.Context, reason string) error {
	logger.Debugw(ctx, "update-device-reason", log.Fields{"device-id": agent.deviceID, "reason": reason})

	var err error
	var desc string
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}

	cloned := agent.cloneDeviceWithoutLock()
	cloned.Reason = reason
	if err = agent.updateDeviceAndReleaseLock(ctx, cloned); err == nil {
		requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
	}
	return err
}

func (agent *Agent) ChildDeviceLost(ctx context.Context, device *voltha.Device) error {
	logger.Debugw(ctx, "child-device-lost", log.Fields{"child-device-id": device.Id, "parent-device-id": agent.deviceID})

	var err error
	var desc string
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	// Remove the associated peer ports on the parent device
	if portHandle, have := agent.portLoader.Lock(device.ParentPortNo); have {
		oldPort := portHandle.GetReadOnly()
		updatedPeers := make([]*voltha.Port_PeerPort, 0)
		for _, peerPort := range oldPort.Peers {
			if peerPort.DeviceId != device.Id {
				updatedPeers = append(updatedPeers, peerPort)
			}
		}
		newPort := proto.Clone(oldPort).(*voltha.Port)
		newPort.Peers = updatedPeers
		if err = portHandle.Update(ctx, newPort); err != nil {
			portHandle.Unlock()
			return nil
		}
		portHandle.Unlock()
	}
	if err = agent.deviceMgr.canAdapterRequestProceed(ctx, agent.deviceID); err != nil {
		logger.Errorw(ctx, "adapter-request-cannot-proceed", log.Fields{"device-id": agent.deviceID, "error": err})
		return nil // as we are returning the err for childdevice lost call , and canAdapterRequestProceed will fail for forceDeleteDevice of OLT , so returning nil here
	}
	// send request to adapter
	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": device.AdapterEndpoint,
			})
		return err
	}
	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
	requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS

	defer cancel()
	_, err = client.ChildDeviceLost(subCtx, device)
	if err == nil {
		agent.onSuccess(subCtx, nil, nil, true)
	} else {
		agent.onFailure(subCtx, err, nil, nil, true)
	}

	return nil
}

func (agent *Agent) startOmciTest(ctx context.Context, omcitestrequest *omci.OmciTestRequest) (*omci.TestResponse, error) {
	var err error
	var desc string
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	// OMCI test may be performed on a pre-provisioned device.  If a device is in that state both its device type and endpoint
	// may not have been set yet.
	// First check if we need to update the type or endpoint
	cloned, err := agent.getDeviceReadOnly(ctx)
	if err != nil {
		return nil, err
	}
	if cloned.Type == "" || cloned.AdapterEndpoint == "" {
		if err = agent.updateDeviceTypeAndEndpoint(ctx); err != nil {
			return nil, err
		}
		cloned, err = agent.getDeviceReadOnly(ctx)
		if err != nil {
			return nil, err
		}
	}

	// Send request to the adapter
	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": cloned.AdapterEndpoint,
			})
		return nil, err
	}

	res, err := client.StartOmciTest(ctx, &ca.OMCITest{
		Device:  cloned,
		Request: omcitestrequest,
	})
	if err == nil {
		requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
	}
	return res, err
}

func (agent *Agent) getExtValue(ctx context.Context, pdevice *voltha.Device, cdevice *voltha.Device, valueparam *extension.ValueSpecifier) (*extension.ReturnValues, error) {
	logger.Debugw(ctx, "get-ext-value", log.Fields{"device-id": agent.deviceID, "onu-id": valueparam.Id, "value-type": valueparam.Value})
	var err error
	var desc string
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return nil, err
	}

	// send request to adapter synchronously
	client, err := agent.adapterMgr.GetAdapterClient(ctx, pdevice.AdapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": pdevice.AdapterEndpoint,
			})
		agent.requestQueue.RequestComplete()
		return nil, err
	}

	// Release lock before sending to adapter
	agent.requestQueue.RequestComplete()

	retVal, err := client.GetExtValue(ctx, &ca.GetExtValueMessage{
		ParentDevice: pdevice,
		ChildDevice:  cdevice,
		ValueType:    valueparam.Value,
	})
	if err == nil {
		requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
	}
	return retVal, err
}

func (agent *Agent) setExtValue(ctx context.Context, device *voltha.Device, value *extension.ValueSet) (*emptypb.Empty, error) {
	logger.Debugw(ctx, "set-ext-value", log.Fields{"device-id": value.Id})

	var err error
	var desc string
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return nil, err
	}

	// send request to adapter
	// send request to adapter synchronously
	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": device.AdapterEndpoint,
			})
		agent.requestQueue.RequestComplete()
		return nil, err
	}
	// Release lock before sending request to adapter
	agent.requestQueue.RequestComplete()

	retVal, err := client.SetExtValue(ctx, &ca.SetExtValueMessage{
		Device: device,
		Value:  value,
	})
	if err == nil {
		requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
	}
	return retVal, err
}

func (agent *Agent) getSingleValue(ctx context.Context, request *extension.SingleGetValueRequest) (*extension.SingleGetValueResponse, error) {
	logger.Debugw(ctx, "get-single-value", log.Fields{"device-id": request.TargetId})

	var err error
	var desc string
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return nil, err
	}

	cloned := agent.cloneDeviceWithoutLock()

	// send request to adapter
	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        cloned.Id,
				"adapter-endpoint": cloned.AdapterEndpoint,
			})
		agent.requestQueue.RequestComplete()
		return nil, err
	}
	// Release lock before sending request to adapter
	agent.requestQueue.RequestComplete()

	resp, err := client.GetSingleValue(ctx, request)
	if err == nil {
		requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
	}
	return resp, err
}

func (agent *Agent) setSingleValue(ctx context.Context, request *extension.SingleSetValueRequest) (*extension.SingleSetValueResponse, error) {
	logger.Debugw(ctx, "set-single-value", log.Fields{"device-id": request.TargetId})

	var err error
	var desc string
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return nil, err
	}

	cloned := agent.cloneDeviceWithoutLock()

	// send request to adapter
	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": cloned.AdapterEndpoint,
			})
		agent.requestQueue.RequestComplete()
		return nil, err
	}
	// Release lock before sending request to adapter
	agent.requestQueue.RequestComplete()

	resp, err := client.SetSingleValue(ctx, request)
	if err == nil {
		requestStatus.Code = common.OperationResp_OPERATION_SUCCESS
	}
	return resp, err
}

func (agent *Agent) proceedWithRequest(device *voltha.Device) bool {
	return !agent.isDeletionInProgress() && !agent.isInReconcileState(device)
}

func (agent *Agent) stopReconcile() {
	agent.stopReconcilingMutex.Lock()
	if agent.stopReconciling != nil {
		close(agent.stopReconciling)
	}
	agent.stopReconcilingMutex.Unlock()
}

// abortAllProcessing is invoked when an adapter managing this device is restarted
func (agent *Agent) abortAllProcessing(ctx context.Context) error {
	logger.Infow(ctx, "aborting-current-running-requests", log.Fields{"device-id": agent.deviceID})
	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}
	logger.Infow(ctx, "aborting-current-running-requests-after-wait", log.Fields{"device-id": agent.deviceID})

	defer agent.requestQueue.RequestComplete()

	// If any reconciling is in progress just abort it. The adapter is gone.
	agent.stopReconcile()
	agent.stopReconcilingMutex.Lock()
	agent.stopReconciling = nil
	agent.stopReconcilingMutex.Unlock()

	logger.Infow(ctx, "aborting-current-running-requests-after-sendstop", log.Fields{"device-id": agent.deviceID})

	// Update the Core device transient state accordingly
	var updatedState core.DeviceTransientState_Types
	switch agent.getTransientState() {
	case core.DeviceTransientState_RECONCILE_IN_PROGRESS:
		updatedState = core.DeviceTransientState_NONE
	case core.DeviceTransientState_FORCE_DELETING:
		updatedState = core.DeviceTransientState_DELETE_FAILED
	case core.DeviceTransientState_DELETING_FROM_ADAPTER:
		updatedState = core.DeviceTransientState_DELETE_FAILED
	case core.DeviceTransientState_DELETE_FAILED:
	case core.DeviceTransientState_REBOOT_IN_PROGRESS:
		// do not change state
		return nil
	default:
		updatedState = core.DeviceTransientState_NONE
	}
	if err := agent.updateTransientState(ctx, updatedState); err != nil {
		logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
		return err
	}
	return nil
}

func (agent *Agent) DeleteDevicePostAdapterRestart(ctx context.Context) error {
	logger.Debugw(ctx, "delete-post-restart", log.Fields{"device-id": agent.deviceID})
	ctx = coreutils.WithNewSpanAndRPCMetadataContext(ctx, "DelteDevicePostAdapterRestart")

	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}

	device := agent.getDeviceReadOnlyWithoutLock()
	if device.AdminState == voltha.AdminState_PREPROVISIONED {
		logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
		agent.requestQueue.RequestComplete()
		return nil
	}
	// Change device transient state to FORCE_DELETING
	if err := agent.updateTransientState(ctx, core.DeviceTransientState_FORCE_DELETING); err != nil {
		logger.Errorw(ctx, "failure-updating-transient-state", log.Fields{"error": err, "device-id": agent.deviceID})
		agent.requestQueue.RequestComplete()
		return err
	}

	// Ensure we have a valid grpc client available as we have just restarted
	deleteBackoff := backoff.NewExponentialBackOff()
	deleteBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
	deleteBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
	deleteBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval
	var backoffTimer *time.Timer
	var err error
	var client adapter_service.AdapterServiceClient
retry:
	for {
		client, err = agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
		if err == nil {
			break retry
		}
		duration := deleteBackoff.NextBackOff()
		if duration == backoff.Stop {
			deleteBackoff.Reset()
			duration = deleteBackoff.NextBackOff()
		}
		backoffTimer = time.NewTimer(duration)
		select {
		case <-backoffTimer.C:
			logger.Debugw(ctx, "backoff-timer-expires", log.Fields{"device-id": agent.deviceID})
		case <-ctx.Done():
			err = ctx.Err()
			break retry
		}
	}
	if backoffTimer != nil && !backoffTimer.Stop() {
		select {
		case <-backoffTimer.C:
		default:
		}
	}
	if err != nil || client == nil {
		agent.requestQueue.RequestComplete()
		return err
	}

	// Release the device lock to allow for device state update, if any
	agent.requestQueue.RequestComplete()

	// Send the delete request to the adapter
	// Use the incoming context deadline if it's longer than the configured rpcTimeout,
	// otherwise use rpcTimeout. This allows clients to specify longer timeouts for
	// operations like device delete that may take significant time.
	timeout := agent.rpcTimeout
	if deadline, ok := ctx.Deadline(); ok {
		if ctxTimeout := time.Until(deadline); ctxTimeout > timeout {
			timeout = ctxTimeout
		}
	}
	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), timeout)
	defer cancel()
	_, err = client.DeleteDevice(subCtx, device)
	agent.onForceDeleteResponse(subCtx, nil, nil, err)
	return err
}

func (agent *Agent) UpdateTransientStateToReconcile(ctx context.Context) error {
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	var desc string

	// set transient state to RECONCILE IN PROGRESS
	err := agent.updateTransientState(ctx, core.DeviceTransientState_RECONCILE_IN_PROGRESS)
	if err != nil {
		logger.Errorw(ctx, "setting-transient-state-failed", log.Fields{"error": err})
		agent.logDeviceUpdate(ctx, nil, nil, requestStatus, nil, desc)
		return err
	}
	return nil
}

func (agent *Agent) StartReconcileWithRetry(ctx context.Context) {

	var reconcileErr error
	state := agent.getTransientState()
	logger.Debugw(ctx, "starting-reconcile", log.Fields{"device-id": agent.deviceID, "state": state})

	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	var desc string

	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
		return
	}

	device := agent.getDeviceReadOnlyWithoutLock()
	if device.AdminState == voltha.AdminState_PREPROVISIONED {
		agent.requestQueue.RequestComplete()
		logger.Debugw(ctx, "device-in-preprovisioning-state-reconcile-not-needed", log.Fields{"device-id": device.Id})
		return
	}

	if agent.isDeletionInProgress() {
		agent.requestQueue.RequestComplete()
		err := fmt.Errorf("cannot complete operation as device deletion is in progress device : %s", device.Id)
		logger.Errorw(ctx, "reconcile-failed", log.Fields{"error": err})
		agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
		return
	}

	reconcilingBackoff := backoff.NewExponentialBackOff()
	reconcilingBackoff.InitialInterval = agent.config.BackoffRetryInitialInterval
	reconcilingBackoff.MaxElapsedTime = agent.config.BackoffRetryMaxElapsedTime
	reconcilingBackoff.MaxInterval = agent.config.BackoffRetryMaxInterval

	// making here to keep lifecycle of this channel within the scope of retryReconcile
	agent.stopReconcilingMutex.Lock()
	if agent.stopReconciling != nil {
		logger.Warnw(ctx, "Reconciling with retries is already in progress, don't proceed further", log.Fields{"device-id": device.Id})
		agent.stopReconcilingMutex.Unlock()
		agent.requestQueue.RequestComplete()
		return
	} else {
		agent.stopReconciling = make(chan int, 1)
	}
	agent.stopReconcilingMutex.Unlock()

	// defined outside the retry loop so it can be cleaned
	// up when the loop breaks
	var backoffTimer *time.Timer
	isDeviceReconciledErr := false

retry:
	for {

		// Use an exponential back off to prevent getting into a tight loop
		duration := reconcilingBackoff.NextBackOff()
		// This case should never occur in default case as max elapsed time for backoff is 0(by default) , so it will never return stop
		if duration == backoff.Stop {
			// If we have received device reconciled error and the retry intervals have elapsed
			// clean up the reconcile and break the retry loop
			if isDeviceReconciledErr {
				logger.Warnw(ctx, "reached max retry with device reconciled error", log.Fields{"max-reconciling-backoff": reconcilingBackoff.MaxElapsedTime,
					"device-id": device.Id})
				// Release lock before reconcile clean up
				agent.requestQueue.RequestComplete()
				err := agent.reconcilingCleanup(ctx)
				if err != nil {
					logger.Errorf(ctx, "Error during reconcile cleanup", err.Error())
				}
				break retry
			}
			// If we reach a maximum then warn and reset the backoff
			// timer and keep attempting.
			logger.Warnw(ctx, "maximum-reconciling-backoff-reached--resetting-backoff-timer",
				log.Fields{"max-reconciling-backoff": reconcilingBackoff.MaxElapsedTime,
					"device-id": device.Id})
			reconcilingBackoff.Reset()
			duration = reconcilingBackoff.NextBackOff()
		}

		backoffTimer = time.NewTimer(duration)

		logger.Debugw(ctx, "retrying-reconciling", log.Fields{"deviceID": device.Id, "endpoint": device.AdapterEndpoint})
		// Release lock before sending request to adapter
		agent.requestQueue.RequestComplete()

		if state != core.DeviceTransientState_REBOOT_IN_PROGRESS {
			var err error
			logger.Debugw(ctx, "reconciling-state", log.Fields{"deviceID": device.Id, "endpoint": device.AdapterEndpoint})
			if state != core.DeviceTransientState_RECONCILE_IN_PROGRESS {
				// set transient state to RECONCILE IN PROGRESS
				if err = agent.UpdateTransientStateToReconcile(ctx); err != nil {
					logger.Warn(ctx, "setting-transient-state-failed, will be retried", log.Fields{"error": err})
				}
			}
			if err == nil {
				reconcileErr = agent.sendReconcileRequestToAdapter(ctx, device)
			}
		}

		if errors.Is(reconcileErr, errContextExpired) || errors.Is(reconcileErr, errReconcileAborted) {
			logger.Errorw(ctx, "reconcile-aborted", log.Fields{"error": reconcileErr})
			requestStatus = &common.OperationResp{Code: common.OperationResp_OperationReturnCode(common.OperStatus_FAILED)}
			desc = "aborted"
			agent.logDeviceUpdate(ctx, nil, nil, requestStatus, reconcileErr, desc)
			break retry
		}
		st, ok := status.FromError(reconcileErr)
		if ok {
			// Decode the error code and error message
			errorCode := st.Code()
			// When the device is deleted and the core is still retrying the reconcile, adapter will send Not Found error to core when DeviceStateUpdate fails.
			// This will enable core to stop the reconcile for the deleted device.
			/*
				 As the grcp connection is now reused (on adapters restart), there could be two scenarios where we could land
				 in a stiuation where the reconcile is in a loop for already reconciled ONUs.
				 1	If the reconcile lands on an older instance of the adapters we get this error "device already reconciled",
					   in which case core will retry. Once the new adapter instance comes up and old adapter instance is stopped, this will go through successful.
				 2	If the core restarts during reconcile of device, the reconcile will be done by the adapter but before
					   the response reaches the core could have restarted. This leaves the tansient state of the device in reconciling.
					   Once the core comes up , the reconcile is fired again and as the device was previously reconciled adapter returns this error.
					 Either of the cases we retry for retryinterval and assume that the device is already reconciled in the new adapter.
			*/
			if errorCode == codes.AlreadyExists {
				logger.Warnw(ctx, "device already reconciled", log.Fields{"error": reconcileErr})
				// Reset the backoff current interval when we receive the device reconciled error for the first time
				// so that we can retry for one full backoff cycle and then break
				if !isDeviceReconciledErr {
					isDeviceReconciledErr = true
					reconcilingBackoff.Reset()
				}
			} else {
				isDeviceReconciledErr = false
			}
		}
		// If the reconcile request fails or the device transient state is in OLT reboot in progress state, retry the reconcile
		if reconcileErr != nil || state == core.DeviceTransientState_REBOOT_IN_PROGRESS {
			agent.logDeviceUpdate(ctx, nil, nil, requestStatus, reconcileErr, desc)
			select {
			case <-backoffTimer.C:
				// backoffTimer expired continue
				// Take lock back before retrying
				if reconcileErr = agent.requestQueue.WaitForGreenLight(ctx); reconcileErr != nil {
					desc = "failed-to-acquire-lock"
					agent.logDeviceUpdate(ctx, nil, nil, requestStatus, reconcileErr, desc)
					break retry
				}
				state = agent.getTransientState()
				continue

			case _, ok := (<-agent.exitChannel):
				// If the device is deleted, the agent would be stopped. Check for agent status before retrying the reconcile
				if !ok {
					logger.Warnw(ctx, "device agent stopped, aborting reconcile", log.Fields{"device-id": agent.deviceID})
					desc = "device-agent-stopped"
					agent.logDeviceUpdate(ctx, nil, nil, requestStatus, reconcileErr, desc)
					break retry
				}
			case _, ok := <-agent.stopReconciling:
				// This case is executed when the reconciling request is either not sent to the adapter
				// or fails in the adapter, and the reconciling aborted by the abortAllProcessing function.
				if !ok {
					logger.Warnw(ctx, "Stop Reconciling channel closed", log.Fields{"device-id": agent.deviceID})
					err := fmt.Errorf("reconciling channel closed:%w", errReconcileAborted)
					desc = "reconciling-channel-closed"
					agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
				}
				break retry
			}
		}
		// Success
		requestStatus = &common.OperationResp{Code: common.OperationResp_OPERATION_SUCCESS}
		desc = "adapter-response"
		agent.logDeviceUpdate(ctx, nil, nil, requestStatus, reconcileErr, desc)
		break retry
	}

	logger.Debugw(ctx, "reconcile-retry-ends", log.Fields{"adapter-endpoint": agent.adapterEndpoint})

	// Retry loop is broken, so stop any timers and drain the channel
	if backoffTimer != nil && !backoffTimer.Stop() {

		// As per documentation and stack overflow when a timer is stopped its
		// channel should be drained. The issue is that Stop returns false
		// either if the timer has already been fired "OR" if the timer can be
		// stopped before being fired. This means that in some cases the
		// channel has already be emptied so attempting to read from it means
		// a blocked thread. To get around this use a select so if the
		// channel is already empty the default case hits and we are not
		// blocked.
		select {
		case <-backoffTimer.C:
		default:
		}
	}
}

func (agent *Agent) ReconcileDevice(ctx context.Context) {

	agent.StartReconcileWithRetry(ctx)
}

func (agent *Agent) sendReconcileRequestToAdapter(ctx context.Context, device *voltha.Device) error {
	logger.Debugw(ctx, "sending-reconcile-to-adapter", log.Fields{"device-id": device.Id, "endpoint": agent.adapterEndpoint})
	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		return err
	}
	adapterResponse := make(chan error)
	go func() {
		_, err = client.ReconcileDevice(ctx, device)
		adapterResponse <- err
	}()
	select {
	// wait for response
	case err = <-adapterResponse:
		if err != nil {
			return err
		}
		// In case of success quit retrying and wait for adapter to reset operation state of device
		agent.stopReconcilingMutex.Lock()
		agent.stopReconciling = nil
		agent.stopReconcilingMutex.Unlock()
		return nil

	// if reconciling need to be stopped
	case _, ok := <-agent.stopReconciling:
		if !ok {
			// channel-closed
			return fmt.Errorf("reconcile channel closed:%w", errReconcileAborted)
		}
		return fmt.Errorf("reconciling aborted:%w", errReconcileAborted)
	// Context expired
	case <-ctx.Done():
		agent.stopReconcilingMutex.Lock()
		agent.stopReconciling = nil
		agent.stopReconcilingMutex.Unlock()
		return fmt.Errorf("context expired:%s :%w", ctx.Err(), errContextExpired)
	}
}

func (agent *Agent) reconcilingCleanup(ctx context.Context) error {
	var desc string
	var err error
	operStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, operStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		desc = "reconcile-cleanup-failed"
		return err
	}
	defer agent.requestQueue.RequestComplete()
	agent.stopReconcile()
	agent.stopReconcilingMutex.Lock()
	agent.stopReconciling = nil
	agent.stopReconcilingMutex.Unlock()
	err = agent.updateTransientState(ctx, core.DeviceTransientState_NONE)
	if err != nil {
		logger.Errorf(ctx, "transient-state-update-failed", log.Fields{"error": err})
		return err
	}
	operStatus.Code = common.OperationResp_OPERATION_SUCCESS
	return nil
}

func (agent *Agent) isAdapterConnectionUp(ctx context.Context) bool {
	c, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	return c != nil && err == nil
}

func (agent *Agent) canDeviceRequestProceed(ctx context.Context) error {
	if err := agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}
	defer agent.requestQueue.RequestComplete()
	if agent.proceedWithRequest(agent.device) {
		return nil
	}
	return fmt.Errorf("device-cannot-process-request-%s", agent.deviceID)
}

func (agent *Agent) disableOnuDevice(ctx context.Context, onuAgent *Agent) error {
	var err error
	var desc string
	var prevAdminState, currAdminState common.AdminState_Types
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}

	defer func() {
		onuAgent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc)
	}()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		desc = "Failed while waiting for green light"
		return err
	}

	defer agent.requestQueue.RequestComplete()

	parentDevice := agent.getDeviceReadOnlyWithoutLock()

	if parentDevice.ConnectStatus != voltha.ConnectStatus_REACHABLE {
		err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device :%s is in operstatus:%s ,connect-status:%s", agent.deviceID, parentDevice.OperStatus, parentDevice.ConnectStatus)
		agent.requestQueue.RequestComplete()
		return err
	}

	logger.Debugw(ctx, "disable-child-device", log.Fields{"device-id": onuAgent.deviceID, "serial-number": onuAgent.device.SerialNumber})

	prevAdminState = onuAgent.device.AdminState

	cloned, desc, err := onuAgent.checkDisableEligibility()
	if err != nil {
		return err
	}

	cloned.AdminState = voltha.AdminState_DISABLED

	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": agent.adapterEndpoint,
			})
		desc = fmt.Sprintf("failed to get adapter client for endpoint %s", agent.adapterEndpoint)
		return err
	}

	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
	defer cancel()
	requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS

	resultCh := make(chan error, 1)

	go func() {
		if _, callErr := client.DisableOnuDevice(subCtx, cloned); callErr == nil {
			logger.Infow(subCtx, "disable-child-device-success", log.Fields{"device-id": cloned.Id, "serial-number": cloned.SerialNumber})
			agent.onSuccess(subCtx, nil, nil, true)
			resultCh <- nil
		} else {
			logger.Errorw(subCtx, "disable-child-device-failed", log.Fields{"device-id": cloned.Id, "serial-number": cloned.SerialNumber, "error": callErr})
			agent.onFailure(subCtx, callErr, nil, nil, true)
			trimmedErr := extractLastRpcError(callErr)
			desc = fmt.Sprintf("disableOnuDevice call failed: %s", trimmedErr)
			resultCh <- fmt.Errorf("disable-child-device-failed: %s", trimmedErr)
		}
	}()

	select {
	case resultErr := <-resultCh:
		if resultErr != nil {
			return resultErr
		}
	case <-ctx.Done():
		err = fmt.Errorf("operation cancelled or timed out: %w", ctx.Err())
		desc = "disable ONU device operation cancelled or timed out"
		return err
	}

	onuAgent.updateDeviceAttribute(subCtx, "AdminState", cloned.AdminState)

	currAdminState = cloned.AdminState
	return nil
}

func (agent *Agent) enableOnuDevice(ctx context.Context, onuAgent *Agent) error {
	var err error
	var desc string
	var prevAdminState, currAdminState common.AdminState_Types
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}

	defer func() { onuAgent.logDeviceUpdate(ctx, &prevAdminState, &currAdminState, requestStatus, err, desc) }()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		desc = "Failed while waiting for green light"
		return err
	}

	defer agent.requestQueue.RequestComplete()
	parentDevice := agent.getDeviceReadOnlyWithoutLock()

	if parentDevice.ConnectStatus != voltha.ConnectStatus_REACHABLE {
		err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device :%s is in operstatus:%s ,connect-status:%s", agent.deviceID, parentDevice.OperStatus, parentDevice.ConnectStatus)
		return err
	}

	logger.Debugw(ctx, "enable-child-device", log.Fields{"device-id": onuAgent.deviceID, "serial-number": onuAgent.device.SerialNumber})

	prevAdminState = onuAgent.device.AdminState

	newDevice, desc, err := onuAgent.checkEnableEligibility()

	if err != nil {
		return err
	}

	// Update the Admin State and set the operational state to activating before sending the request to the Adapters
	newDevice.AdminState = voltha.AdminState_ENABLED

	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": agent.adapterEndpoint,
			})

		desc = fmt.Sprintf("failed to get adapter client for endpoint %s", agent.adapterEndpoint)
		return err
	}
	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
	defer cancel()
	requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS

	resultCh := make(chan error, 1)

	go func() {
		_, callErr := client.EnableOnuDevice(subCtx, newDevice)

		if callErr == nil {
			logger.Infow(subCtx, "enable-child-device-success", log.Fields{"device-id": newDevice.Id, "serial-number": newDevice.SerialNumber})
			agent.onSuccess(subCtx, nil, nil, true)
			resultCh <- nil
		} else {
			logger.Errorw(subCtx, "enable-child-device-failed", log.Fields{"device-id": newDevice.Id, "serial-number": newDevice.SerialNumber, "error": callErr})
			agent.onFailure(subCtx, callErr, nil, nil, true)
			trimmedErr := extractLastRpcError(callErr)
			desc = fmt.Sprintf(" enableOnuDevice call failed: %s", trimmedErr)
			resultCh <- fmt.Errorf("enable-child-device-failed: %s", trimmedErr)
		}
	}()

	select {
	case resultErr := <-resultCh:
		if resultErr != nil {
			return resultErr
		}
	case <-ctx.Done():
		err = fmt.Errorf("operation cancelled or timed out: %w", ctx.Err())
		desc = "enable ONU device operation cancelled or timed out"
		return err
	}

	onuAgent.updateDeviceAttribute(subCtx, "AdminState", newDevice.AdminState)

	currAdminState = newDevice.AdminState
	return nil

}

func (agent *Agent) disableOnuSerialNumber(ctx context.Context, device *voltha.OnuSerialNumberOnOLTPon) error {
	var err error
	var desc string
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}

	defer func() {
		agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
	}()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		desc = "Failed while waiting for green light"
		return err
	}

	defer agent.requestQueue.RequestComplete()
	parentDevice := agent.getDeviceReadOnlyWithoutLock()

	if parentDevice.ConnectStatus != voltha.ConnectStatus_REACHABLE {
		err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device :%s is in operstatus:%s ,connect-status:%s", agent.deviceID, parentDevice.OperStatus, parentDevice.ConnectStatus)
		return err
	}

	logger.Debugw(ctx, "disable-child-serial-number", log.Fields{"device-id": agent.deviceID, "serial-number": agent.device.SerialNumber})

	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": agent.adapterEndpoint,
			})

		desc = fmt.Sprintf("failed to get adapter client for endpoint %s", agent.adapterEndpoint)
		return err
	}

	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
	defer cancel()
	requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS

	resultCh := make(chan error, 1)

	go func() {
		_, callErr := client.DisableOnuSerialNumber(subCtx, device)
		if callErr == nil {
			logger.Infow(subCtx, "disable-child-serial-number-success", log.Fields{"serial-number": device.SerialNumber})
			agent.onSuccess(subCtx, nil, nil, true)
			resultCh <- nil
		} else {
			logger.Errorw(subCtx, "disable-child-serial-number-failed", log.Fields{"serial-number": device.SerialNumber, "error": callErr})
			agent.onFailure(subCtx, callErr, nil, nil, true)
			trimmedErr := extractLastRpcError(callErr)
			desc = fmt.Sprintf("disableOnuSerialNumber call failed: %s", trimmedErr)
			resultCh <- fmt.Errorf("disable-child-serial-number-failed: %s", trimmedErr)
		}
	}()

	select {
	case resultErr := <-resultCh:
		if resultErr != nil {
			return resultErr
		}

	case <-ctx.Done():
		err = fmt.Errorf("operation cancelled or timed out: %w", ctx.Err())
		desc = "disable ONU serial number operation cancelled or timed out"
		return err
	}

	return nil

}

func (agent *Agent) enableOnuSerialNumber(ctx context.Context, device *voltha.OnuSerialNumberOnOLTPon) error {
	var err error
	var desc string
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}

	defer func() {
		agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc)
	}()

	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		desc = "Failed while waiting for green light"
		return err
	}

	defer agent.requestQueue.RequestComplete()

	logger.Debugw(ctx, "enable-child-serial-number", log.Fields{"serial-number": device.SerialNumber})

	parentDevice := agent.getDeviceReadOnlyWithoutLock()

	if parentDevice.ConnectStatus != voltha.ConnectStatus_REACHABLE {
		err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device :%s is in operstatus:%s ,connect-status:%s", agent.deviceID, parentDevice.OperStatus, parentDevice.ConnectStatus)
		return err
	}

	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": agent.adapterEndpoint,
			})

		desc = fmt.Sprintf("failed to get adapter client for endpoint %s", agent.adapterEndpoint)
		return err
	}

	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
	defer cancel()
	requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS

	resultCh := make(chan error, 1)

	go func() {
		_, callErr := client.EnableOnuSerialNumber(subCtx, device)
		if callErr == nil {
			logger.Infow(subCtx, "enable-child-serial-number-success", log.Fields{"serial number": device.SerialNumber})
			agent.onSuccess(subCtx, nil, nil, true)
			resultCh <- nil
		} else {
			logger.Errorw(subCtx, "enable-child-serial-number-failed", log.Fields{"serial number": device.SerialNumber, "error": callErr})
			agent.onFailure(subCtx, callErr, nil, nil, true)
			trimmedErr := extractLastRpcError(callErr)
			desc = fmt.Sprintf("enableOnuSerialNumber call failed: %s", trimmedErr)
			resultCh <- fmt.Errorf("enable-child-serial-number-failed: %s", trimmedErr)
		}
	}()

	select {
	case resultErr := <-resultCh:
		if resultErr != nil {
			return resultErr
		}

	case <-ctx.Done():
		err = fmt.Errorf("operation cancelled or timed out: %w", ctx.Err())
		desc = "enable ONU serial number operation cancelled or timed out"
		return err
	}

	return nil
}

func (agent *Agent) checkDisableEligibility() (*voltha.Device, string, error) {
	cloned := agent.cloneDeviceWithoutLock()

	if !agent.proceedWithRequest(cloned) {
		err := status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress/failed: %s", agent.deviceID)
		return nil, "device-deletion-in-progress", err
	}

	if cloned.AdminState == voltha.AdminState_DISABLED {
		err := status.Errorf(codes.FailedPrecondition, "device-already-disabled: %s", agent.deviceID)
		return nil, "device-already-disabled", err
	}
	if cloned.AdminState == voltha.AdminState_PREPROVISIONED {
		err := status.Errorf(codes.FailedPrecondition, "deviceId:%s, invalid-admin-state:%s", agent.deviceID, cloned.AdminState)
		return nil, "invalid-admin-state", err
	}

	return cloned, "", nil
}

func (agent *Agent) checkEnableEligibility() (*voltha.Device, string, error) {
	device := agent.getDeviceReadOnlyWithoutLock()

	if !agent.proceedWithRequest(device) {
		err := status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress or reconciling is in progress/failed: %s", agent.deviceID)
		return nil, "device-deletion-in-progress", err
	}

	if device.AdminState == voltha.AdminState_ENABLED && device.OperStatus != voltha.OperStatus_FAILED {
		err := status.Errorf(codes.FailedPrecondition, "device-already-enabled: %s", device.Id)
		return nil, "device-already-enabled", err
	}

	cloned := agent.cloneDeviceWithoutLock()
	return cloned, "", nil
}

// extracting only the error message from the rpc error
func extractLastRpcError(err error) string {
	s := err.Error()
	if i := strings.LastIndex(s, "rpc error:"); i >= 0 {
		return strings.TrimSpace(s[i:])
	}
	return s
}

// UpdateDevice updates the configuration of a device, such as changing the IP address of an OLT device.
func (agent *Agent) updateDevice(ctx context.Context, config *voltha.UpdateDevice) error {
	var desc string
	var err error
	requestStatus := &common.OperationResp{Code: common.OperationResp_OPERATION_FAILURE}
	defer func() { agent.logDeviceUpdate(ctx, nil, nil, requestStatus, err, desc) }()

	clonedDevice := agent.cloneDeviceWithoutLock()
	if err = agent.requestQueue.WaitForGreenLight(ctx); err != nil {
		return err
	}

	agent.UpdateAddress(ctx, clonedDevice, config)

	logger.Infow(ctx, "update-device-ip-address", log.Fields{
		"device-id": agent.deviceID,
		"config":    config,
	})

	if !agent.proceedWithRequest(clonedDevice) && clonedDevice.OperStatus != voltha.OperStatus_RECONCILING {
		agent.requestQueue.RequestComplete()
		err = status.Errorf(codes.FailedPrecondition, "cannot complete operation as device deletion is in progress/failed: %s", agent.deviceID)
		return err
	}

	client, err := agent.adapterMgr.GetAdapterClient(ctx, agent.adapterEndpoint)
	if err != nil {
		agent.requestQueue.RequestComplete()
		logger.Errorw(ctx, "grpc-client-nil",
			log.Fields{
				"error":            err,
				"device-id":        agent.deviceID,
				"device-type":      agent.deviceType,
				"adapter-endpoint": clonedDevice.AdapterEndpoint,
			})
		return err
	}

	subCtx, cancel := context.WithTimeout(coreutils.WithAllMetadataFromContext(ctx), agent.rpcTimeout)
	defer cancel()
	requestStatus.Code = common.OperationResp_OPERATION_IN_PROGRESS
	logger.Infow(ctx, "sending-update-to-adapter", log.Fields{"device-id": agent.deviceID, "adapter-endpoint": clonedDevice.AdapterEndpoint, "config": config})
	if _, err = client.UpdateDevice(subCtx, config); err == nil {
		agent.onSuccess(subCtx, nil, nil, true)
	} else {
		logger.Errorw(ctx, "update-device-ip-failed", log.Fields{"device-id": agent.deviceID, "error": err})
		agent.onFailure(subCtx, err, nil, nil, true)
		agent.requestQueue.RequestComplete()
		return err
	}
	return agent.updateDeviceAndReleaseLock(ctx, clonedDevice)
}

func (agent *Agent) UpdateAddress(ctx context.Context, device *voltha.Device, deviceConfig *voltha.UpdateDevice) {
	switch addr := deviceConfig.Address.(type) {
	case *voltha.UpdateDevice_Ipv4Address:
		device.Address = &voltha.Device_Ipv4Address{Ipv4Address: addr.Ipv4Address}
	case *voltha.UpdateDevice_Ipv6Address:
		device.Address = &voltha.Device_Ipv6Address{Ipv6Address: addr.Ipv6Address}
	case *voltha.UpdateDevice_HostAndPort:
		device.Address = &voltha.Device_HostAndPort{HostAndPort: addr.HostAndPort}
	}
}
