Blame - tests/dt-workflow/Voltha_DT_FailureScenarios_VGC.robot - voltha-system-tests

${cmd} Set Variable ps -ef | grep -E "[k]ubectl.*-n[[:space:]]*voltha.*port-forward.*voltha-voltha-api.*55555:55555" | awk '{print $2}' | xargs -r kill -9

289

${rc} ${pid} Run And Return Rc And Output ${cmd}

290

${port_fwd} Start Process kubectl -n voltha port-forward svc/${kafka} ${KAFKA_PORT}:${KAFKA_PORT} & shell=true

Akash Soni

35cafc3

2023-12-20 12:48:08 +0530

[diff] [blame]

291

292

Verify OLT Soft Reboot for DT

293

[Documentation] Test soft reboot of the OLT using voltctl command

294

[Tags] VOL-2818 OLTSoftRebootDt functionalDt raj

295

[Setup] Start Logging OLTSoftRebootDt

296

[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs

297

... AND Stop Logging OLTSoftRebootDt

298

FOR ${I} IN RANGE 0 ${olt_count}

299

${olt_serial_number}= Get From Dictionary ${olt_ids}[${I}] sn

300

${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}

301

Run Keyword And Continue On Failure Wait Until Keyword Succeeds 360s 5s

302

... Validate OLT Device ENABLED ACTIVE

303

... REACHABLE ${olt_serial_number}

304

# Reboot the OLT using "voltctl device reboot" command

Guru Prasanna

0ae8e69

2025-05-02 15:03:40 +0530

[diff] [blame]

305

Wait Until Keyword Succeeds 360s 5s Reboot Device ${olt_device_id}

Akash Soni

35cafc3

2023-12-20 12:48:08 +0530

[diff] [blame]

306

# Wait for the OLT to actually go down

307

Wait Until Keyword Succeeds 360s 5s Validate OLT Device ENABLED UNKNOWN UNREACHABLE

308

... ${olt_serial_number}

309

END

310

#Verify that ping fails

311

FOR ${I} IN RANGE 0 ${num_all_onus}

312

${src}= Set Variable ${hosts.src[${I}]}

313

${dst}= Set Variable ${hosts.dst[${I}]}

314

Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

315

... Wait Until Keyword Succeeds ${timeout} 2s

316

... Check Ping False ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}

317

... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}

318

END

319

# Check OLT states

320

FOR ${I} IN RANGE 0 ${olt_count}

321

${olt_serial_number}= Get From Dictionary ${list_olts}[${I}] sn

322

${olt_ssh_ip}= Get From Dictionary ${list_olts}[${I}] sship

323

${olt_device_id}= Get OLTDeviceID From OLT List ${olt_serial_number}

324

# Wait for the OLT to come back up

325

Run Keyword If ${has_dataplane} Wait Until Keyword Succeeds 120s 10s

326

... Check Remote System Reachability True ${olt_ssh_ip}

327

# Check OLT states

328

Wait Until Keyword Succeeds 360s 5s

329

... Validate OLT Device ENABLED ACTIVE

330

... REACHABLE ${olt_serial_number}

331

END

332

# Waiting extra time for the ONUs to come up

333

Sleep 60s

334

#Check after reboot that ONUs are active, DHCP/pingable

335

Run Keyword If ${has_dataplane} Clean Up Linux

336

Perform Sanity Test DT

337

338

Verify restart openonu-adapter container for DT

339

[Documentation] Restart openonu-adapter container after VOLTHA is operational.

340

... Run the ping continuously in background during container restart,

341

... and verify that there should be no affect on the dataplane.

342

... Also, verify that the voltha control plane functionality is not affected.

343

[Tags] functionalDt RestartOpenOnuPingDt raj

344

[Setup] Start Logging RestartOpenOnuPingDt

345

[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs

346

... AND Stop Logging RestartOpenOnuPingDt

347

Clear All Devices Then Create New Device

348

# Performing Sanity Test to make sure subscribers are all DHCP and pingable

349

Run Keyword If ${has_dataplane} Clean Up Linux

350

Perform Sanity Test DT

351

FOR ${I} IN RANGE 0 ${num_all_onus}

352

${src}= Set Variable ${hosts.src[${I}]}

353

${dst}= Set Variable ${hosts.dst[${I}]}

354

${ping_output_file}= Set Variable /tmp/${src['onu']}_ping

355

Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

356

... Wait Until Keyword Succeeds ${timeout} 2s

357

... Run Ping In Background ${ping_output_file} ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}

358

... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}

359

END

360

${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}

361

Log ${podStatusOutput}

362

${countBeforeRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l

363

${podName} Set Variable adapter-open-onu

364

Wait Until Keyword Succeeds ${timeout} 15s Delete K8s Pods By Label ${NAMESPACE} app ${podName}

365

Wait Until Keyword Succeeds ${timeout} 2s Validate Pods Status By Label ${NAMESPACE}

366

... app ${podName} Running

367

# Wait for 1 min after openonu adapter is restarted

368

${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}

369

Log ${podStatusOutput}

370

${countAfterRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l

371

Should Be Equal As Strings ${countAfterRestart} ${countBeforeRestart}

372

FOR ${I} IN RANGE 0 ${num_all_onus}

373

${src}= Set Variable ${hosts.src[${I}]}

374

${dst}= Set Variable ${hosts.dst[${I}]}

375

Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

376

... Wait Until Keyword Succeeds ${timeout} 2s

377

... Stop Ping Running In Background ${src['ip']} ${src['user']} ${src['pass']}

378

... ${src['container_type']} ${src['container_name']}

379

END

380

FOR ${I} IN RANGE 0 ${num_all_onus}

381

${src}= Set Variable ${hosts.src[${I}]}

382

${dst}= Set Variable ${hosts.dst[${I}]}

383

${ping_output_file}= Set Variable /tmp/${src['onu']}_ping

384

${ping_output}= Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

385

... Wait Until Keyword Succeeds ${timeout} 2s

386

... Retrieve Remote File Contents ${ping_output_file} ${src['ip']} ${src['user']} ${src['pass']}

387

... ${src['container_type']} ${src['container_name']}

388

Run Keyword If ${has_dataplane} Check Ping Result True ${ping_output}

389

END

390

# Verify Control Plane Functionality by Deleting and Re-adding the Subscriber

rbodapat

fe5bb15

2025-03-17 11:34:42 +0530

[diff] [blame]

391

# "Once the onu adapter is restarted, it takes a bit of time for the OLT's/ONUs to reconcile, if the OLT is deleted

392

# before the ONUs are reconiled successfully there would be stale entries. This scenario is not handled in VOLTHA as

393

# of now. And there is no other to check if the reconcile has happened for all the ONUs. Due to this limitations a

394

# sleep of 60s is introduced to give enough time for onu adapter to reconcile the ONUs."

395

Sleep 60s

Akash Soni

35cafc3

2023-12-20 12:48:08 +0530

[diff] [blame]

396

Verify Control Plane After Pod Restart DT

397

398

Verify restart openolt-adapter container for DT

399

[Documentation] Restart openolt-adapter container after VOLTHA is operational.

400

... Run the ping continuously in background during container restart,

401

... and verify that there should be no affect on the dataplane.

402

... Also, verify that the voltha control plane functionality is not affected.

403

[Tags] functionalDt RestartOpenOltPingDt raj

404

[Setup] Start Logging RestartOpenOltPingDt

405

[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs

406

... AND Stop Logging RestartOpenOltPingDt

407

Clear All Devices Then Create New Device

408

# Performing Sanity Test to make sure subscribers are all DHCP and pingable

409

Run Keyword If ${has_dataplane} Clean Up Linux

410

Perform Sanity Test DT

411

FOR ${I} IN RANGE 0 ${num_all_onus}

412

${src}= Set Variable ${hosts.src[${I}]}

413

${dst}= Set Variable ${hosts.dst[${I}]}

414

${ping_output_file}= Set Variable /tmp/${src['onu']}_ping

415

Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

416

... Wait Until Keyword Succeeds ${timeout} 2s

417

... Run Ping In Background ${ping_output_file} ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}

418

... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}

419

END

420

${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}

421

Log ${podStatusOutput}

422

${countBeforeRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l

423

${podName} Set Variable ${OLT_ADAPTER_APP_LABEL}

424

Wait Until Keyword Succeeds ${timeout} 15s Delete K8s Pods By Label ${NAMESPACE} app ${podName}

425

Wait Until Keyword Succeeds ${timeout} 2s Validate Pods Status By Label ${NAMESPACE}

426

... app ${podName} Running

427

# Wait for 1 min after openolt adapter is restarted

428

Sleep 60s

429

${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}

430

Log ${podStatusOutput}

431

${countAfterRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l

432

Should Be Equal As Strings ${countAfterRestart} ${countBeforeRestart}

433

FOR ${I} IN RANGE 0 ${num_all_onus}

434

${src}= Set Variable ${hosts.src[${I}]}

435

${dst}= Set Variable ${hosts.dst[${I}]}

436

Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

437

... Wait Until Keyword Succeeds ${timeout} 2s

438

... Stop Ping Running In Background ${src['ip']} ${src['user']} ${src['pass']}

439

... ${src['container_type']} ${src['container_name']}

440

END

441

FOR ${I} IN RANGE 0 ${num_all_onus}

442

${src}= Set Variable ${hosts.src[${I}]}

443

${dst}= Set Variable ${hosts.dst[${I}]}

444

${ping_output_file}= Set Variable /tmp/${src['onu']}_ping

445

${ping_output}= Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

446

... Wait Until Keyword Succeeds ${timeout} 2s

447

... Retrieve Remote File Contents ${ping_output_file} ${src['ip']} ${src['user']} ${src['pass']}

448

... ${src['container_type']} ${src['container_name']}

449

Run Keyword If ${has_dataplane} Check Ping Result True ${ping_output}

450

END

451

# Verify Control Plane Functionality by Deleting and Re-adding the Subscriber

rbodapat

fe5bb15

2025-03-17 11:34:42 +0530

[diff] [blame]

452

# "Once the olt adapter is restarted, it takes a bit of time for the OLT's/ONUs to reconcile, if try to delete OLT

453

# before the OLT's are reconiled successfully there would be recocile error. This scenario is not handled in VOLTHA as

454

# of now. And there is no other to check if the reconcile has happened for all the OLTs. Due to this limitations a

455

# sleep of 60s is introduced to give enough time for OLT adapter to reconcile the OLTs."

456

Sleep 60s

Akash Soni

35cafc3

2023-12-20 12:48:08 +0530

[diff] [blame]

457

Verify Control Plane After Pod Restart DT

458

459

Verify restart rw-core container for DT

460

[Documentation] Restart rw-core container after VOLTHA is operational.

461

... Run the ping continuously in background during container restart,

462

... and verify that there should be no affect on the dataplane.

463

... Also, verify that the voltha control plane functionality is not affected.

464

[Tags] functionalDt RestartRwCorePingDt raj

465

[Setup] Start Logging RestartRwCorePingDt

466

[Teardown] Run Keywords Run Keyword If ${logging} Collect Logs

467

... AND Stop Logging RestartRwCorePingDt

468

Clear All Devices Then Create New Device

469

# Performing Sanity Test to make sure subscribers are all DHCP and pingable

470

Run Keyword If ${has_dataplane} Clean Up Linux

471

Perform Sanity Test DT

472

FOR ${I} IN RANGE 0 ${num_all_onus}

473

${src}= Set Variable ${hosts.src[${I}]}

474

${dst}= Set Variable ${hosts.dst[${I}]}

475

${ping_output_file}= Set Variable /tmp/${src['onu']}_ping

476

Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

477

... Wait Until Keyword Succeeds ${timeout} 2s

478

... Run Ping In Background ${ping_output_file} ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}

479

... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}

480

END

481

${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}

482

Log ${podStatusOutput}

483

${countBeforeRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l

484

${podName} Set Variable rw-core

485

Wait Until Keyword Succeeds ${timeout} 15s Delete K8s Pods By Label ${NAMESPACE} app ${podName}

486

Wait Until Keyword Succeeds ${timeout} 2s Validate Pods Status By Label ${NAMESPACE}

487

... app ${podName} Running

488

# Wait for 1 min after rw-core is restarted

489

Sleep 60s

490

# For some reason scaling down and up the POD behind a service causes the port forward to stop working,

491

# so restart the port forwarding for the API service

492

Restart VOLTHA Port Forward voltha-api

493

${podStatusOutput}= Run kubectl get pods -n ${NAMESPACE}

494

Log ${podStatusOutput}

495

${countAfterRestart}= Run kubectl get pods -n ${NAMESPACE} | grep Running | wc -l

496

Should Be Equal As Strings ${countAfterRestart} ${countBeforeRestart}

497

FOR ${I} IN RANGE 0 ${num_all_onus}

498

${src}= Set Variable ${hosts.src[${I}]}

499

${dst}= Set Variable ${hosts.dst[${I}]}

500

Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

501

... Wait Until Keyword Succeeds ${timeout} 2s

502

... Stop Ping Running In Background ${src['ip']} ${src['user']} ${src['pass']}

503

... ${src['container_type']} ${src['container_name']}

504

END

505

FOR ${I} IN RANGE 0 ${num_all_onus}

506

${src}= Set Variable ${hosts.src[${I}]}

507

${dst}= Set Variable ${hosts.dst[${I}]}

508

${ping_output_file}= Set Variable /tmp/${src['onu']}_ping

509

${ping_output}= Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

510

... Wait Until Keyword Succeeds ${timeout} 2s

511

... Retrieve Remote File Contents ${ping_output_file} ${src['ip']} ${src['user']} ${src['pass']}

512

... ${src['container_type']} ${src['container_name']}

513

Run Keyword If ${has_dataplane} Check Ping Result True ${ping_output}

514

END

515

# Verify Control Plane Functionality by Deleting and Re-adding the Subscriber

rbodapat

fe5bb15

2025-03-17 11:34:42 +0530

[diff] [blame]

516

# "Once the rw core is restarted, it takes a bit of time for the OLT's/ONUs to reconcile, if try to delete OLT

517

# before the OLT's are reconiled successfully there would be recocile error. This scenario is not handled in VOLTHA as

518

# of now. And there is no other to check if the reconcile has happened for all the OLTs. Due to this limitations a

519

# sleep of 60s is introduced to give enough time for rw core to reconcile the OLTs."

520

Sleep 60s

Abhay Kumar

107b03b

2026-01-30 07:20:08 +0000

[diff] [blame]

521

${cmd} Set Variable ps -ef | grep -E "[k]ubectl.*-n[[:space:]]*voltha.*port-forward.*voltha-voltha-api.*55555:55555" | awk '{print $2}' | xargs -r kill -9

522

${rc} ${pid} Run And Return Rc And Output ${cmd}

523

${port_fwd} Start Process kubectl -n voltha port-forward svc/${kafka} ${KAFKA_PORT}:${KAFKA_PORT} & shell=true

Akash Soni

35cafc3

2023-12-20 12:48:08 +0530

[diff] [blame]

524

Verify Control Plane After Pod Restart DT

*** Keywords ***

Setup Suite

[Documentation] Set up the test suite

529

Common Test Suite Setup

530

#power_switch.robot needs it to support different vendor's power switch

531

${switch_type}= Get Variable Value ${web_power_switch.type}

532

Run Keyword If "${switch_type}"!="" Set Global Variable ${powerswitch_type} ${switch_type}

533

# Run Pre-test Setup for Soak Job

534

# Note: As soak requirement, it expects that the devices under test are already created and enabled

535

Run Keyword If '${SOAK_TEST}'=='True' Setup Soak

536

537

538

Clear All Devices Then Create New Device

539

[Documentation] Remove any devices from VOLTHA and ONOS

540

# Remove all devices from voltha and nos

541

Delete All Devices and Verify

542

# Execute normal test Setup Keyword

543

Setup

544

545

Verify Control Plane After Pod Restart DT

546

[Documentation] Verifies the control plane functionality after the voltha pod restart

547

... by deleting and re-adding the subscriber

548

FOR ${I} IN RANGE 0 ${num_all_onus}

549

${src}= Set Variable ${hosts.src[${I}]}

550

${dst}= Set Variable ${hosts.dst[${I}]}

551

${of_id}= Get ofID From OLT List ${src['olt']}

552

${nni_port}= Wait Until Keyword Succeeds ${timeout} 2s Get NNI Port in VGC ${of_id}

553

${onu_port}= Wait Until Keyword Succeeds ${timeout} 2s

554

... Get ONU Port in VGC ${src['onu']} ${of_id} ${src['uni_id']}

555

${onu_device_id}= Wait Until Keyword Succeeds ${timeout} 2s Get Device ID From SN ${src['onu']}

556

# Remove Subscriber Access

557

Remove Subscriber Access ${of_id} ${onu_port}

558

Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

559

... Wait Until Keyword Succeeds ${timeout} 2s

560

... Check Ping False ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}

561

... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}

562

# Disable and Re-Enable the ONU (To replicate DT current workflow)

563

# TODO: Delete and Auto-Discovery Add of ONU (not yet supported)

564

Disable Device ${onu_device_id}

565

Wait Until Keyword Succeeds ${timeout} 5s

566

... Validate Device DISABLED UNKNOWN

567

... REACHABLE ${src['onu']}

568

Enable Device ${onu_device_id}

569

Wait Until Keyword Succeeds ${timeout} 5s

570

... Validate Device ENABLED ACTIVE

571

... REACHABLE ${src['onu']}

572

# Add Subscriber Access

573

Add Subscriber Details ${of_id} ${onu_port}

574

# Verify subscriber access flows are added for the ONU port

575

Run Keyword And Continue On Failure Wait Until Keyword Succeeds ${timeout} 5s

576

... Verify Subscriber Access Flows Added for ONU DT in VGC ${VGC_SSH_IP} ${VGC_SSH_PORT} ${of_id}

577

... ${onu_port} ${nni_port} ${src['s_tag']}

578

Wait Until Keyword Succeeds ${timeout} 5s

579

... Validate Device ENABLED ACTIVE

580

... REACHABLE ${src['onu']} onu=True onu_reason=omci-flows-pushed

581

# Workaround for issue seen in VOL-4489. Keep this workaround until VOL-4489 is fixed.

582

Run Keyword If ${has_dataplane} Reboot XGSPON ONU ${src['olt']} ${src['onu']} omci-flows-pushed

583

# Workaround ends here for issue seen in VOL-4489.

584

Run Keyword If ${has_dataplane} Run Keyword And Continue On Failure

585

... Wait Until Keyword Succeeds ${timeout} 2s

586

... Check Ping True ${dst['dp_iface_ip_qinq']} ${src['dp_iface_name']}

587

... ${src['ip']} ${src['user']} ${src['pass']} ${src['container_type']} ${src['container_name']}

588

END