---
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: 2026 Open Networking Foundation Contributors

name: "bbsim-tests"
description: "VOLTHA BBSim E2E Tests - Simulates OLT/ONUs using bbsim and runs robot framework tests"

inputs:
  branch:
    description: "Branch to test (master, voltha-2.15, etc.)"
    required: true
    type: string
  test-targets:
    description: "YAML string defining test targets to run"
    required: true
    type: string
  gerrit-project:
    description: "Gerrit project name if building a patch"
    required: false
    type: string
    default: ""
  gerrit-refspec:
    description: "Gerrit refspec if building a patch"
    required: false
    type: string
    default: ""
  voltha-system-tests-change:
    description: "Gerrit change number for voltha-system-tests"
    required: false
    type: string
    default: ""
  voltha-helm-charts-change:
    description: "Gerrit change number for voltha-helm-charts"
    required: false
    type: string
    default: ""
  extra-helm-flags:
    description: "Additional Helm flags for deployment"
    required: false
    type: string
    default: ""
  log-level:
    description: "Log level for VOLTHA components (DEBUG, INFO, WARN, ERROR)"
    required: false
    type: string
    default: "WARN"
  timeout:
    description: "Timeout in minutes for the entire action"
    required: false
    type: string
    default: "240"
  cluster-name:
    description: "Name of the kind cluster"
    required: false
    type: string
    default: "kind-ci"
  docker-registry:
    description: "Docker registry to use"
    required: false
    type: string
    default: "linuxfoundation.jfrog.io/voltha-docker"
  olts:
    description: "Number of OLTs to simulate"
    required: false
    type: string
    default: "1"
  with-monitoring:
    description: "Enable monitoring with prometheus"
    required: false
    type: boolean
    default: false
  enable-mac-learning:
    description: "Enable MAC learning in VOLTHA"
    required: false
    type: boolean
    default: false
  extra-robot-args:
    description: "Additional arguments for Robot Framework"
    required: false
    type: string
    default: ""

outputs:
  test-results:
    description: "Path to test results"
    value: ${{ steps.test-execution.outputs.results-path }}

runs:
  using: "composite"
  steps:
    # -----------------------------------------------------------------------
    # Setup environment variables
    # -----------------------------------------------------------------------
    - name: Setup environment
      shell: bash
      run: |
        echo "KUBECONFIG=$HOME/.kube/kind-${{ inputs.cluster-name }}" >> $GITHUB_ENV
        echo "VOLTCONFIG=$HOME/.volt/config" >> $GITHUB_ENV
        echo "PATH=$PATH:$GITHUB_WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> $GITHUB_ENV
        echo "DIAGS_PROFILE=VOLTHA_PROFILE" >> $GITHUB_ENV
        echo "SSHPASS=karaf" >> $GITHUB_ENV
        mkdir -p $GITHUB_WORKSPACE/bin

    # -----------------------------------------------------------------------
    # Install dependencies
    # -----------------------------------------------------------------------
    - name: Install system dependencies
      shell: bash
      run: |
        echo "Installing system dependencies..."
        sudo apt-get update
        sudo apt-get install -y \
          curl \
          wget \
          git \
          make \
          jq \
          sshpass \
          python3 \
          python3-pip \
          python3-venv \
          rsync

    # -----------------------------------------------------------------------
    # Install kubectl
    # -----------------------------------------------------------------------
    - name: Install kubectl
      shell: bash
      run: |
        echo "Installing kubectl..."
        if [ ! -f "$GITHUB_WORKSPACE/bin/kubectl" ]; then
          KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt)
          curl -Lo "$GITHUB_WORKSPACE/bin/kubectl" \
            "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
          chmod +x "$GITHUB_WORKSPACE/bin/kubectl"
        fi
        kubectl version --client

    # -----------------------------------------------------------------------
    # Install Helm
    # -----------------------------------------------------------------------
    - name: Install Helm
      shell: bash
      run: |
        echo "Installing Helm..."
        if ! command -v helm &> /dev/null; then
          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
        fi
        helm version

    # -----------------------------------------------------------------------
    # Install kind
    # -----------------------------------------------------------------------
    - name: Install kind
      shell: bash
      run: |
        echo "Installing kind..."
        if [ ! -f "$GITHUB_WORKSPACE/bin/kind" ]; then
          KIND_VERSION="v0.20.0"
          curl -Lo "$GITHUB_WORKSPACE/bin/kind" \
            "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
          chmod +x "$GITHUB_WORKSPACE/bin/kind"
        fi
        kind version

    # -----------------------------------------------------------------------
    # Install kail
    # -----------------------------------------------------------------------
    - name: Install kail
      shell: bash
      run: |
        echo "Installing kail..."
        if [ ! -f "$GITHUB_WORKSPACE/bin/kail" ]; then
          KAIL_VERSION="v0.17.4"
          wget -O /tmp/kail.tar.gz \
            "https://github.com/boz/kail/releases/download/${KAIL_VERSION}/kail_${KAIL_VERSION}_linux_amd64.tar.gz"
          tar -xzf /tmp/kail.tar.gz -C "$GITHUB_WORKSPACE/bin" kail
          chmod +x "$GITHUB_WORKSPACE/bin/kail"
          rm /tmp/kail.tar.gz
        fi

    # -----------------------------------------------------------------------
    # Install voltctl
    # -----------------------------------------------------------------------
    - name: Install voltctl
      shell: bash
      run: |
        echo "Installing voltctl..."
        if [ "${{ inputs.gerrit-project }}" != "voltctl" ]; then
          if [ ! -f "$GITHUB_WORKSPACE/bin/voltctl" ]; then
            VOLTCTL_VERSION="1.8.45"
            curl -Lo "$GITHUB_WORKSPACE/bin/voltctl" \
              "https://github.com/opencord/voltctl/releases/download/v${VOLTCTL_VERSION}/voltctl-${VOLTCTL_VERSION}-linux-amd64"
            chmod +x "$GITHUB_WORKSPACE/bin/voltctl"
          fi
        fi

    # -----------------------------------------------------------------------
    # Checkout VOLTHA repositories
    # -----------------------------------------------------------------------
    - name: Checkout voltha-system-tests
      uses: actions/checkout@v4
      with:
        repository: opencord/voltha-system-tests
        ref: ${{ inputs.branch }}
        path: voltha-system-tests

    - name: Apply voltha-system-tests patch
      if: inputs.voltha-system-tests-change != ''
      shell: bash
      working-directory: voltha-system-tests
      run: |
        echo "Applying voltha-system-tests change ${{ inputs.voltha-system-tests-change }}"
        git fetch https://gerrit.lfbroadband.org/voltha-system-tests \
          refs/changes/${{ inputs.voltha-system-tests-change }}
        git checkout FETCH_HEAD

    - name: Checkout voltha-helm-charts
      uses: actions/checkout@v4
      with:
        repository: opencord/voltha-helm-charts
        ref: ${{ inputs.branch }}
        path: voltha-helm-charts

    - name: Apply voltha-helm-charts patch
      if: inputs.voltha-helm-charts-change != ''
      shell: bash
      working-directory: voltha-helm-charts
      run: |
        echo "Applying voltha-helm-charts change ${{ inputs.voltha-helm-charts-change }}"
        git fetch https://gerrit.lfbroadband.org/voltha-helm-charts \
          refs/changes/${{ inputs.voltha-helm-charts-change }}
        git checkout FETCH_HEAD

    # -----------------------------------------------------------------------
    # Checkout and build gerrit project if specified
    # -----------------------------------------------------------------------
    - name: Checkout gerrit project
      if: inputs.gerrit-project != ''
      uses: actions/checkout@v4
      with:
        repository: opencord/${{ inputs.gerrit-project }}
        ref: ${{ inputs.gerrit-refspec }}
        path: ${{ inputs.gerrit-project }}

    - name: Build gerrit project
      if: inputs.gerrit-project != ''
      shell: bash
      working-directory: ${{ inputs.gerrit-project }}
      run: |
        echo "Building ${{ inputs.gerrit-project }}..."
        if [ -f "Makefile" ]; then
          make docker-build
        fi

    - name: Build and install voltctl from source
      if: inputs.gerrit-project == 'voltctl'
      shell: bash
      working-directory: voltctl
      run: |
        echo "Building voltctl from source..."
        make build
        cp voltctl "$GITHUB_WORKSPACE/bin/"
        chmod +x "$GITHUB_WORKSPACE/bin/voltctl"

    # -----------------------------------------------------------------------
    # Create kind cluster
    # -----------------------------------------------------------------------
    - name: Check if kind cluster exists
      id: cluster-check
      shell: bash
      run: |
        if kind get clusters | grep -q "^${{ inputs.cluster-name }}$"; then
          echo "exists=true" >> $GITHUB_OUTPUT
        else
          echo "exists=false" >> $GITHUB_OUTPUT
        fi

    - name: Create kind cluster
      if: steps.cluster-check.outputs.exists == 'false'
      shell: bash
      run: |
        echo "Creating kind cluster ${{ inputs.cluster-name }}..."
        cat <<EOF > /tmp/kind-config.yaml
        kind: Cluster
        apiVersion: kind.x-k8s.io/v1alpha4
        nodes:
          - role: control-plane
          - role: worker
          - role: worker
        EOF

        kind create cluster --name ${{ inputs.cluster-name }} --config /tmp/kind-config.yaml

        # Configure kubeconfig
        mkdir -p $HOME/.kube
        kind export kubeconfig --name ${{ inputs.cluster-name }} --kubeconfig $HOME/.kube/kind-${{ inputs.cluster-name }}

        # Wait for cluster to be ready
        kubectl wait --for=condition=Ready nodes --all --timeout=300s

    # -----------------------------------------------------------------------
    # Load images to kind if building a component
    # -----------------------------------------------------------------------
    - name: Load images to kind
      if: inputs.gerrit-project != '' && inputs.gerrit-project != 'voltctl'
      shell: bash
      run: |
        echo "Loading images to kind cluster..."
        PROJECT="${{ inputs.gerrit-project }}"
        IMAGE_NAME="${PROJECT//-/_}"

        # Find the built image
        IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep -E "voltha/${PROJECT}|opencord/${PROJECT}" | head -1)

        if [ -n "$IMAGE" ]; then
          echo "Loading image: $IMAGE"
          kind load docker-image "$IMAGE" --name ${{ inputs.cluster-name }}
        else
          echo "Warning: No image found for project $PROJECT"
        fi

    # -----------------------------------------------------------------------
    # Setup Helm repositories
    # -----------------------------------------------------------------------
    - name: Setup Helm repositories
      shell: bash
      run: |
        echo "Adding Helm repositories..."
        helm repo add onf https://charts.lfbroadband.org
        helm repo add cord https://charts.lfbroadband.org
        helm repo update

    # -----------------------------------------------------------------------
    # Run tests
    # -----------------------------------------------------------------------
    - name: Install PyYAML for test parsing
      shell: bash
      run: |
        pip3 install pyyaml

    - name: Create test execution script
      shell: bash
      run: |
        # Create the test execution script inline
        cat > /tmp/execute_test.sh <<'EXECUTE_TEST_SCRIPT'
        #!/bin/bash
        set -euo pipefail

        # Get environment variables
        TEST_TARGET="${TEST_TARGET:-functional-single-kind-dt}"
        WORKFLOW="${WORKFLOW:-dt}"
        TEST_FLAGS="${TEST_FLAGS:-}"
        TEARDOWN="${TEARDOWN:-true}"
        TEST_LOGGING="${TEST_LOGGING:-True}"
        VGC_ENABLED="${VGC_ENABLED:-false}"
        INFRA_NAMESPACE="${INFRA_NAMESPACE:-default}"
        VOLTHA_NAMESPACE="${VOLTHA_NAMESPACE:-voltha}"
        LOGS_DIR="${LOGS_DIR:-$GITHUB_WORKSPACE/logs}"
        EXTRA_HELM_FLAGS="${EXTRA_HELM_FLAGS:-}"
        LOG_LEVEL="${LOG_LEVEL:-WARN}"
        DOCKER_REGISTRY="${DOCKER_REGISTRY:-linuxfoundation.jfrog.io/voltha-docker}"
        OLTS="${OLTS:-1}"
        WITH_MONITORING="${WITH_MONITORING:-false}"
        ENABLE_MAC_LEARNING="${ENABLE_MAC_LEARNING:-false}"
        EXTRA_ROBOT_ARGS="${EXTRA_ROBOT_ARGS:-}"
        BRANCH="${BRANCH:-master}"
        GERRIT_PROJECT="${GERRIT_PROJECT:-}"

        # Helper functions
        banner() {
            echo ""
            echo "** -----------------------------------------------------------------------"
            echo "** $1"
            echo "** -----------------------------------------------------------------------"
            echo ""
        }

        cleanup_port_forward() {
            echo "Cleaning up port-forward processes..."
            pkill -f "kubectl.*port-forward" || true
        }

        get_pods_info() {
            local logs_dir=$1
            mkdir -p "$logs_dir"
            kubectl get pods --all-namespaces -o wide > "$logs_dir/pods.txt" || true
            kubectl get nodes -o wide > "$logs_dir/nodes.txt" || true
            kubectl describe pods -n "$VOLTHA_NAMESPACE" > "$logs_dir/voltha-pods-describe.txt" || true
            kubectl describe pods -n "$INFRA_NAMESPACE" > "$logs_dir/infra-pods-describe.txt" || true
        }

        # Stage: Cleanup (if teardown enabled)
        if [ "$TEARDOWN" = "true" ]; then
            banner "Cleanup - Helm Teardown"
            cleanup_port_forward
            for namespace in default "$INFRA_NAMESPACE" "$VOLTHA_NAMESPACE"; do
                echo "Cleaning up Helm releases in namespace: $namespace"
                helm list -n "$namespace" -q | xargs -r helm uninstall -n "$namespace" || true
            done
            echo "Waiting for pods to terminate..."
            kubectl wait --for=delete pods --all -n "$VOLTHA_NAMESPACE" --timeout=120s || true
            kubectl wait --for=delete pods --all -n "$INFRA_NAMESPACE" --timeout=120s || true
        fi

        # Stage: Deploy Common Infrastructure
        banner "Deploying Common Infrastructure"
        if [ "$WITH_MONITORING" = "true" ]; then
            echo "Deploying monitoring stack..."
            helm install nem-monitoring onf/nem-monitoring \
                --set prometheus.alertmanager.enabled=false \
                --set prometheus.pushgateway.enabled=false \
                --set kpi_exporter.enabled=false \
                --set dashboards.xos=false \
                --set dashboards.onos=false \
                --set dashboards.aaa=false \
                --set dashboards.voltha=false \
                --wait || true
        fi

        # Stage: Deploy VOLTHA
        if [ "$TEARDOWN" = "true" ]; then
            banner "Deploying VOLTHA"
            ONOS_LOG="${LOGS_DIR}/onos-voltha-startup-combined.log"
            mkdir -p "$LOGS_DIR"
            touch "$ONOS_LOG"
            _TAG=kail-startup kail -n "${INFRA_NAMESPACE}" -n "${VOLTHA_NAMESPACE}" > "$ONOS_LOG" &
            KAIL_PID=$!

            LOCAL_CHARTS="false"
            if [ -d "$GITHUB_WORKSPACE/voltha-helm-charts" ]; then
                LOCAL_CHARTS="true"
            fi

            HELM_FLAGS="--set global.log_level=${LOG_LEVEL^^}"
            if [ "$VGC_ENABLED" != "true" ]; then
                HELM_FLAGS="$HELM_FLAGS --set onos-classic.onosSshPort=30115"
                HELM_FLAGS="$HELM_FLAGS --set onos-classic.onosApiPort=30120"
                HELM_FLAGS="$HELM_FLAGS --set onos-classic.onosOfPort=31653"
                HELM_FLAGS="$HELM_FLAGS --set onos-classic.individualOpenFlowNodePorts=true"
            fi
            if [ -n "$EXTRA_HELM_FLAGS" ]; then
                HELM_FLAGS="$HELM_FLAGS $EXTRA_HELM_FLAGS"
            fi
            if [ -n "$TEST_FLAGS" ]; then
                HELM_FLAGS="$HELM_FLAGS $TEST_FLAGS"
            fi

            if [ -n "$GERRIT_PROJECT" ] && [ "$GERRIT_PROJECT" != "voltctl" ]; then
                IMAGE_TAG="citest"
                case "$GERRIT_PROJECT" in
                    voltha-go)
                        HELM_FLAGS="$HELM_FLAGS --set voltha.image.tag=$IMAGE_TAG"
                        ;;
                    voltha-openolt-adapter)
                        HELM_FLAGS="$HELM_FLAGS --set voltha-adapter-openolt.image.tag=$IMAGE_TAG"
                        ;;
                    voltha-openonu-adapter-go)
                        HELM_FLAGS="$HELM_FLAGS --set voltha-adapter-openonu.image.tag=$IMAGE_TAG"
                        ;;
                    ofagent-go)
                        HELM_FLAGS="$HELM_FLAGS --set voltha.ofagent.image.tag=$IMAGE_TAG"
                        ;;
                    bbsim)
                        HELM_FLAGS="$HELM_FLAGS --set bbsim.image.tag=$IMAGE_TAG"
                        ;;
                esac
            fi

            echo "Deploying VOLTHA with workflow: $WORKFLOW"
            echo "Helm flags: $HELM_FLAGS"

            if [ "$LOCAL_CHARTS" = "true" ]; then
                echo "Using local charts from $GITHUB_WORKSPACE/voltha-helm-charts"
                helm dependency update "$GITHUB_WORKSPACE/voltha-helm-charts/voltha-infra" || true
                helm dependency update "$GITHUB_WORKSPACE/voltha-helm-charts/voltha-stack" || true
                helm install voltha-infra "$GITHUB_WORKSPACE/voltha-helm-charts/voltha-infra" \
                    -n "$INFRA_NAMESPACE" --create-namespace --wait --timeout 120m
                helm install voltha "$GITHUB_WORKSPACE/voltha-helm-charts/voltha-stack" \
                    -n "$VOLTHA_NAMESPACE" --create-namespace \
                    --set global.stack_name=voltha \
                    --set global.voltha_infra_name=voltha-infra \
                    --set global.voltha_infra_namespace="$INFRA_NAMESPACE" \
                    $(echo "$HELM_FLAGS") --wait --timeout 120m
            else
                echo "Using charts from Helm repository"
                helm install voltha-infra onf/voltha-infra \
                    -n "$INFRA_NAMESPACE" --create-namespace --wait --timeout 120m
                helm install voltha onf/voltha-stack \
                    -n "$VOLTHA_NAMESPACE" --create-namespace \
                    --set global.stack_name=voltha \
                    --set global.voltha_infra_name=voltha-infra \
                    --set global.voltha_infra_namespace="$INFRA_NAMESPACE" \
                    $(echo "$HELM_FLAGS") --wait --timeout 120m
            fi

            for i in $(seq 0 $((OLTS - 1))); do
                echo "Deploying bbsim${i}..."
                if [ "$LOCAL_CHARTS" = "true" ]; then
                    helm install "bbsim${i}" "$GITHUB_WORKSPACE/voltha-helm-charts/bbsim" \
                        -n "$VOLTHA_NAMESPACE" --set olt_id="${i}" --wait || true
                else
                    helm install "bbsim${i}" onf/bbsim \
                        -n "$VOLTHA_NAMESPACE" --set olt_id="${i}" --wait || true
                fi
            done

            if [ -n "${KAIL_PID:-}" ]; then
                kill "$KAIL_PID" || true
                wait "$KAIL_PID" 2>/dev/null || true
            fi
            gzip -f "$ONOS_LOG" || true

            echo "Setting up port forwarding..."
            kubectl port-forward --address 0.0.0.0 -n "$INFRA_NAMESPACE" svc/voltha-infra-kafka 9092:9092 &
            bbsim_dmi_port=50075
            for i in $(seq 0 $((OLTS - 1))); do
                kubectl port-forward --address 0.0.0.0 -n "$VOLTHA_NAMESPACE" "svc/bbsim${i}" "${bbsim_dmi_port}:50075" &
                ((bbsim_dmi_port++))
            done
            if [ "$WITH_MONITORING" = "true" ]; then
                kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80 &
            fi
            if [ "$VGC_ENABLED" = "true" ]; then
                kubectl port-forward --address 0.0.0.0 -n "$VOLTHA_NAMESPACE" svc/voltha-voltha-go-controller 8181:8181 &
            fi
            sleep 5
        fi

        # Stage: Run Tests
        banner "Running test ${TEST_TARGET} on workflow ${WORKFLOW}"
        if [ "$WITH_MONITORING" = "true" ]; then
            echo "Collecting initial memory consumption..."
            mkdir -p "$GITHUB_WORKSPACE/voltha-pods-mem-consumption-${WORKFLOW}"
            cd "$GITHUB_WORKSPACE/voltha-system-tests"
            if [ -f "requirements.txt" ]; then
                python3 -m venv .venv || true
                source .venv/bin/activate || true
                pip install -r requirements.txt || true
                if [ -f "scripts/mem_consumption.py" ]; then
                    python scripts/mem_consumption.py \
                        -o "$GITHUB_WORKSPACE/voltha-pods-mem-consumption-${WORKFLOW}" \
                        -a 0.0.0.0:31301 -n "$VOLTHA_NAMESPACE" || true
                fi
            fi
        fi

        echo "Running Robot Framework tests..."
        mkdir -p "$LOGS_DIR"
        export ROBOT_MISC_ARGS="-d ${LOGS_DIR} ${EXTRA_ROBOT_ARGS}"
        ROBOT_MISC_ARGS="${ROBOT_MISC_ARGS} -v ONOS_SSH_PORT:30115"
        ROBOT_MISC_ARGS="${ROBOT_MISC_ARGS} -v ONOS_REST_PORT:30120"
        ROBOT_MISC_ARGS="${ROBOT_MISC_ARGS} -v NAMESPACE:${VOLTHA_NAMESPACE}"
        ROBOT_MISC_ARGS="${ROBOT_MISC_ARGS} -v INFRA_NAMESPACE:${INFRA_NAMESPACE}"
        ROBOT_MISC_ARGS="${ROBOT_MISC_ARGS} -v container_log_dir:${LOGS_DIR}"
        ROBOT_MISC_ARGS="${ROBOT_MISC_ARGS} -v logging:${TEST_LOGGING}"
        export ROBOT_MISC_ARGS
        export KVSTOREPREFIX="voltha/voltha_voltha"
        cd "$GITHUB_WORKSPACE/voltha-system-tests"
        make "${TEST_TARGET}" || TEST_RESULT=$?

        get_pods_info "$LOGS_DIR"

        if [ "$WITH_MONITORING" = "true" ]; then
            echo "Collecting final memory consumption..."
            cd "$GITHUB_WORKSPACE/voltha-system-tests"
            source .venv/bin/activate || true
            if [ -f "scripts/mem_consumption.py" ]; then
                python scripts/mem_consumption.py \
                    -o "$GITHUB_WORKSPACE/voltha-pods-mem-consumption-${WORKFLOW}" \
                    -a 0.0.0.0:31301 -n "$VOLTHA_NAMESPACE" || true
            fi
        fi

        echo "Compressing logs..."
        cd "$LOGS_DIR"
        gzip *-combined.log 2>/dev/null || true
        banner "Test execution completed"
        exit ${TEST_RESULT:-0}
        EXECUTE_TEST_SCRIPT

        chmod +x /tmp/execute_test.sh

    - name: Execute tests
      id: test-execution
      shell: bash
      run: |
        echo "Executing tests..."

        # Create logs directory
        mkdir -p $GITHUB_WORKSPACE/logs
        echo "results-path=$GITHUB_WORKSPACE/logs" >> $GITHUB_OUTPUT

        # Parse test targets from YAML
        cat > /tmp/test-targets.yaml <<'EOF'
        ${{ inputs.test-targets }}
        EOF

        # Run each test target
        python3 - <<'PYTHON_SCRIPT'
        import yaml
        import subprocess
        import os
        import sys

        with open('/tmp/test-targets.yaml', 'r') as f:
            tests = yaml.safe_load(f)

        if not tests:
            print("No tests defined")
            sys.exit(0)

        workspace = os.environ['GITHUB_WORKSPACE']
        test_failures = []

        for idx, test in enumerate(tests):
            target = test.get('target', '')
            workflow = test.get('workflow', '')
            flags = test.get('flags', '')
            teardown = test.get('teardown', True)
            logging = test.get('logging', True)
            vgc_enabled = test.get('vgcEnabled', False)

            print(f"\n** -----------------------------------------------------------------------")
            print(f"** Executing test {target} on workflow {workflow}")
            print(f"** -----------------------------------------------------------------------\n")

            # Set environment variables
            env = os.environ.copy()
            env['TEST_TARGET'] = target
            env['WORKFLOW'] = workflow
            env['TEST_FLAGS'] = flags
            env['TEARDOWN'] = str(teardown).lower()
            env['TEST_LOGGING'] = 'True' if logging else 'False'
            env['VGC_ENABLED'] = 'true' if vgc_enabled else 'false'
            env['INFRA_NAMESPACE'] = 'default'
            env['VOLTHA_NAMESPACE'] = 'voltha'
            env['LOGS_DIR'] = f"{workspace}/logs/{target}"
            env['EXTRA_HELM_FLAGS'] = "${{ inputs.extra-helm-flags }}"
            env['LOG_LEVEL'] = "${{ inputs.log-level }}"
            env['DOCKER_REGISTRY'] = "${{ inputs.docker-registry }}"
            env['OLTS'] = "${{ inputs.olts }}"
            env['WITH_MONITORING'] = "${{ inputs.with-monitoring }}"
            env['ENABLE_MAC_LEARNING'] = "${{ inputs.enable-mac-learning }}"
            env['EXTRA_ROBOT_ARGS'] = "${{ inputs.extra-robot-args }}"
            env['BRANCH'] = "${{ inputs.branch }}"
            env['GERRIT_PROJECT'] = "${{ inputs.gerrit-project }}"

            # Create logs directory for this test
            os.makedirs(env['LOGS_DIR'], exist_ok=True)

            # Execute the test using the inline script
            script_path = "/tmp/execute_test.sh"

            print(f"Running test script: {script_path}")

            try:
                result = subprocess.run(
                    ['/bin/bash', script_path],
                    env=env,
                    check=True
                )
                print(f"\nTest {target} completed successfully\n")
            except subprocess.CalledProcessError as e:
                print(f"\nTest {target} failed with exit code {e.returncode}\n")
                test_failures.append(target)
                # Continue with other tests
                continue
            except Exception as e:
                print(f"\nTest {target} failed with exception: {e}\n")
                test_failures.append(target)
                continue

        if test_failures:
            print(f"\n** Failed tests: {', '.join(test_failures)}")
            sys.exit(1)
        else:
            print("\n** All tests passed successfully")

        PYTHON_SCRIPT

    # -----------------------------------------------------------------------
    # Collect artifacts
    # -----------------------------------------------------------------------
    - name: Collect pod information
      if: always()
      shell: bash
      run: |
        echo "Collecting pod information..."
        mkdir -p $GITHUB_WORKSPACE/logs/artifacts

        kubectl get pods --all-namespaces -o wide > $GITHUB_WORKSPACE/logs/artifacts/pods.txt || true
        kubectl get nodes -o wide > $GITHUB_WORKSPACE/logs/artifacts/nodes.txt || true
        kubectl get events --all-namespaces --sort-by='.lastTimestamp' > $GITHUB_WORKSPACE/logs/artifacts/events.txt || true

    - name: Collect VOLTHA logs
      if: always()
      shell: bash
      run: |
        echo "Collecting VOLTHA logs..."
        kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha \
          > $GITHUB_WORKSPACE/logs/artifacts/voltha.log || true

    - name: Cleanup port-forward processes
      if: always()
      shell: bash
      run: |
        echo "Cleaning up port-forward processes..."
        pkill -f "kubectl.*port-forward" || true

    - name: Upload test results
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: bbsim-test-results-${{ github.run_id }}
        path: |
          ${{ github.workspace }}/logs/**/*.log
          ${{ github.workspace }}/logs/**/*.gz
          ${{ github.workspace }}/logs/**/*.txt
          ${{ github.workspace }}/logs/**/*.html
          ${{ github.workspace }}/logs/**/*.xml
          ${{ github.workspace }}/logs/**/voltha-pods-mem-consumption-*/*
        retention-days: 30
        if-no-files-found: warn

    # -----------------------------------------------------------------------
    # Cleanup
    # -----------------------------------------------------------------------
    - name: Delete kind cluster
      if: always()
      shell: bash
      run: |
        # Note: Cluster deletion is optional and can be controlled by workflow
        echo "To delete cluster manually, run: kind delete cluster --name ${{ inputs.cluster-name }}"