---
# Model Matrix for LLM Performance Tests
# Defines which LLM models run which test scenarios

matrix:
  test_suite: "llm-models"
  description: "Performance evaluation for vLLM LLM models on CPU"
  vllm_version_required: ">=0.6.0"

  # LLM model definitions
  llm_models:
    # Llama 3 Family
    - name: "llama-3.2-1b-instruct"
      full_name: "meta-llama/Llama-3.2-1B-Instruct"
      gated: true  # Requires HuggingFace token and license acceptance
      architecture_family: "Llama 3 Decoder"
      application_focus: "Prefill-Heavy (Baseline)"
      parameters: "1.2B"
      context_length: 8192
      dtype: "bfloat16"
      architecture_specs:
        hidden_size: 2048
        num_attention_heads: 32
        num_hidden_layers: 16
        num_key_value_heads: 8
      inference_characteristics:
        prefill_performance: "high"
        decode_performance: "medium"
        balance: "prefill-heavy"
      kv_cache_sizes:
        chat: "2GiB"          # 1024 tokens, 32 concurrent
        rag: "11GiB"          # 8192 tokens (7680+512), 32 concurrent
        code: "3GiB"          # 2048 tokens, 32 concurrent
        summarization: "3GiB" # 2304 tokens, 32 concurrent
        reasoning: "3GiB"     # 2304 tokens, 32 concurrent
        chat_var: "3GiB"      # 2048 max tokens, 32 concurrent, 1.5x safety
        code_var: "6GiB"      # 4096 max tokens, 32 concurrent, 1.5x safety
      default_workloads:
        - chat
      test_suites:
        - concurrent-load
        - scalability

    - name: "llama-3.2-3b-instruct"
      full_name: "meta-llama/Llama-3.2-3B-Instruct"
      gated: true  # Requires HuggingFace token and license acceptance
      architecture_family: "Llama 3 Decoder"
      application_focus: "Prefill-Heavy (Baseline)"
      parameters: "3.2B"
      context_length: 8192
      dtype: "bfloat16"
      architecture_specs:
        hidden_size: 3072
        num_attention_heads: 24
        num_hidden_layers: 28
        num_key_value_heads: 8
      inference_characteristics:
        prefill_performance: "high"
        decode_performance: "medium"
        balance: "prefill-heavy"
      kv_cache_sizes:
        chat: "5GiB"          # 1024 tokens, 32 concurrent
        rag: "36GiB"          # 8192 tokens (7680+512), 32 concurrent
        code: "9GiB"          # 2048 tokens, 32 concurrent
        summarization: "9GiB" # 2304 tokens, 32 concurrent
        reasoning: "10GiB"    # 2304 tokens, 32 concurrent
        chat_var: "11GiB"     # 2048 max tokens, 32 concurrent, 1.5x safety
        code_var: "21GiB"     # 4096 max tokens, 32 concurrent, 1.5x safety
      default_workloads:
        - chat
      test_suites:
        - concurrent-load
        - scalability

    # Llama 2 Family (Small-Scale)
    - name: "tinyllama-1.1b-chat"
      full_name: "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
      gated: false  # Public model, no token required
      architecture_family: "Llama 2 Decoder"
      application_focus: "Prefill/Decode (Small-Scale)"
      parameters: "1.1B"
      context_length: 2048
      dtype: "bfloat16"
      architecture_specs:
        hidden_size: 2048
        num_attention_heads: 32
        num_hidden_layers: 22
        num_key_value_heads: 4
      inference_characteristics:
        prefill_performance: "medium"
        decode_performance: "medium"
        balance: "balanced"
      kv_cache_sizes:
        chat: "1GiB"          # 1024 tokens, 32 concurrent
        # rag workload removed: context_length 2048 < 8704 required
        code: "2GiB"          # 2048 tokens, 32 concurrent
        summarization: "2GiB" # 2304 tokens, 32 concurrent
        reasoning: "2GiB"     # 2304 tokens, 32 concurrent
        chat_var: "3GiB"      # 2048 max tokens, 32 concurrent, 1.5x safety
        code_var: "5GiB"      # 4096 max tokens, 32 concurrent, 1.5x safety
      default_workloads:
        - chat
      test_suites:
        - concurrent-load
        - scalability

    # IBM Granite Family
    - name: "granite-3.2-2b-instruct"
      full_name: "ibm-granite/granite-3.2-2b-instruct"
      gated: false  # Public model (Apache 2.0), no token required
      architecture_family: "IBM Granite Decoder"
      application_focus: "Balanced (Enterprise Baseline)"
      parameters: "2B"
      context_length: 4096
      dtype: "bfloat16"
      architecture_specs:
        hidden_size: 2048
        num_attention_heads: 32
        num_hidden_layers: 40
        num_key_value_heads: 8
      inference_characteristics:
        prefill_performance: "medium"
        decode_performance: "medium"
        balance: "balanced"
      kv_cache_sizes:
        chat: "3GiB"          # 1024 tokens, 32 concurrent
        # rag workload skipped: context_length 4096 < 8192 required
        code: "5GiB"          # 2048 tokens, 32 concurrent
        summarization: "4GiB" # 2304 tokens, 32 concurrent
        reasoning: "5GiB"     # 2304 tokens, 32 concurrent
        chat_var: "8GiB"      # 2048 max tokens, 32 concurrent, 1.5x safety
        code_var: "15GiB"     # 4096 max tokens, 32 concurrent, 1.5x safety
      default_workloads:
        - chat
      test_suites:
        - concurrent-load
        - scalability

    # Qwen Family
    - name: "qwen3-0.6b"
      full_name: "Qwen/Qwen3-0.6B"
      gated: false  # Public model, no token required
      architecture_family: "Qwen 3 Decoder"
      application_focus: "Balanced (High-Efficiency)"
      parameters: "0.6B"
      context_length: 8192
      dtype: "bfloat16"
      architecture_specs:
        hidden_size: 1024
        num_attention_heads: 16
        num_hidden_layers: 28
        num_key_value_heads: 8
      inference_characteristics:
        prefill_performance: "medium"
        decode_performance: "high"
        balance: "balanced"
      kv_cache_sizes:
        chat: "2GiB"          # 1024 tokens, 32 concurrent
        rag: "12GiB"          # 8192 tokens (7680+512), 32 concurrent
        code: "3GiB"          # 2048 tokens, 32 concurrent
        summarization: "3GiB" # 2304 tokens, 32 concurrent
        reasoning: "4GiB"     # 2304 tokens, 32 concurrent
        chat_var: "6GiB"      # 2048 max tokens, 32 concurrent, 1.5x safety
        code_var: "11GiB"     # 4096 max tokens, 32 concurrent, 1.5x safety
      default_workloads:
        - chat
        - code
      test_suites:
        - concurrent-load
        - scalability

    - name: "qwen2.5-3b-instruct"
      full_name: "Qwen/Qwen2.5-3B-Instruct"
      gated: false  # Public model, no token required
      architecture_family: "Qwen 2.5 Decoder"
      application_focus: "Balanced (High-Efficiency)"
      parameters: "3B"
      context_length: 8192
      dtype: "bfloat16"
      architecture_specs:
        hidden_size: 2048
        num_attention_heads: 16
        num_hidden_layers: 36
        num_key_value_heads: 2
      inference_characteristics:
        prefill_performance: "medium"
        decode_performance: "high"
        balance: "balanced"
      kv_cache_sizes:
        chat: "1GiB"          # 1024 tokens, 32 concurrent
        rag: "7GiB"           # 8192 tokens (7680+512), 32 concurrent
        code: "2GiB"          # 2048 tokens, 32 concurrent
        summarization: "2GiB" # 2304 tokens, 32 concurrent
        reasoning: "2GiB"     # 2304 tokens, 32 concurrent
        chat_var: "4GiB"      # 2048 max tokens, 32 concurrent, 1.5x safety
        code_var: "7GiB"      # 4096 max tokens, 32 concurrent, 1.5x safety
      default_workloads:
        - chat
        - code
      test_suites:
        - scalability

    # OpenAI GPT-OSS Family
    - name: "gpt-oss-20b"
      full_name: "openai/gpt-oss-20b"
      gated: false  # Public model, no token required
      architecture_family: "Transformer MoE"
      application_focus: "Scalability Testing (Large-Scale)"
      parameters: "21B (3.6B active per token)"
      context_length: 128000
      dtype: "bfloat16"  # Activations and KV cache use bfloat16, MoE weights use MXFP4
      architecture_specs:
        hidden_size: 2880
        num_attention_heads: 64
        num_hidden_layers: 24
        num_key_value_heads: 8
      inference_characteristics:
        prefill_performance: "high"
        decode_performance: "high"
        balance: "balanced"
        moe_routing: "Top-4"
        active_parameters: "3.6B per token"
      architecture_details:
        type: "Mixture-of-Experts"
        layers: 24
        experts: 32
        routing: "Top-4"
        attention: "Grouped Multi-Query (group size 8)"
        activation: "SwiGLU"
        positional_encoding: "RoPE"
        quantization: "MXFP4 (MoE weights)"
      memory_footprint: "~16GB (MXFP4 quantization)"
      kv_cache_sizes:
        chat: "2GiB"          # 1024 tokens, 32 concurrent
        rag: "14GiB"          # 8192 tokens (7680+512), 32 concurrent
        code: "4GiB"          # 2048 tokens, 32 concurrent
        summarization: "3GiB" # 2304 tokens, 32 concurrent
        reasoning: "4GiB"     # 2304 tokens, 32 concurrent
        chat_var: "4GiB"      # 2048 max tokens, 32 concurrent, 1.5x safety
        code_var: "7GiB"      # 4096 max tokens, 32 concurrent, 1.5x safety
      default_workloads:
        - rag       # Primary - leverages large context
        - chat      # Baseline conversational
        - code      # Code generation baseline
      test_suites:
        - concurrent-load
        - scalability
      notes: |
        - Largest model in test suite
        - Excellent for CPU scalability testing at higher parameter counts
        - MoE architecture provides unique performance characteristics
        - 3.6B active parameters per token (efficient inference)
        - Native 128k context support enables long-context testing
        - Good candidate for tensor parallelism testing (TP=2, TP=4)

  # Workload definitions
  # NOTE: Each workload is optimized for BEST PERFORMANCE by limiting max-model-len
  # to the minimum required. This minimizes KV cache allocation and maximizes throughput.
  workloads:
    chat:
      name: "Chat"
      input_tokens: 512
      output_tokens: 512
      max_model_len: 2048
      description: "Primary working point for conversational AI"
      use_case: "General chat applications"

    rag:
      name: "RAG (Retrieval-Augmented Generation)"
      input_tokens: 7680
      output_tokens: 512
      max_model_len: 16384
      description: "ISL-heavy is fine, keep OSL short (7680+512=8192 total to fit 8K context models)"
      use_case: "Document Q&A, knowledge retrieval"

    code:
      name: "Code Generation"
      input_tokens: 1024
      output_tokens: 1024
      max_model_len: 4096
      description: "1024 OSL is near practical ceiling on CPU"
      use_case: "Code completion, generation"

    summarization:
      name: "Summarization"
      input_tokens: 2048
      output_tokens: 256
      max_model_len: 4096
      description: "Short output is realistic and achievable"
      use_case: "Document summarization"

    reasoning:
      name: "Reasoning"
      input_tokens: 256
      output_tokens: 2048
      max_model_len: 4096
      description: "New workload for reasoning tasks with longer outputs"
      use_case: "Chain-of-thought reasoning, problem solving"

    # Variable Workloads (Realistic Traffic Simulation)
    chat_var:
      name: "Chat (Variable)"
      input_tokens_mean: 512
      input_tokens_stdev: 128
      output_tokens_mean: 512
      output_tokens_stdev: 128
      input_range: "128-1024"
      output_range: "256-1024"
      variability: true
      description: "Variable chat workload with realistic token distribution"
      use_case: "Realistic conversational AI with variable message lengths"

    code_var:
      name: "Code Generation (Variable)"
      input_tokens_mean: 1024
      input_tokens_stdev: 256
      output_tokens_mean: 1024
      output_tokens_stdev: 256
      input_range: "512-2048"
      output_range: "512-2048"
      variability: true
      description: "Variable code generation with realistic output variance"
      use_case: "Realistic code generation with varying complexity"

  # Common test parameters
  common_parameters:
    affinity: "FULL"  # All physical cores
    dtype: "Model-specific (defined per model)"
    kv_cache: "Model and workload specific (defined per model)"
    quantization: false  # Full precision for baseline (except gpt-oss-20b with MXFP4 MoE weights)

  # Test suite mappings
  test_suite_configs:
    concurrent-load:
      concurrency_levels: [1, 2, 4, 8, 16, 32]
      test_tool: "guidellm"
      profile: "concurrent"
      primary_metrics:
        - "p95_e2el_ms"
        - "p99_e2el_ms"
        - "throughput_rps"
        - "ttft_ms"
      test_phases:
        phase1_baseline:
          name: "Phase 1: Baseline Tests"
          description: "Fixed tokens, no caching"
          workloads: ["chat", "rag", "code", "summarization", "reasoning"]
          caching_mode: "baseline"
        phase2_realistic:
          name: "Phase 2: Realistic Tests"
          description: "Variable tokens, no caching"
          workloads: ["chat_var", "code_var"]
          caching_mode: "baseline"
        phase3_production:
          name: "Phase 3: Production Tests"
          description: "Realistic datasets, with caching"
          workloads: ["chat", "rag"]
          caching_mode: "production"
          models: ["llama-3.2-1b-instruct", "granite-3.2-2b-instruct", "gpt-oss-20b"]
          status: "not_yet_implemented"
          note: "Pending realistic dataset selection and integration"

    scalability:
      test_types:
        - sweep
        - synchronous
        - poisson
      test_tool: "guidellm"
      primary_metrics:
        - "max_throughput"
        - "saturation_point"
        - "ttft_scaling"
        - "itl_scaling"

    resource-contention:
      status: "planned"
      test_scenarios:
        - fractional_cores
        - numa_isolation
        - noisy_neighbor

  # Results storage structure
  results_structure: |
    results/by-suite/
    ├── concurrent-load/
    │   ├── llama-3.2-1b-instruct/
    │   │   ├── chat-concurrent-8.json
    │   │   ├── chat-concurrent-16.json
    │   │   ├── rag-concurrent-8.json
    │   │   └── ...
    │   └── granite-3.2-2b-instruct/
    │       └── ...
    └── scalability/
        ├── llama-3.2-1b-instruct/
        │   ├── chat-sweep.json
        │   ├── chat-synchronous.json
        │   ├── rag-sweep.json
        │   └── ...
        └── ...