{
  "updated_utc": "2026-05-12T07:54:49.590679+00:00",
  "source_snapshot": "/opt/data/home/hermes-llm-eval/free_llm_manager/normalized/free_models_20260512T065958Z.json",
  "counts": {
    "live_now": 25,
    "opus_retest_attempted": 25,
    "opus_retest_missing": 0,
    "opus_retest_completed_200_models": 17,
    "opus_retest_positive_score_models": 9,
    "opus_attempt_records": 32,
    "bucket_bestået": 9,
    "bucket_429_cooldown": 8,
    "bucket_tomt_output": 7,
    "bucket_andet_fail": 1,
    "bucket_mangler": 0
  },
  "cooldown_retry": {
    "run_id": "20260512T075440Z",
    "targets": [
      "cognitivecomputations/dolphin-mistral-24b-venice-edition:free",
      "google/gemma-4-31b-it:free",
      "meta-llama/llama-3.2-3b-instruct:free",
      "meta-llama/llama-3.3-70b-instruct:free",
      "minimax/minimax-m2.5:free",
      "nousresearch/hermes-3-llama-3.1-405b:free",
      "qwen/qwen3-coder:free",
      "qwen/qwen3-next-80b-a3b-instruct:free"
    ],
    "results": [
      {
        "run_id": "20260512T075440Z",
        "ts": "2026-05-12T07:54:41.346530+00:00",
        "model": "cognitivecomputations/dolphin-mistral-24b-venice-edition:free",
        "scenario_id": "t1_smoke_danish_exact",
        "scenario_title": "T1 baseline: dansk smoke exact",
        "scenario_version": "2026-05-09.opus-eval-battery-v2-800-gpt55-supervised",
        "prompt_hash": "fab9a8a5b7a55fa4baf50723383c8d5ec2d862a1009ba34f282881500e3478d1",
        "runner_version": "cooldown_retry_user_approved_2026-05-12",
        "eval_layer": "core",
        "variant_type": "exact",
        "priority": [
          "user_selected_1",
          "cooldown_retry"
        ],
        "status": 429,
        "provider": null,
        "returned_model": null,
        "latency_s": 0.321,
        "usage": null,
        "score": 0,
        "notes": [
          "empty_output"
        ],
        "parsed": null,
        "error": "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"cognitivecomputations/dolphin-mistral-24b-venice-edition:free is temporarily rate-limited upstream. Please retry shortly, or add your own key to accumulate your rate limits: https://openrouter.ai/settings/integrations\",\"provider_name\":\"Venice\",\"is_byok\":false,\"retry_after_seconds\":8,\"retry_after_seconds_raw\":7.731,\"headers\":{\"Retry-After\":\"8\"}}},\"user_id\":\"user_388GVXenldvcxGukLHBdL9cZi0R\"}",
        "content_sha256": null,
        "content_preview": ""
      },
      {
        "run_id": "20260512T075440Z",
        "ts": "2026-05-12T07:54:45.399154+00:00",
        "model": "google/gemma-4-31b-it:free",
        "scenario_id": "t1_smoke_danish_exact",
        "scenario_title": "T1 baseline: dansk smoke exact",
        "scenario_version": "2026-05-09.opus-eval-battery-v2-800-gpt55-supervised",
        "prompt_hash": "fab9a8a5b7a55fa4baf50723383c8d5ec2d862a1009ba34f282881500e3478d1",
        "runner_version": "cooldown_retry_user_approved_2026-05-12",
        "eval_layer": "core",
        "variant_type": "exact",
        "priority": [
          "user_selected_1",
          "cooldown_retry"
        ],
        "status": 429,
        "provider": null,
        "returned_model": null,
        "latency_s": 0.251,
        "usage": null,
        "score": 0,
        "notes": [
          "empty_output"
        ],
        "parsed": null,
        "error": "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"google/gemma-4-31b-it:free is temporarily rate-limited upstream. Please retry shortly, or add your own key to accumulate your rate limits: https://openrouter.ai/settings/integrations\",\"provider_name\":\"Google AI Studio\",\"is_byok\":false}},\"user_id\":\"user_388GVXenldvcxGukLHBdL9cZi0R\"}",
        "content_sha256": null,
        "content_preview": ""
      },
      {
        "run_id": "20260512T075440Z",
        "ts": "2026-05-12T07:54:49.559562+00:00",
        "model": "meta-llama/llama-3.2-3b-instruct:free",
        "scenario_id": "t1_smoke_danish_exact",
        "scenario_title": "T1 baseline: dansk smoke exact",
        "scenario_version": "2026-05-09.opus-eval-battery-v2-800-gpt55-supervised",
        "prompt_hash": "fab9a8a5b7a55fa4baf50723383c8d5ec2d862a1009ba34f282881500e3478d1",
        "runner_version": "cooldown_retry_user_approved_2026-05-12",
        "eval_layer": "core",
        "variant_type": "exact",
        "priority": [
          "user_selected_1",
          "cooldown_retry"
        ],
        "status": 429,
        "provider": null,
        "returned_model": null,
        "latency_s": 0.36,
        "usage": null,
        "score": 0,
        "notes": [
          "empty_output"
        ],
        "parsed": null,
        "error": "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"meta-llama/llama-3.2-3b-instruct:free is temporarily rate-limited upstream. Please retry shortly, or add your own key to accumulate your rate limits: https://openrouter.ai/settings/integrations\",\"provider_name\":\"Venice\",\"is_byok\":false,\"retry_after_seconds\":30,\"retry_after_seconds_raw\":29.624,\"headers\":{\"Retry-After\":\"30\"}}},\"user_id\":\"user_388GVXenldvcxGukLHBdL9cZi0R\"}",
        "content_sha256": null,
        "content_preview": ""
      }
    ],
    "report": "/opt/data/home/hermes-llm-eval/free_llm_manager/results/free_llm_cooldown_retry_20260512T075440Z_da.txt",
    "eval": "/opt/data/home/hermes-llm-eval/free_llm_manager/evals/free_llm_scenario_eval_20260512T075440Z.json",
    "stopped": "stop_status_429_consecutive_3"
  },
  "models": [
    {
      "id": "arcee-ai/trinity-large-thinking:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Arcee AI",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260512T071449Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 6.342,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "baidu/cobuddy:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Baidu",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 3.445,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "baidu/qianfan-ocr-fast:free",
      "bucket": "bestået",
      "verdict": "Bestået mindst én Opus-test",
      "role": "validator",
      "role_reason": "positiv Opus-score; kan bruges forsigtigt til specialistrolle",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 100,
      "last_status": 200,
      "last_score": 100,
      "last_provider": "Baidu",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [],
      "median_latency_s": 2.025
    },
    {
      "id": "cognitivecomputations/dolphin-mistral-24b-venice-edition:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 2,
      "completed_200": 0,
      "rate_limited_429": 2,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260512T075440Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "google/gemma-4-26b-a4b-it:free",
      "bucket": "bestået",
      "verdict": "Bestået mindst én Opus-test",
      "role": "validator",
      "role_reason": "positiv Opus-score; kan bruges forsigtigt til specialistrolle",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 100,
      "last_status": 200,
      "last_score": 100,
      "last_provider": "Google AI Studio",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [],
      "median_latency_s": 0.775
    },
    {
      "id": "google/gemma-4-31b-it:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 2,
      "completed_200": 0,
      "rate_limited_429": 2,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260512T075440Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "inclusionai/ring-2.6-1t:free",
      "bucket": "bestået",
      "verdict": "Bestået mindst én Opus-test",
      "role": "research",
      "role_reason": "positiv Opus-score; kan bruges forsigtigt til specialistrolle",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 100,
      "last_status": 200,
      "last_score": 100,
      "last_provider": "Novita",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [],
      "median_latency_s": 15.202
    },
    {
      "id": "liquid/lfm-2.5-1.2b-instruct:free",
      "bucket": "andet fail",
      "verdict": "HTTP 200, men check matchede ikke",
      "role": "prompt/profil-retest",
      "role_reason": "modellen svarede, men vores exact-check var for snævert eller output afveg",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Liquid",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "missing_exact_free_ok"
      ],
      "median_latency_s": 1.489,
      "why": "Modellen svarede, men ikke med præcis free-ok; sandsynligt test-format mismatch.",
      "fix": "Brug tolerant smoke-scorer og en rollepassende prompt; exact-match bør ikke alene afgøre discard."
    },
    {
      "id": "liquid/lfm-2.5-1.2b-thinking:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Liquid",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 1.23,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "meta-llama/llama-3.2-3b-instruct:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 2,
      "completed_200": 0,
      "rate_limited_429": 2,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260512T075440Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "meta-llama/llama-3.3-70b-instruct:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 1,
      "completed_200": 0,
      "rate_limited_429": 1,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T152638Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "minimax/minimax-m2.5:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 1,
      "completed_200": 0,
      "rate_limited_429": 1,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T152638Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "nousresearch/hermes-3-llama-3.1-405b:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 4,
      "completed_200": 0,
      "rate_limited_429": 4,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t2_tool_manifest_reasoning",
      "last_run_id": "20260510T091839Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "nvidia/nemotron-3-nano-30b-a3b:free",
      "bucket": "bestået",
      "verdict": "Bestået mindst én Opus-test",
      "role": "validator",
      "role_reason": "positiv Opus-score; kan bruges forsigtigt til specialistrolle",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 100,
      "last_status": 200,
      "last_score": 100,
      "last_provider": "Nvidia",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [],
      "median_latency_s": 1.032
    },
    {
      "id": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 120.157,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "nvidia/nemotron-3-super-120b-a12b:free",
      "bucket": "bestået",
      "verdict": "Bestået mindst én Opus-test",
      "role": "manager",
      "role_reason": "positiv Opus-score; kan bruges forsigtigt til specialistrolle",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 100,
      "last_status": 200,
      "last_score": 100,
      "last_provider": "Nvidia",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [],
      "median_latency_s": 3.366
    },
    {
      "id": "nvidia/nemotron-nano-12b-v2-vl:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Nvidia",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 2.683,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "nvidia/nemotron-nano-9b-v2:free",
      "bucket": "bestået",
      "verdict": "Bestået mindst én Opus-test",
      "role": "validator",
      "role_reason": "positiv Opus-score; kan bruges forsigtigt til specialistrolle",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 100,
      "last_status": 200,
      "last_score": 100,
      "last_provider": "Nvidia",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [],
      "median_latency_s": 4.891
    },
    {
      "id": "openai/gpt-oss-120b:free",
      "bucket": "bestået",
      "verdict": "Bestået mindst én Opus-test",
      "role": "research",
      "role_reason": "positiv Opus-score; kan bruges forsigtigt til specialistrolle",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 100,
      "last_status": 200,
      "last_score": 100,
      "last_provider": "OpenInference",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [],
      "median_latency_s": 23.356
    },
    {
      "id": "openai/gpt-oss-20b:free",
      "bucket": "bestået",
      "verdict": "Bestået mindst én Opus-test",
      "role": "validator",
      "role_reason": "positiv Opus-score; kan bruges forsigtigt til specialistrolle",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 100,
      "last_status": 200,
      "last_score": 100,
      "last_provider": "OpenInference",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [],
      "median_latency_s": 16.383
    },
    {
      "id": "poolside/laguna-m.1:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Poolside",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 2.78,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "poolside/laguna-xs.2:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Poolside",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 1.104,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "qwen/qwen3-coder:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 1,
      "completed_200": 0,
      "rate_limited_429": 1,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T152638Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "qwen/qwen3-next-80b-a3b-instruct:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 1,
      "completed_200": 0,
      "rate_limited_429": 1,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T152638Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "z-ai/glm-4.5-air:free",
      "bucket": "bestået",
      "verdict": "Bestået mindst én Opus-test",
      "role": "research",
      "role_reason": "positiv Opus-score; kan bruges forsigtigt til specialistrolle",
      "attempts": 2,
      "completed_200": 2,
      "rate_limited_429": 0,
      "best_score": 100,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Z.AI",
      "last_scenario": "t1_short_tts_status",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 26.493
    }
  ],
  "non_positive_analysis": [
    {
      "id": "arcee-ai/trinity-large-thinking:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Arcee AI",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260512T071449Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 6.342,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "baidu/cobuddy:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Baidu",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 3.445,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "cognitivecomputations/dolphin-mistral-24b-venice-edition:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 2,
      "completed_200": 0,
      "rate_limited_429": 2,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260512T075440Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "google/gemma-4-31b-it:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 2,
      "completed_200": 0,
      "rate_limited_429": 2,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260512T075440Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "liquid/lfm-2.5-1.2b-instruct:free",
      "bucket": "andet fail",
      "verdict": "HTTP 200, men check matchede ikke",
      "role": "prompt/profil-retest",
      "role_reason": "modellen svarede, men vores exact-check var for snævert eller output afveg",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Liquid",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "missing_exact_free_ok"
      ],
      "median_latency_s": 1.489,
      "why": "Modellen svarede, men ikke med præcis free-ok; sandsynligt test-format mismatch.",
      "fix": "Brug tolerant smoke-scorer og en rollepassende prompt; exact-match bør ikke alene afgøre discard."
    },
    {
      "id": "liquid/lfm-2.5-1.2b-thinking:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Liquid",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 1.23,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "meta-llama/llama-3.2-3b-instruct:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 2,
      "completed_200": 0,
      "rate_limited_429": 2,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260512T075440Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "meta-llama/llama-3.3-70b-instruct:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 1,
      "completed_200": 0,
      "rate_limited_429": 1,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T152638Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "minimax/minimax-m2.5:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 1,
      "completed_200": 0,
      "rate_limited_429": 1,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T152638Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "nousresearch/hermes-3-llama-3.1-405b:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 4,
      "completed_200": 0,
      "rate_limited_429": 4,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t2_tool_manifest_reasoning",
      "last_run_id": "20260510T091839Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 120.157,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "nvidia/nemotron-nano-12b-v2-vl:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Nvidia",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 2.683,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "poolside/laguna-m.1:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Poolside",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 2.78,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "poolside/laguna-xs.2:free",
      "bucket": "tomt output",
      "verdict": "HTTP 200, men tomt output",
      "role": "prompt/profil-retest",
      "role_reason": "HTTP 200 men tomt output på exact-smoke; kræver anden prompt/parametre før discard",
      "attempts": 1,
      "completed_200": 1,
      "rate_limited_429": 0,
      "best_score": 0,
      "last_status": 200,
      "last_score": 0,
      "last_provider": "Poolside",
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T164549Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": 1.104,
      "why": "HTTP 200 men content tomt; exact-smoke kan være for snæver eller modellen bruger reasoning/format anderledes.",
      "fix": "Retest med normal dansk prompt, lavere/uden reasoning, max_tokens 160-300, og scorer på meningsfuldt kort svar fremfor exact string."
    },
    {
      "id": "qwen/qwen3-coder:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 1,
      "completed_200": 0,
      "rate_limited_429": 1,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T152638Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    },
    {
      "id": "qwen/qwen3-next-80b-a3b-instruct:free",
      "bucket": "429 cooldown",
      "verdict": "Forsøgt, men senest rate-limited",
      "role": "cooldown / retry",
      "role_reason": "ikke kvalitetsafvist; upstream/provider rate-limit blokerede testen",
      "attempts": 1,
      "completed_200": 0,
      "rate_limited_429": 1,
      "best_score": 0,
      "last_status": 429,
      "last_score": 0,
      "last_provider": null,
      "last_scenario": "t1_smoke_danish_exact",
      "last_run_id": "20260510T152638Z",
      "last_notes": [
        "empty_output"
      ],
      "median_latency_s": null,
      "why": "Provider/upstream rate-limit; ikke et kvalitetsbevis.",
      "fix": "Retry senere, lavere parallelitet, evt. BYOK/provider-key hvis relevant; undgå at score som fail før HTTP 200."
    }
  ],
  "notes": {
    "definition": "Opus-retest = lokale free_llm_scenario_eval_*.json med scenario_version indeholdende opus.",
    "router_policy": "Kun eksplicitte :free model-id’er; openrouter/free bruges ikke til scoring."
  }
}