{
  "title": "starmo Production AI Benchmark 2026",
  "version": "1.0",
  "published": "2026-08-18",
  "license": "CC BY 4.0",
  "publisher": "starmo.ai",
  "url": "https://starmo.ai/reports/production-ai-benchmark-2026",
  "note": "First-party operator-reported data from 5 production AI systems built and run by starmo. n=5 systems, one team. Not a survey.",
  "data": [
    {
      "metric": "production_rag_cost_per_query_low",
      "value": "0.006",
      "unit": "USD",
      "system": "DocsAI (aviation retrieval assistant)",
      "measurement_method": "Per-query tracing in production admin console; includes hybrid retrieval + LLM rerank + generation + sufficiency check"
    },
    {
      "metric": "production_rag_cost_per_query_high",
      "value": "0.02",
      "unit": "USD",
      "system": "DocsAI (aviation retrieval assistant)",
      "measurement_method": "Per-query tracing in production admin console; includes hybrid retrieval + LLM rerank + generation + sufficiency check"
    },
    {
      "metric": "regulatory_monitoring_cost_per_weekly_run",
      "value": "0.02",
      "unit": "USD",
      "system": "Regulation Monitoring (airline production module)",
      "measurement_method": "LLM spend per weekly cycle across 4 scraper architectures; summaries generated only for changed sections"
    },
    {
      "metric": "agent_eval_score_baseline",
      "value": "59.0",
      "unit": "percent",
      "system": "docsai-autoresearch (closed-loop optimizer)",
      "measurement_method": "Deterministic rubric: tool-selection 50% + content checks 25% + error-free completion 15% + efficiency 10%; 28-case eval vs mocked production API"
    },
    {
      "metric": "agent_eval_score_best",
      "value": "72.4",
      "unit": "percent",
      "system": "docsai-autoresearch (closed-loop optimizer)",
      "measurement_method": "Same rubric after ~20 accept/reject-gated improvement iterations"
    },
    {
      "metric": "automated_tests_total",
      "value": "1609",
      "unit": "tests",
      "system": "All 5 systems",
      "measurement_method": "Test functions counted in source (630 DocsAI + 173 RegNotify + 51 Wabel/eFMCG + 755 nowreply)"
    },
    {
      "metric": "golden_eval_cases",
      "value": "100",
      "unit": "cases",
      "system": "DocsAI",
      "measurement_method": "Golden question set with LLM-as-judge scoring run against releases"
    },
    {
      "metric": "regulations_monitored",
      "value": "5048",
      "unit": "documents",
      "system": "RegNotify (live SaaS)",
      "measurement_method": "Live corpus count at regnotify.com on 2026-08-18; ingestion runs every 6 hours"
    },
    {
      "metric": "aircraft_types_covered",
      "value": "2764",
      "unit": "types",
      "system": "RegNotify (live SaaS)",
      "measurement_method": "Live count at regnotify.com"
    },
    {
      "metric": "seo_urls_generated_single_pass",
      "value": "7453",
      "unit": "URLs",
      "system": "RegNotify (live SaaS)",
      "measurement_method": "AI generation pipeline producing page content + RAG context + FAQ JSON-LD in one pass"
    },
    {
      "metric": "database_migrations_nowreply",
      "value": "46",
      "unit": "migrations",
      "system": "nowreply (live SaaS)",
      "measurement_method": "Alembic migration count in repository"
    },
    {
      "metric": "backend_tests_nowreply",
      "value": "755",
      "unit": "tests",
      "system": "nowreply (live SaaS)",
      "measurement_method": "Test functions counted in source"
    },
    {
      "metric": "vector_database_used",
      "value": "pgvector",
      "unit": "software",
      "system": "All 5 systems",
      "measurement_method": "Postgres + pgvector (HNSW) served retrieval in every system; no dedicated vector database required"
    }
  ]
}