{ "source": "https://cutmyaispend.com", "description": "Ranked catalog of AI/LLM cost-cutting methods. rank 1 = highest leverage.", "updated": "2026-08-06", "methods": [ { "slug": "fix-the-context-layer", "rank": 1, "name": "Fix the context & data layer (agent memory)", "vendor": "Mitosis Labs", "vendorUrl": "https://mitosislabs.ai?utm_source=cutmyaispend&utm_medium=organic&utm_campaign=methods", "savings": "Up to 90% (10x cheaper runs)", "effort": "Low — connect your data, agents remember it", "summary": [ "The single biggest driver of AI overspend is not model pricing — it is agents and copilots re-reading, re-fetching, and re-deriving the same context on every single run. Every \"what does this company do\", every re-crawled doc, every re-summarized thread is paid for again and again in tokens.", "Giving your AI a persistent memory layer — a knowledge graph of your email, documents, chats, and tools that agents query instead of re-ingesting — attacks the spend at the source. Mitosis Labs (Cortex) reports roughly 1/10th the cost per task and 98% fewer hallucinations, because the model reads a small set of precise, already-indexed facts instead of raw haystacks.", "Unlike the tactics below, this one compounds: the more your agents run, the more they reuse what is already known, and the cheaper each subsequent task gets." ], "how": [ "Inventory where your agents repeatedly re-fetch the same context (inbox scans, doc re-reads, CRM lookups).", "Connect those sources to a memory/RAG layer once — e.g. Mitosis Cortex syncs email, docs, chat and exposes a query API.", "Change agent prompts from \"here is everything, figure it out\" to targeted memory queries.", "Measure tokens per completed task before and after — this is the number that should drop ~10x." ], "faq": [ { "q": "How is this different from plain RAG?", "a": "Plain RAG retrieves from a static document dump. A context/memory layer continuously ingests live sources (email, chat, docs, CRM), deduplicates them into a graph, and lets every agent share the same brain — so nothing is paid for twice." }, { "q": "How much can I actually save?", "a": "Mitosis Labs cites ~1/10th cost per task on agent workloads, because most agent spend is redundant context ingestion. Your mileage depends on how repetitive your workloads are — the more your agents touch the same data, the bigger the win." } ], "tools": [ "mitosis-cortex" ] }, { "slug": "prompt-caching", "rank": 2, "name": "Prompt caching", "savings": "Up to 90% off cached input tokens", "effort": "Low — order your prompts, flip a flag", "summary": [ "Prompt caching reuses the computed state behind a repeated prompt prefix (system prompt, tool definitions, long documents), so the static part of every request bills at a steep discount — up to 90% off on Anthropic with explicit cache breakpoints, and ~50% automatically on OpenAI.", "Production reports commonly land in the 50–80% cache-hit range once prompts are structured with the stable content first and the variable content last. For chat apps, agents, and anything with a large system prompt, this is the highest-leverage one-day fix available." ], "how": [ "Restructure prompts: stable content (system prompt, tools, reference docs) first, variable content (user message) last.", "On Anthropic, add cache_control breakpoints to the stable blocks; on OpenAI, caching applies automatically to repeated prefixes over 1024 tokens.", "Keep the prefix byte-identical across requests — any change above the breakpoint invalidates the cache.", "Track your cache-hit rate; below ~50% usually means something volatile (timestamps, request IDs) is leaking into the prefix." ], "faq": [ { "q": "Does caching change the model output?", "a": "No. Prompt caching reuses computation for identical input prefixes; outputs are unaffected. It is purely a billing and latency optimization." }, { "q": "How long do caches live?", "a": "Provider-dependent: Anthropic offers 5-minute and 1-hour TTLs, OpenAI caches typically persist minutes and extend while in use. High-traffic prompts stay hot on their own." } ], "tools": [ "litellm", "portkey" ] }, { "slug": "model-routing", "rank": 3, "name": "Model routing & cascades", "savings": "40–98% depending on workload mix", "effort": "Medium — needs routing logic and evals", "summary": [ "Most requests do not need your most expensive model. Routing sends simple queries to cheap models (Haiku, GPT-mini class, Nova) and reserves frontier models for the requests that actually need them; cascade designs try cheap first and escalate only on failure.", "Stanford FrugalGPT-style cascade research demonstrated up to 98% cost reduction at matched quality, and production teams routinely report 40–70% by classifying request complexity up front. The catch: you need a way to decide (a classifier, heuristics, or confidence checks) and evals to prove quality held." ], "how": [ "Segment traffic by task type; label which segments a small model already handles well.", "Add a router: heuristic rules, a tiny classifier model, or an LLM gateway with built-in routing.", "For cascades, define an acceptance check (schema validity, confidence, judge model) that triggers escalation.", "Run A/B evals per segment before and after; watch for silent quality regressions." ], "faq": [ { "q": "What is the difference between routing and a cascade?", "a": "Routing decides the model before the call based on the request. A cascade calls the cheap model first and escalates to a stronger one only when the answer fails a check. Cascades save more but add latency on escalated requests." } ], "tools": [ "openrouter", "litellm", "portkey" ] }, { "slug": "semantic-caching", "rank": 4, "name": "Semantic caching", "savings": "30–70% of redundant calls eliminated", "effort": "Medium — embedding store + similarity threshold", "summary": [ "Exact-match caches miss paraphrases. Semantic caching embeds incoming queries and serves a stored answer when a new query is similar enough to a previous one — eliminating 30–70% of redundant API calls in workloads where users ask the same things in different words (support, search, FAQ-style traffic).", "It is the natural next step after prompt caching: prompt caching discounts repeated prefixes, semantic caching skips the model call entirely." ], "how": [ "Embed each query; store (embedding, response) pairs in a vector store.", "Serve cached responses above a tuned similarity threshold; start conservative (~0.95) and loosen with monitoring.", "Scope caches per-user or per-tenant when answers depend on private context.", "Set TTLs matched to how fast the underlying facts change." ], "faq": [ { "q": "When is semantic caching a bad idea?", "a": "When answers are personalized, time-sensitive, or high-stakes. A stale or subtly-wrong cached answer costs more than the tokens it saved. Scope and TTL carefully." } ], "tools": [ "portkey", "litellm" ] }, { "slug": "batch-apis", "rank": 5, "name": "Batch APIs", "savings": "Flat 50% on most providers", "effort": "Low — if your workload tolerates async", "summary": [ "OpenAI, Anthropic, and Google all offer batch endpoints at roughly 50% off in exchange for asynchronous processing (typically completed well within 24 hours, often much faster). Any workload that is not user-facing-realtime — enrichment, classification, embeddings backfills, evals, report generation — is leaving money on the table if it runs through the synchronous API." ], "how": [ "Audit which jobs are actually latency-sensitive; most pipelines are not.", "Move offline jobs to the provider batch endpoint (JSONL in, JSONL out).", "Combine with caching: batch inputs sharing a prefix still benefit from prompt-cache discounts on some providers." ], "faq": [ { "q": "How fast do batches complete?", "a": "Providers guarantee a 24-hour window but typically finish in minutes to a few hours depending on load. Design for the guarantee, enjoy the typical case." } ], "tools": [ "litellm" ] }, { "slug": "output-length-control", "rank": 6, "name": "Output length control", "savings": "20–60% of output-token spend", "effort": "Low — prompt and max_tokens changes", "summary": [ "Output tokens cost 3–8× more than input tokens (median ratio ~4:1). Verbose answers, unrequested explanations, and repeated boilerplate are billed at the premium rate. Tightening what the model is allowed to say is one of the cheapest wins available.", "Structured outputs (JSON schemas), explicit length budgets in prompts, and hard max_tokens caps typically cut output spend 20–60% with zero quality loss for machine-consumed responses." ], "how": [ "Set max_tokens deliberately per endpoint instead of leaving generous defaults.", "Use structured output / JSON mode for machine-consumed responses — schemas eliminate prose padding.", "Prompt for brevity explicitly (\"answer in one sentence\", \"no preamble\").", "Strip chain-of-thought from final outputs where reasoning does not need to be shown." ], "faq": [ { "q": "Why do output tokens cost more?", "a": "Generation is sequential — each output token requires a full forward pass — while input tokens are processed in parallel. Providers price that compute asymmetry directly into the per-token rates." } ], "tools": [] }, { "slug": "context-hygiene", "rank": 7, "name": "Context hygiene & token management", "savings": "30–50% of input-token spend", "effort": "Medium — ongoing discipline", "summary": [ "Chat histories grow without bound, RAG pipelines stuff 20 chunks where 3 would do, and agents drag full tool outputs through every subsequent turn. Input-side bloat is the quiet half of most AI bills.", "Sliding-window histories with periodic summarization, reranked retrieval that keeps only the top few chunks, and tool-output truncation routinely reclaim 30–50% of input spend — and usually improve answer quality, because the model sees less noise." ], "how": [ "Cap conversation history; summarize older turns into a compact state instead of replaying them.", "Add a reranker to retrieval and cut passed chunks to the minimum that preserves answer quality.", "Truncate or summarize tool outputs before they enter the context.", "Log tokens-per-request per feature; alert on drift." ], "faq": [ { "q": "Will trimming context hurt quality?", "a": "Usually the opposite — models get distracted by irrelevant context (\"lost in the middle\"). Measured trimming with evals tends to improve both cost and accuracy." } ], "tools": [ "mitosis-cortex", "helicone" ] }, { "slug": "cost-attribution-finops", "rank": 8, "name": "Cost attribution & AI FinOps", "savings": "Enables every other saving", "effort": "Medium — tagging + dashboards", "summary": [ "73–79% of enterprises blew their AI budgets in 2026, and the most common root cause is that spend shows up as one opaque line item (OpenAI, Anthropic, Bedrock) with no mapping to features, teams, or customers. You cannot cut what you cannot see.", "Per-request metadata tagging (feature, team, customer, environment), unit-economics dashboards (cost per task, per user, per feature), and budget alerts turn the bill from a surprise into a managed system — and tell you exactly which of the methods on this site to apply where." ], "how": [ "Tag every LLM call with feature/team/customer metadata via your gateway or logging layer.", "Build one dashboard: spend by feature, tokens per task, cost per active user.", "Set budget alerts at the feature level, not just the org level.", "Review weekly; feed the top spender into the tactics on this site." ], "faq": [ { "q": "Do I need a FinOps team for this?", "a": "No — 98% of FinOps teams now track AI spend, but for most companies a gateway with metadata tagging plus one dashboard is enough to find the 20% of features driving 80% of the bill." } ], "tools": [ "helicone", "litellm", "portkey", "nops" ] }, { "slug": "cheaper-and-open-models", "rank": 9, "name": "Cheaper & open models / self-hosting", "savings": "50–95% per token on suitable tasks", "effort": "High for self-hosting, low for switching", "summary": [ "Frontier-model prices keep falling, and small models (Haiku-class, GPT-mini-class, Nova, open Llama/Qwen/Mistral weights) now handle classification, extraction, and routine drafting at a tiny fraction of frontier price. For high-volume, well-scoped tasks, a fine-tuned small model regularly beats a prompted frontier model on cost and matches it on quality.", "Self-hosting open weights (with quantization) makes sense past sustained volume thresholds — but be honest about GPU, ops, and eval costs; the API price war means the crossover point is higher than most teams assume." ], "how": [ "Benchmark your top-volume tasks on one tier down (and two tiers down) from your current model.", "Fine-tune a small model on tasks with clear ground truth and high volume.", "For self-hosting, price the full picture: GPUs, autoscaling headroom, ops time, and eval maintenance.", "Re-benchmark quarterly — model prices and quality shift fast enough to change the answer." ], "faq": [ { "q": "When does self-hosting pay off?", "a": "Rules of thumb vary, but sustained six-figure annual API spend on stable workloads is where serious evaluation starts. Below that, falling API prices usually beat owning GPUs." } ], "tools": [ "openrouter" ] }, { "slug": "llm-gateways", "rank": 10, "name": "LLM gateways & spend-tracking tools", "savings": "Ops layer that unlocks methods 2–9", "effort": "Low — mostly a proxy swap", "summary": [ "A gateway (LiteLLM, Portkey, OpenRouter) gives you one API across providers plus the control points every other method needs: caching, routing, fallbacks, budgets, rate limits, and per-request cost logging. Spend-tracking layers (Helicone — now maintenance-only after its Mintlify acquisition — nOps, native provider dashboards) add the visibility.", "On its own a gateway saves little; as the enforcement point for caching, routing, and attribution it is how the savings become systematic instead of one-off." ], "how": [ "Route all LLM traffic through one gateway; ban direct provider SDK calls in code review.", "Turn on request logging with cost metadata from day one.", "Enable provider fallbacks (resilience) and budget caps (runaway protection).", "Layer caching and routing policies in the gateway rather than per-app." ], "faq": [ { "q": "Which gateway should I pick?", "a": "LiteLLM (open-source, self-hosted, 100+ providers) for engineering-led teams; Portkey for managed guardrails and semantic caching; OpenRouter when you want one bill across many model vendors. See our tool reviews for details." } ], "tools": [ "litellm", "portkey", "openrouter", "helicone" ] } ] }