Service tiers

Terminal provides two inference windows.

Window	Best for	Expected behavior
Standard	Interactive requests, agents, user-facing chat	Fast responses, 100+ tokens/sec on average
Flex	Offline jobs, batch-like work, cost-sensitive workloads	Same endpoint waits for completion, usually 5-20 minutes

Standard window

Standard is the default window. Use it when latency matters or when the model is serving an interactive workflow.

from openai import OpenAI

client = OpenAI(
    base_url="https://api.terminal.fyi/v1",
    api_key="TERMINAL_API_KEY",
)

response = client.chat.completions.create(
    model="gemma-4-31b",
    messages=[
        {
            "role": "user",
            "content": "Turn this deploy log into a one-paragraph on-call update.",
        }
    ],
    max_tokens=8000,
)

print(response.choices[0].message.content)

import OpenAI from "openai";

const client = new OpenAI({
  baseURL: "https://api.terminal.fyi/v1",
  apiKey: process.env.TERMINAL_API_KEY,
});

const response = await client.chat.completions.create({
  model: "gemma-4-31b",
  messages: [
    {
      role: "user",
      content: "Turn this deploy log into a one-paragraph on-call update.",
    },
  ],
  max_tokens: 8000,
});

console.log(response.choices[0].message.content);

import requests

response = requests.post(
    "https://api.terminal.fyi/v1/chat/completions",
    headers={
        "Authorization": "Bearer TERMINAL_API_KEY",
        "Content-Type": "application/json",
    },
    json={
        "model": "gemma-4-31b",
        "messages": [
            {
                "role": "user",
                "content": "Turn this deploy log into a one-paragraph on-call update.",
            }
        ],
        "max_tokens": 8000,
    },
    timeout=60,
)
response.raise_for_status()

completion = response.json()
print(completion["choices"][0]["message"]["content"])

const response = await fetch("https://api.terminal.fyi/v1/chat/completions", {
  method: "POST",
  headers: {
    Authorization: `Bearer ${process.env.TERMINAL_API_KEY}`,
    "Content-Type": "application/json",
  },
  body: JSON.stringify({
    model: "gemma-4-31b",
    messages: [
      {
        role: "user",
        content: "Turn this deploy log into a one-paragraph on-call update.",
      },
    ],
    max_tokens: 8000,
  }),
});
if (!response.ok) throw new Error(await response.text());

const completion = await response.json();
console.log(completion.choices[0].message.content);

curl -sS https://api.terminal.fyi/v1/chat/completions \
  -H "Authorization: Bearer $TERMINAL_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gemma-4-31b",
    "messages": [
      {
        "role": "user",
        "content": "Turn this deploy log into a one-paragraph on-call update."
      }
    ],
    "max_tokens": 8000
  }' | jq

Flex window

Flex is for workloads that can wait. It uses the same Chat Completions endpoint and returns when the request completes. Configure your HTTP client with a long timeout. Gemma 4 31B supports the Flex window with service_tier: "flex".

from openai import OpenAI

client = OpenAI(
    base_url="https://api.terminal.fyi/v1",
    api_key="TERMINAL_API_KEY",
    timeout=1800.0,
)

response = client.chat.completions.create(
    model="gemma-4-31b",
    service_tier="flex",
    messages=[
        {
            "role": "user",
            "content": "Review these nightly eval notes and return three follow-up tasks.",
        }
    ],
    max_tokens=8000,
)

print(response.choices[0].message.content)

import OpenAI from "openai";

const client = new OpenAI({
  baseURL: "https://api.terminal.fyi/v1",
  apiKey: process.env.TERMINAL_API_KEY,
  timeout: 1_800_000,
});

const response = await client.chat.completions.create({
  model: "gemma-4-31b",
  service_tier: "flex",
  messages: [
    {
      role: "user",
      content: "Review these nightly eval notes and return three follow-up tasks.",
    },
  ],
  max_tokens: 8000,
});

console.log(response.choices[0].message.content);

import requests

response = requests.post(
    "https://api.terminal.fyi/v1/chat/completions",
    headers={
        "Authorization": "Bearer TERMINAL_API_KEY",
        "Content-Type": "application/json",
    },
    json={
        "model": "gemma-4-31b",
        "service_tier": "flex",
        "messages": [
            {
                "role": "user",
                "content": "Review these nightly eval notes and return three follow-up tasks.",
            }
        ],
        "max_tokens": 8000,
    },
    timeout=1800,
)
response.raise_for_status()

completion = response.json()
print(completion["choices"][0]["message"]["content"])

const response = await fetch("https://api.terminal.fyi/v1/chat/completions", {
  method: "POST",
  headers: {
    Authorization: `Bearer ${process.env.TERMINAL_API_KEY}`,
    "Content-Type": "application/json",
  },
  body: JSON.stringify({
    model: "gemma-4-31b",
    service_tier: "flex",
    messages: [
      {
        role: "user",
        content: "Review these nightly eval notes and return three follow-up tasks.",
      },
    ],
    max_tokens: 8000,
  }),
  signal: AbortSignal.timeout(1_800_000),
});
if (!response.ok) throw new Error(await response.text());

const completion = await response.json();
console.log(completion.choices[0].message.content);

curl -sS --max-time 1800 https://api.terminal.fyi/v1/chat/completions \
  -H "Authorization: Bearer $TERMINAL_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gemma-4-31b",
    "service_tier": "flex",
    "messages": [
      {
        "role": "user",
        "content": "Review these nightly eval notes and return three follow-up tasks."
      }
    ],
    "max_tokens": 8000
  }' | jq

Use Flex for background jobs, long-running analysis, and other work where a 5-20 minute response time is acceptable.

Custom SLAs

Terminal will support request-level SLA hints in headers so you can define latency and cost preferences without changing the request body. These headers are not live yet.

from openai import OpenAI

client = OpenAI(
    base_url="https://api.terminal.fyi/v1",
    api_key="TERMINAL_API_KEY",
)

response = client.chat.completions.create(
    model="gemma-4-31b",
    messages=[
        {
            "role": "user",
            "content": "Extract launch-review action items and group them by owner.",
        }
    ],
    max_tokens=8000,
    extra_headers={
        "X-Terminal-SLA-Deadline": "10m",
        "X-Terminal-SLA-Priority": "cost",
    },
    timeout=1800.0,
)

print(response.choices[0].message.content)

import OpenAI from "openai";

const client = new OpenAI({
  baseURL: "https://api.terminal.fyi/v1",
  apiKey: process.env.TERMINAL_API_KEY,
});

const response = await client.chat.completions.create(
  {
    model: "gemma-4-31b",
    messages: [
      {
        role: "user",
        content: "Extract launch-review action items and group them by owner.",
      },
    ],
    max_tokens: 8000,
  },
  {
    headers: {
      "X-Terminal-SLA-Deadline": "10m",
      "X-Terminal-SLA-Priority": "cost",
    },
    timeout: 1_800_000,
  },
);

console.log(response.choices[0].message.content);

import requests

response = requests.post(
    "https://api.terminal.fyi/v1/chat/completions",
    headers={
        "Authorization": "Bearer TERMINAL_API_KEY",
        "Content-Type": "application/json",
        "X-Terminal-SLA-Deadline": "10m",
        "X-Terminal-SLA-Priority": "cost",
    },
    json={
        "model": "gemma-4-31b",
        "messages": [
            {
                "role": "user",
                "content": "Extract launch-review action items and group them by owner.",
            }
        ],
        "max_tokens": 8000,
    },
    timeout=1800,
)
response.raise_for_status()

completion = response.json()
print(completion["choices"][0]["message"]["content"])

const response = await fetch("https://api.terminal.fyi/v1/chat/completions", {
  method: "POST",
  headers: {
    Authorization: `Bearer ${process.env.TERMINAL_API_KEY}`,
    "Content-Type": "application/json",
    "X-Terminal-SLA-Deadline": "10m",
    "X-Terminal-SLA-Priority": "cost",
  },
  body: JSON.stringify({
    model: "gemma-4-31b",
    messages: [
      {
        role: "user",
        content: "Extract launch-review action items and group them by owner.",
      },
    ],
    max_tokens: 8000,
  }),
  signal: AbortSignal.timeout(1_800_000),
});
if (!response.ok) throw new Error(await response.text());

const completion = await response.json();
console.log(completion.choices[0].message.content);

curl -sS --max-time 1800 https://api.terminal.fyi/v1/chat/completions \
  -H "Authorization: Bearer $TERMINAL_API_KEY" \
  -H "Content-Type: application/json" \
  -H "X-Terminal-SLA-Deadline: 10m" \
  -H "X-Terminal-SLA-Priority: cost" \
  -d '{
    "model": "gemma-4-31b",
    "messages": [
      {
        "role": "user",
        "content": "Extract launch-review action items and group them by owner."
      }
    ],
    "max_tokens": 8000
  }' | jq