Introducing SkyServe: 50% Cheaper AI Serving on Any Cloud with High Availability

# mixtral.yaml
service:
  readiness_probe:
    path: /v1/chat/completions
    post_data:
      model: mistralai/Mixtral-8x7B-Instruct-v0.1
      messages:
        - role: user
          content: Hello! What is your name?
      max_tokens: 1
    initial_delay_seconds: 1800
  replicas: 2

# Fields below describe each replica.
resources:
  ports: 8080
  accelerators: {L4:8, A10g:8, A100:4, A100:8, A100-80GB:2, A100-80GB:4, A100-80GB:8}

setup: |
  conda create -n vllm python=3.10 -y
  conda activate vllm
  pip install vllm==0.3.0 transformers==4.37.2  

run: |
  conda activate vllm
  export PATH=$PATH:/sbin
  python -m vllm.entrypoints.openai.api_server \
    --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \
    --host 0.0.0.0 --port 8080 \
    --model mistralai/Mixtral-8x7B-Instruct-v0.1  

# SkyServe YAML to run HuggingFace TGI
#
# Usage:
#   sky serve up -n tgi huggingface-tgi.yaml
# Then visit the endpoint printed in the console. You could also
# check the endpoint by running:
#   sky serve status --endpoint tgi

envs:
  MODEL_ID: lmsys/vicuna-13b-v1.5

service:
  readiness_probe: /health
  replicas: 2

resources:
  ports: 8082
  accelerators: A100:1

run: |
  docker run --gpus all --shm-size 1g -p 8082:80 \
    -v ~/data:/data ghcr.io/huggingface/text-generation-inference \
    --model-id $MODEL_ID  

curl -L $(sky serve status tgi --endpoint)/generate \
    -X POST \
    -H 'Content-Type: application/json' \
    -d '{
      "inputs": "What is Deep Learning?",
      "parameters": {
        "max_new_tokens": 20
      }
    }'

# Example Output:
{"generated_text":"\n\nDeep learning is a subset of machine learning that uses artificial neural networks to model and solve"}

# An example yaml for serving Code Llama model from Meta with an OpenAI API.
# Usage:
#  1. Launch on a single instance: `sky launch -c code-llama codellama.yaml`
#  2. Scale up to multiple replicas with a single endpoint:
#     `sky serve up -n code-llama codellama.yaml`
service:
  readiness_probe:
    path: /v1/completions
    post_data:
      model: codellama/CodeLlama-70b-Instruct-hf
      prompt: "def hello_world():"
      max_tokens: 1
    initial_delay_seconds: 1800
  replicas: 2

resources:
  accelerators: {L4:8, A10g:8, A10:8, A100:4, A100:8, A100-80GB:2, A100-80GB:4, A100-80GB:8}
  disk_size: 1024
  disk_tier: best
  memory: 32+
  ports: 8000

setup: |
  conda activate codellama
  if [ $? -ne 0 ]; then
    conda create -n codellama python=3.10 -y
    conda activate codellama
  fi

  pip list | grep vllm || pip install "git+https://github.com/vllm-project/vllm.git"
  pip install git+https://github.com/huggingface/transformers  

run: |
  conda activate codellama
  export PATH=$PATH:/sbin
  # Reduce --max-num-seqs to avoid OOM during loading model on L4:8
  python -u -m vllm.entrypoints.openai.api_server \
    --host 0.0.0.0 \
    --model codellama/CodeLlama-70b-Instruct-hf \
    --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \
    --max-num-seqs 64 | tee ~/openai_api_server.log  

curl -L $(sky serve status codellama --endpoint)/v1/completions \
    -H "Content-Type: application/json" \
    -d '{
      "model": "codellama/CodeLlama-70b-Instruct-hf",
      "prompt": "def quick_sort(a: List[int]):",
      "max_tokens": 512
  }'

# Example Output:
{"id":"cmpl-f0ceac768bf847269f64c2e5bc08126f","object":"text_completion","created":1770,"model":"codellama/CodeLlama-70b-Instruct-hf","choices":[{"index":0,"text":"\n    if len(a) < 2:\n        return a\n    else:\n        pivot = a[0]\n        lesser = quick_sort([x for x in a[1:] if x < pivot])\n        greater = quick_sort([x for x in a[1:] if x >= pivot])\n        return lesser + pivot + greaterq\n        \n","logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":13,"total_tokens":97,"completion_tokens":84}}

# SkyServe YAML for vLLM inference engine
#
# Usage:
#   set your HF_TOKEN in the envs section or pass it in CLI
#   sky serve up -n vllm vllm.yaml
# Then visit the endpoint printed in the console. You could also
# check the endpoint by running:
#   sky serve status --endpoint vllm
service:
  # Specifying the path to the endpoint to check the readiness of the service.
  readiness_probe: /v1/models
  # How many replicas to manage.
  replicas: 2

envs:
  MODEL_NAME: meta-llama/Llama-2-7b-chat-hf
  HF_TOKEN: <your-huggingface-token> # Change to your own huggingface token

resources:
  accelerators: {L4:1, A10G:1, A10:1, A100:1, A100-80GB:1}
  ports:
    - 8000

setup: |
  conda activate vllm
  if [ $? -ne 0 ]; then
    conda create -n vllm python=3.10 -y
    conda activate vllm
  fi

  pip list | grep vllm || pip install vllm==0.3.0
  pip list | grep transformers || pip install transformers==4.37.2

  python -c "import huggingface_hub; huggingface_hub.login('${HF_TOKEN}')"  


run: |
  conda activate vllm
  echo 'Starting vllm openai api server...'
  python -m vllm.entrypoints.openai.api_server \
    --model $MODEL_NAME --tokenizer hf-internal-testing/llama-tokenizer \
    --host 0.0.0.0  

curl -L $(sky serve status vllm --endpoint)/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-2-7b-chat-hf",
    "prompt": "San Francisco is a",
    "max_tokens": 7,
    "temperature": 0
  }'

# Example Output:
{"id":"cmpl-6e3b960a5e7b47dbadb8846e76aef87d","object":"text_completion","created":1479,"model":"meta-llama/Llama-2-7b-chat-hf","choices":[{"index":0,"text":" city in Northern California that is known","logprobs":null,"finish_reason":"length"}],"usage":{"prompt_tokens":5,"total_tokens":12,"completion_tokens":7}}

# SkyServe YAML for SGLang
#
# Usage:
#   set your HF_TOKEN in the envs section or pass it in CLI
#   sky serve up -n sglang sglang.yaml --env HF_TOKEN=<your-huggingface-token>
# Then visit the endpoint printed in the console. You could also
# check the endpoint by running:
#   sky serve status --endpoint sglang
service:
  # Specifying the path to the endpoint to check the readiness of the service.
  readiness_probe: /health
  # How many replicas to manage.
  replicas: 2

envs:
  MODEL_NAME: liuhaotian/llava-v1.6-vicuna-7b
  TOKENIZER_NAME: llava-hf/llava-1.5-7b-hf

resources:
  accelerators: {L4:1, A10G:1, A10:1, A100:1, A100-80GB:1}
  ports:
    - 8000

setup: |
  conda activate sglang
  if [ $? -ne 0 ]; then
    conda create -n sglang python=3.10 -y
    conda activate sglang
  fi

  pip list | grep sglang || pip install "sglang[all]"
  pip list | grep transformers || pip install transformers==4.37.2  

run: |
  conda activate sglang
  echo 'Starting sglang openai api server...'
  export PATH=$PATH:/sbin/
  python -m sglang.launch_server --model-path $MODEL_NAME \
    --tokenizer-path $TOKENIZER_NAME \
    --chat-template vicuna_v1.1 \
    --host 0.0.0.0 --port 8000  

curl -L $(sky serve status sglang --endpoint)/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "liuhaotian/llava-v1.6-vicuna-7b",
    "messages": [
      {
        "role": "user",
        "content": [
            {"type": "text", "text": "Describe this image"},
            {
              "type": "image_url",
              "image_url": {
                "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/examples/quick_start/images/cat.jpeg"
              }
            }
        ]
      }
    ]
  }'

# Example Output:
{"id":"eea519a1e8a6496fa449d3b96a3c99c1","object":"chat.completion","created":1707709646,"model":"liuhaotian/llava-v1.6-vicuna-7b","choices":[{"index":0,"message":{"role":"assistant","content":" This is an image of an anthropomorphic cat character wearing sunglass"},"finish_reason":null}],"usage":{"prompt_tokens":46,"total_tokens":2205,"completion_tokens":2159}}

# SkyServe YAML for code assistant in VSCode with Tabby
#
# Usage:
#   sky serve up -n tabby tabby.yaml
# Then visit the endpoint printed in the console. You could also
# check the endpoint by running:
#   sky serve status --endpoint tabby

service:
  readiness_probe: /v1/health
  replicas: 2

resources:
  ports: 8080
  accelerators: {T4:1, L4:1, A100:1, A10G:1}

run: |
  docker run --gpus all -p 8080:8080 -v ~/.tabby:/data \
    tabbyml/tabby \
    serve --model TabbyML/StarCoder-1B --device cuda  

curl -L $(sky serve status tabby --endpoint)/v1/completions \
  -X POST \
  -H "Content-Type: application/json" \
  -d '{
    "language": "python",
    "segments": {
      "prefix": "def fib(n):\n    ",
      "suffix": "\n        return fib(n - 1) + fib(n - 2)"
    }
  }'

# Example Output:
{"id":"cmpl-7305a984-429a-4342-bf95-3d47f668c8cd","choices":[{"index":0,"text":"    if n == 0:\n            return 0\n        if n == 1:\n            return 1"}]}

# SkyServe YAML for multi-LoRA inference server using LoRAX
#
# Usage:
#   sky serve up -n lorax lorax.yaml
# Then visit the endpoint printed in the console. You could also
# check the endpoint by running:
#   sky serve status --endpoint lorax

envs:
  MODEL_ID: mistralai/Mistral-7B-Instruct-v0.1

service:
  readiness_probe:
    path: /generate
    post_data:
      inputs: "[INST] Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May? [/INST]"
      parameters:
        max_new_tokens: 64
        adapter_id: vineetsharma/qlora-adapter-Mistral-7B-Instruct-v0.1-gsm8k
  replicas: 2

resources:
  accelerators: {L4:1, A100:1, A10G:1}
  memory: 32+
  ports: 
    - 8080

run: |
  docker run --gpus all --shm-size 1g -p 8080:80 -v ~/data:/data \
    ghcr.io/predibase/lorax:latest \
    --model-id $MODEL_ID  

curl -L $(sky serve status lorax --endpoint)/generate \
    -X POST \
    -H 'Content-Type: application/json' \
    -d '{
      "inputs": "[INST] Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May? [/INST]",
      "parameters": {
        "max_new_tokens": 64,
        "adapter_id": "vineetsharma/qlora-adapter-Mistral-7B-Instruct-v0.1-gsm8k"
      }
    }'

# Example Output:
{"generated_text":"Natalia sold 48/2 = <<48/2=24>>24 clips in May.\nIn total, Natalia sold 48 + 24 = <<48+24=72>>72 clips in April and May.\n#### 72"}