habity

# 내 GPU가 FP8을 네이티브로 아는지 확인 (8.9 이상이면 FP8 OK)
nvidia-smi --query-gpu=name,compute_cap --format=csv,noheader
# NVIDIA L40S, 8.9

R=nvidia/tensorrt-llm/release
T=$(curl -s "https://nvcr.io/proxy_auth?scope=repository:$R:pull" \
    | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])')

# 버전 태그만 추출
curl -s -H "Authorization: Bearer $T" "https://nvcr.io/v2/$R/tags/list" \
 | python3 -c 'import sys,json,re; t=json.load(sys.stdin)["tags"]; print("\n".join([x for x in t if re.match(r"^\d+\.\d+\.\d+$",x)]))'
# 0.20.0 / 0.21.0 / 1.0.0 / 1.1.0 / 1.2.0 / 1.2.1   ← 최신 stable = 1.2.1

# 이미지 크기(압축 기준) 미리 확인
curl -s -H "Authorization: Bearer $T" \
  -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
  "https://nvcr.io/v2/$R/manifests/1.2.1" \
 | python3 -c 'import sys,json; d=json.load(sys.stdin); print(f"{sum(l[\"size\"] for l in d[\"layers\"])/2**30:.1f} GB")'
# 19.5 GB

# trtllm-shell.yaml — 실습용 셸 파드
apiVersion: v1
kind: Pod
metadata: { name: trtllm-shell, namespace: trtllm }
spec:
  restartPolicy: Never
  nodeSelector: { nvidia.com/gpu.present: "true" }   # ← nodeName 쓰지 말 것 (아래 트러블슈팅 ②)
  containers:
  - name: trtllm
    image: nvcr.io/nvidia/tensorrt-llm/release:1.2.1
    command: ["bash","-lc","sleep infinity"]          # ← -l 필수 (아래 트러블슈팅 ③)
    env: [ { name: HF_HOME, value: /hf } ]
    resources:
      limits: { nvidia.com/gpu: "1", nvidia.com/gpumem: "44000", nvidia.com/gpucores: "100", cpu: "12", memory: 48Gi }
      requests: { cpu: "4", memory: 16Gi }
    volumeMounts:
      - { name: hf,  mountPath: /hf }
      - { name: shm, mountPath: /dev/shm }            # ← MPI/NCCL용
  volumes:
    - { name: hf,  hostPath: { path: /var/lib/hf-cache, type: DirectoryOrCreate } }
    - { name: shm, emptyDir: { medium: Memory, sizeLimit: 16Gi } }

kubectl create ns trtllm
kubectl apply -f trtllm-shell.yaml
kubectl -n trtllm exec trtllm-shell -- bash -lc 'python3 -c "import tensorrt_llm; print(tensorrt_llm.__version__)"'
# TensorRT-LLM 1.2.1

# quickstart.py
from tensorrt_llm import LLM, SamplingParams

def main():                                  # ← 이 감싸기가 필수 (아래 트러블슈팅 ④)
    prompts = [
        "Hello, my name is",
        "The capital of France is",
        "The future of AI is",
    ]
    sampling_params = SamplingParams(temperature=0.8, top_p=0.95)

    llm = LLM(model="TinyLlama/TinyLlama-1.1B-Chat-v1.0")
    outputs = llm.generate(prompts, sampling_params)

    for output in outputs:
        print(f"Prompt: {output.prompt!r}, Generated text: {output.outputs[0].text!r}")

if __name__ == "__main__":
    main()

kubectl -n trtllm exec trtllm-shell -- bash -lc 'python3 /tmp/quickstart.py'

[TensorRT-LLM][INFO] Number of tokens per block: 32.
[TensorRT-LLM][INFO] [MemUsageChange] Allocated 35.84 GiB for max tokens in paged KV cache (1708256).
Prompt: 'Hello, my name is', Generated text: 'John Smith. I am a male and I love to play video games...'
Prompt: 'The capital of France is', Generated text: 'Paris.'
Prompt: 'The future of AI is', Generated text: "something that everyone will see in their own lives..."

# engine.yaml — 서버 + Service (실험마다 args만 교체)
apiVersion: v1
kind: Service
metadata: { name: engine, namespace: trtllm }
spec:
  selector: { app: engine }
  ports: [ { port: 8000, targetPort: 8000 } ]
---
apiVersion: apps/v1
kind: Deployment
metadata: { name: engine, namespace: trtllm }
spec:
  replicas: 1
  strategy: { type: Recreate }          # GPU 1장 → 롤링 불가
  selector: { matchLabels: { app: engine } }
  template:
    metadata: { labels: { app: engine } }
    spec:
      nodeSelector: { nvidia.com/gpu.present: "true" }
      containers:
      - name: engine
        image: nvcr.io/nvidia/tensorrt-llm/release:1.2.1
        command: ["bash","-lc"]
        args: ["exec trtllm-serve Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --port 8000 --backend pytorch"]
        env: [ { name: HF_HOME, value: /hf } ]
        ports: [ { containerPort: 8000 } ]
        resources:
          limits: { nvidia.com/gpu: "1", nvidia.com/gpumem: "44000", nvidia.com/gpucores: "100", cpu: "12", memory: 48Gi }
          requests: { cpu: "4", memory: 16Gi }
        readinessProbe: { httpGet: { path: /health, port: 8000 }, initialDelaySeconds: 30, periodSeconds: 10, failureThreshold: 180 }
        volumeMounts:
          - { name: hf,  mountPath: /hf }
          - { name: shm, mountPath: /dev/shm }
      volumes:
        - { name: hf,  hostPath: { path: /var/lib/hf-cache, type: DirectoryOrCreate } }
        - { name: shm, emptyDir: { medium: Memory, sizeLimit: 16Gi } }

kubectl apply -f engine.yaml
kubectl -n trtllm rollout status deploy/engine --timeout=1200s   # 최초 기동 ~100초 (모델 캐시 있을 때)

kubectl -n trtllm exec bench -- curl -s http://engine:8000/v1/models
# {"object":"list","data":[{"id":"Qwen/Qwen2.5-7B-Instruct","object":"model","owned_by":"tensorrt_llm"}]}

kubectl -n trtllm exec bench -- curl -s http://engine:8000/v1/chat/completions \
  -H 'Content-Type: application/json' -d '{
    "model":"Qwen/Qwen2.5-7B-Instruct",
    "messages":[{"role":"user","content":"What is model quantization? Answer in one sentence."}],
    "max_tokens":60,"temperature":0}'

kubectl -n trtllm logs -l app=engine --tail=1200 | grep -oE \
  "max_batch_size=[0-9]+|max_num_tokens=[0-9]+|free_gpu_memory_fraction=[0-9.]+|max_tokens=[0-9]+|enable_chunked_prefill=[A-Za-z]+|enable_block_reuse=[A-Za-z]+|tokens_per_block=[0-9]+|attn_backend='[A-Z]+'"

┌──────────────┐    OpenAI /v1/completions    ┌──────────────────────────┐
│  bench 파드   │ ───────────────────────────► │  engine Service :8000     │
│ vllm bench    │                              │  ├ trtllm-serve  (TRT-LLM)│
│   serve       │ ◄─────────────────────────── │  └ vllm serve    (vLLM)   │
│ (GPU 미점유)  │      동일 지표 정의            └──────────────────────────┘
└──────────────┘

# 공통 벤치 커맨드 — 엔진과 무관하게 동일
vllm bench serve --backend openai --endpoint /v1/completions \
  --base-url http://engine:8000 --model Qwen/Qwen2.5-7B-Instruct \
  --dataset-name random --random-input-len 1024 --random-output-len 256 \
  --num-prompts 300 --max-concurrency 128 \
  --ignore-eos --temperature 0 --seed 12345 \
  --percentile-metrics ttft,tpot,itl,e2el --metric-percentiles 90,99

# ptq_fp8.py — Qwen2.5-7B-Instruct → FP8 (weight+activation per-tensor)
import torch, time
from transformers import AutoModelForCausalLM, AutoTokenizer
import modelopt.torch.quantization as mtq
from modelopt.torch.export import export_hf_checkpoint

MODEL = "Qwen/Qwen2.5-7B-Instruct"
OUT   = "/hf/qwen2.5-7b-fp8"
NCALIB, SEQLEN = 256, 512

def main():
    tok   = AutoTokenizer.from_pretrained(MODEL)
    model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16, device_map="cuda")

    # 캘리브레이션: 활성값 스케일(amax)을 관측하기 위한 실제 데이터가 필요
    from datasets import load_dataset
    ds = load_dataset("cnn_dailymail", "3.0.0", split=f"train[:{NCALIB}]")
    texts = [x["article"] for x in ds]

    def forward_loop(m):
        for t in texts:
            ids = tok(t, return_tensors="pt", truncation=True, max_length=SEQLEN).input_ids.cuda()
            m(ids)

    model = mtq.quantize(model, mtq.FP8_DEFAULT_CFG, forward_loop)

    with torch.inference_mode():
        export_hf_checkpoint(model, export_dir=OUT)   # ModelOpt 통합 HF 체크포인트
    tok.save_pretrained(OUT)

if __name__ == "__main__":
    main()

kubectl -n trtllm exec quant -- bash -lc 'python3 /tmp/ptq_fp8.py'

Registered Qwen2Attention to _QuantAttention for KV Cache quantization
Inserted 675 quantizers
  calib 256/256
[quantize] 31.0s
[export]    8.5s → /hf/qwen2.5-7b-fp8
[total]    66.7s

# 산출물 확인
du -sh /hf/qwen2.5-7b-fp8          # 8.2G  (BF16 15G 대비 −45%)
cat /hf/qwen2.5-7b-fp8/hf_quant_config.json

{
    "producer": { "name": "modelopt", "version": "0.37.0" },
    "quantization": {
        "quant_algo": "FP8",
        "kv_cache_quant_algo": null,
        "exclude_modules": ["lm_head"]
    }
}

# TRT-LLM
trtllm-serve /hf/qwen2.5-7b-fp8 --host 0.0.0.0 --port 8000 --backend pytorch

# vLLM — hf_quant_config.json 을 읽어 quantization=modelopt 로 자동 감지
vllm serve /hf/qwen2.5-7b-fp8 --served-model-name qwen2.5-7b-fp8 --host 0.0.0.0 --port 8000

# 1) 합성 데이터셋 생성 (W1 형상)
trtllm-bench --model Qwen/Qwen2.5-7B-Instruct prepare-dataset \
  --output /tmp/ds_w1.jsonl \
  token-norm-dist --num-requests 300 \
  --input-mean 1024 --input-stdev 0 --output-mean 256 --output-stdev 0

# 2) BF16
trtllm-bench --model Qwen/Qwen2.5-7B-Instruct \
  throughput --dataset /tmp/ds_w1.jsonl --backend pytorch \
  --concurrency 128 --num_requests 300

# 3) FP8 — 로컬 체크포인트는 --model_path 로 지정 (아래 트러블슈팅 ⑤)
trtllm-bench --model Qwen/Qwen2.5-7B-Instruct --model_path /hf/qwen2.5-7b-fp8 \
  throughput --dataset /tmp/ds_w1.jsonl --backend pytorch \
  --concurrency 128 --num_requests 300

# 전체 벤치 스윕
./run_trtllm_bench.sh <label> <in> <out> <num_prompts> <concurrency> <outfile>

# 엔진 교체는 args 만 patch (Recreate 전략)
kubectl -n trtllm patch deploy engine --type=json -p '[
 {"op":"replace","path":"/spec/template/spec/containers/0/image","value":"vllm/vllm-openai:latest"},
 {"op":"replace","path":"/spec/template/spec/containers/0/args",
  "value":["exec vllm serve Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --port 8000"]}]'
kubectl -n trtllm rollout status deploy/engine --timeout=1200s

# 정리
kubectl delete ns trtllm            # GPU 반납
# 모델 캐시는 노드 hostPath /var/lib/hf-cache 에 잔존

# 1. GPU 노드 및 HAMi 할당 슬롯 확인
kubectl get nodes -o custom-columns='NODE:.metadata.name,GPU:.status.capacity.nvidia\.com/gpu'

# 2. 물리 GPU 세부 사양 및 HAMi 분할 상태 확인 (44GB 메타데이터 확인)
kubectl get node <gpu-node-name> -o jsonpath='{.metadata.annotations.hami\.io/node-nvidia-register}'

# vllm.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm
  namespace: vllm-bench
  labels:
    app: vllm
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: vllm
  template:
    metadata:
      labels:
        app: vllm
    spec:
      containers:
      - name: vllm
        image: vllm/vllm-openai:latest
        args:
          - "--model=Qwen/Qwen2.5-7B-Instruct"
          - "--port=8000"
          - "--gpu-memory-utilization=0.85"
        ports:
        - containerPort: 8000
        env:
        - name: HF_HOME
          value: /root/.cache/huggingface
        resources:
          limits:
            nvidia.com/gpu: "1"
            nvidia.com/gpumem: "44000"
            nvidia.com/gpucores: "100"
            cpu: "10"
            memory: 40Gi
          requests:
            cpu: "4"
            memory: 16Gi
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 30
          periodSeconds: 5
          failureThreshold: 240
        volumeMounts:
        - name: hf-cache
          mountPath: /root/.cache/huggingface
        - name: shm
          mountPath: /dev/shm
      volumes:
      - name: hf-cache
        hostPath:
          path: /var/lib/vllm-hf-cache
          type: DirectoryOrCreate
      - name: shm
        emptyDir:
          medium: Memory
          sizeLimit: 8Gi

# 네임스페이스 생성 및 vLLM 배포
kubectl create ns vllm-bench
kubectl apply -f vllm.yaml

# 롤아웃 대기 (최초 배포 시 모델 다운로드로 수 분 소요)
kubectl -n vllm-bench rollout status deploy/vllm --timeout=1800s

# 서빙 상태 확인
kubectl -n vllm-bench exec deploy/vllm -- curl -s localhost:8000/v1/models

POD=$(kubectl -n vllm-bench get pod -l app=vllm -o name | head -1)

# CLI 기본 도움말 조회
kubectl -n vllm-bench exec $POD -- sh -c 'vllm serve --help=max-num-seqs'
kubectl -n vllm-bench exec $POD -- sh -c 'vllm serve --help=max-num-batched-tokens'
kubectl -n vllm-bench exec $POD -- sh -c 'vllm serve --help=max-model-len'
kubectl -n vllm-bench exec $POD -- sh -c 'vllm serve --help=enable-chunked-prefill'

# 기동 로그에서 실제 산출된 값 확인
kubectl -n vllm-bench logs $POD | grep -iE "max_seq_len|enable_chunked_prefill|enable_prefix_caching|GPU KV cache size|Maximum concurrency"

POD=$(kubectl -n vllm-bench get pod -l app=vllm -o name | head -1)

kubectl -n vllm-bench exec $POD -- sh -c 'vllm bench serve \
  --model Qwen/Qwen2.5-7B-Instruct --host localhost --port 8000 \
  --dataset-name random --random-input-len 1024 --random-output-len 256 \
  --num-prompts 300 --max-concurrency 128 \
  --ignore-eos --temperature 0 --seed 12345 \
  --percentile-metrics ttft,tpot,itl,e2el --metric-percentiles 90,99'

# 예시: max-num-seqs=32 설정 패치 명령
kubectl -n vllm-bench patch deploy vllm --type=json -p \
 '[{"op":"replace","path":"/spec/template/spec/containers/0/args",
    "value":["--model=Qwen/Qwen2.5-7B-Instruct","--port=8000","--gpu-memory-utilization=0.85","--max-num-seqs=32"]}]'

kubectl -n vllm-bench rollout status deploy/vllm --timeout=420s

# 1. Chunked Prefill ON & batched tokens=2048 패치
kubectl -n vllm-bench patch deploy vllm --type=json -p \
 '[{"op":"replace","path":"/spec/template/spec/containers/0/args",
    "value":["--model=Qwen/Qwen2.5-7B-Instruct","--port=8000","--gpu-memory-utilization=0.85","--max-num-batched-tokens=2048"]}]'

# 2. Chunked Prefill OFF 패치
kubectl -n vllm-bench patch deploy vllm --type=json -p \
 '[{"op":"replace","path":"/spec/template/spec/containers/0/args",
    "value":["--model=Qwen/Qwen2.5-7B-Instruct","--port=8000","--gpu-memory-utilization=0.85","--no-enable-chunked-prefill"]}]'

# 튜닝 자동화 스크립트 예시
apply_cfg() {
  ARG_STR=$1
  kubectl -n vllm-bench patch deploy vllm --type=json -p \
   "[{\"op\":\"replace\",\"path\":\"/spec/template/spec/containers/0/args\",
      \"value\":[\"--model=Qwen/Qwen2.5-7B-Instruct\",\"--port=8000\",\"--gpu-memory-utilization=0.85\" ${ARG_STR:+, $ARG_STR}]}]"
  kubectl -n vllm-bench rollout status deploy/vllm --timeout=420s
}

bench() {
  LABEL=$1; IN_LEN=$2; OUT_LEN=$3; NUM_P=$4; CONC=$5
  POD=$(kubectl -n vllm-bench get pod -l app=vllm -o name | head -1)
  echo "=== Running $LABEL ==="
  kubectl -n vllm-bench exec $POD -- sh -c "vllm bench serve \
    --model Qwen/Qwen2.5-7B-Instruct --host localhost --port 8000 \
    --dataset-name random --random-input-len $IN_LEN --random-output-len $OUT_LEN \
    --num-prompts $NUM_P --max-concurrency $CONC \
    --ignore-eos --temperature 0 --seed 12345 \
    --percentile-metrics ttft,tpot,itl,e2el --metric-percentiles 90,99"
}

# 과제 1 실행
apply_cfg ""
bench C1_baseline 1024 256 300 128

apply_cfg '"--max-num-seqs=32"'
bench C1_seqs32 1024 256 300 128

apply_cfg '"--max-num-seqs=512"'
bench C1_seqs512 1024 256 300 128

apply_cfg '"--max-model-len=8192"'
bench C1_modellen8192 1024 256 300 128

# 과제 2 실행
apply_cfg ""
bench C2_baseline 4096 256 200 64

apply_cfg '"--max-num-batched-tokens=2048"'
bench C2_mnbt2048 4096 256 200 64

apply_cfg '"--max-num-batched-tokens=16384"'
bench C2_mnbt16384 4096 256 200 64

apply_cfg '"--no-enable-chunked-prefill"'
bench C2_nochunk 4096 256 200 64

# 자원 자원 해제 (네임스페이스 삭제)
kubectl delete ns vllm-bench

Physical Server (16 Core CPU, 64GB RAM, 1TB Disk)
       │
       ▼
   Hypervisor
 ┌─────┼─────┐
 ▼     ▼     ▼
VM 1  VM 2  VM 3 (각 4 vCPU, 16GB RAM)

[ 전통적인 구조 ]
Application (Ring 3) ──System Call──> Linux Kernel (Ring 0) ──> Hardware

Kubernetes API ──> KubeVirt ──> virt-launcher Pod ──> QEMU / KVM ──> Physical Hardware