Add gh200 support and model (#300)

This PR will add the initial support for GH200 nodes in KubeAI

charts/kubeai/values.yaml

@@ -20,6 +20,7 @@ modelServers:
       default: "vllm/vllm-openai:v0.6.3.post1"
       cpu: "substratusai/vllm:v0.6.3.post1-cpu"
       google-tpu: "substratusai/vllm:v0.6.3.post1-tpu"
+      gh200: "drikster80/vllm-gh200-openai:v0.6.3.post1"
   OLlama:
     images:
       default: "ollama/ollama:latest"
@@ -97,6 +98,17 @@ resourceProfiles:
         operator: "Equal"
         value: "present"
         effect: "NoSchedule"
+  nvidia-gpu-gh200:
+    imageName: "gh200"
+    limits:
+      nvidia.com/gpu: "1"
+    requests:
+      nvidia.com/gpu: "1"
+    tolerations:
+      - key: "nvidia.com/gpu"
+        operator: "Equal"
+        value: "present"
+        effect: "NoSchedule"
   nvidia-gpu-a100-80gb:
     imageName: "nvidia-gpu"
     limits:

charts/models/values.yaml

@@ -161,6 +161,18 @@ catalog:
     # You can also use nvidia-gpu-a100-80gb:8
     resourceProfile: nvidia-gpu-h100:8
     targetRequests: 500
+  llama-3.1-70b-instruct-awq-int4-gh200:
+    enabled: false
+    features: [TextGeneration]
+    url: hf://hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4
+    engine: VLLM
+    args:
+      - --max-model-len=16384
+      - --max-num-batched-token=16384
+      - --enable-prefix-caching
+      - --disable-log-requests
+    resourceProfile: nvidia-gpu-gh200:1
+    targetRequests: 50
   llama-3.1-405b-instruct-fp8-a100-80b:
     features: [TextGeneration]
     url: hf://neuralmagic/Meta-Llama-3.1-405B-Instruct-FP8

manifests/models/llama-3.1-70b-instruct-awq-int4-gh200.yaml

@@ -0,0 +1,17 @@
+apiVersion: kubeai.org/v1
+kind: Model
+metadata:
+  name: llama-3.1-70b-instruct-awq-int4-gh200
+spec:
+  features: [TextGeneration]
+  owner:
+  url: hf://hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4
+  engine: VLLM
+  args:
+    - --max-model-len=16384
+    - --max-num-batched-token=16384
+    - --enable-prefix-caching
+    - --disable-log-requests
+  targetRequests: 50
+  minReplicas: 1
+  resourceProfile: nvidia-gpu-gh200:1