Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fetch the cached weights for Mistral-7B-Instruct-v0.1 from GCS bucket #606

Merged
merged 18 commits into from
Apr 26, 2024
Merged

Fetch the cached weights for Mistral-7B-Instruct-v0.1 from GCS bucket #606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
db1a4e6
Fetch the cached weights for Mistral-7B-Instruct-v0.1 from GCS bucket
gongmax Apr 23, 2024
1d8c806
gongmax Apr 23, 2024
ebc3cc0
Increase the timeout for waiting pod ready
gongmax Apr 24, 2024
6e3b5f7
try to fix long setup
gongmax Apr 24, 2024
b7bafbd
download the cached weights directly in init container instead of usi…
gongmax Apr 25, 2024
6cccc17
fix
gongmax Apr 25, 2024
ea31c36
fix
gongmax Apr 25, 2024
6aed71f
fix terrform format
gongmax Apr 25, 2024
47d8bdf
Increase the ephemeral storage for gpu pool; use smaller image for gs…
gongmax Apr 25, 2024
eba2f56
Increase ephemeral storage for testing cluster
gongmax Apr 25, 2024
3bfdbda
Increase terraform timeout for deployment mistral-7b-instruct
gongmax Apr 25, 2024
9bdef3e
fix
gongmax Apr 25, 2024
2c07e67
fix format
gongmax Apr 25, 2024
c3e8ac1
Also increase progress_deadline_seconds for deployment mistral-7b-ins…
gongmax Apr 25, 2024
a4ed9e9
fix format
gongmax Apr 25, 2024
bc782b3
fix comment
gongmax Apr 26, 2024
27bc5a6
fix comment
gongmax Apr 26, 2024
928da5d
fix comment
gongmax Apr 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions applications/rag/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,7 @@ variable "gpu_pools" {
max_count = optional(number, 3)
local_ssd_count = optional(number, 0)
spot = optional(bool, false)
disk_size_gb = optional(number, 100)
disk_size_gb = optional(number, 200)
disk_type = optional(string, "pd-standard")
image_type = optional(string, "COS_CONTAINERD")
enable_gcfs = optional(bool, false)
Expand All @@ -399,7 +399,7 @@ variable "gpu_pools" {
autoscaling = true
min_count = 1
max_count = 3
disk_size_gb = 100
disk_size_gb = 200
disk_type = "pd-balanced"
enable_gcfs = true
accelerator_count = 2
Expand Down
2 changes: 1 addition & 1 deletion infrastructure/tfvars_tests/standard-gke-public.platform.tfvars
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ gpu_pools = [{
min_count = 2
max_count = 3
accelerator_count = 2
disk_size_gb = 100
disk_size_gb = 200
enable_gcfs = true
logging_variant = "DEFAULT"
disk_type = "pd-balanced"
Expand Down
33 changes: 31 additions & 2 deletions tutorials-and-examples/hf-tgi/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ resource "kubernetes_service" "inference_service" {
}

resource "kubernetes_deployment" "inference_deployment" {
timeouts {
create = "30m"
roberthbailey marked this conversation as resolved.
Show resolved Hide resolved
}
metadata {
name = "mistral-7b-instruct"
namespace = var.namespace
Expand All @@ -56,7 +59,11 @@ resource "kubernetes_deployment" "inference_deployment" {
}

spec {
replicas = 1
# It takes more than 10m for the deployment to be ready on Autopilot cluster
# Set the progress deadline to 30m to avoid the deployment controller
# considering the deployment to be failed
progress_deadline_seconds = 1800
gongmax marked this conversation as resolved.
Show resolved Hide resolved
replicas = 1

selector {
match_labels = merge({
Expand All @@ -72,6 +79,15 @@ resource "kubernetes_deployment" "inference_deployment" {
}

spec {
init_container {
name = "download-model"
image = "google/cloud-sdk:473.0.0-alpine"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gongmax if possible it would be good to use an image hosted on GCR or AR.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will investigate and address in a follow up PR

command = ["gsutil", "cp", "-r", "gs://vertex-model-garden-public-us/mistralai/Mistral-7B-Instruct-v0.1/", "/model-data/"]
volume_mount {
mount_path = "/model-data"
name = "model-storage"
}
}
container {
image = "ghcr.io/huggingface/text-generation-inference:1.1.0"
name = "mistral-7b-instruct"
Expand All @@ -82,9 +98,11 @@ resource "kubernetes_deployment" "inference_deployment" {
protocol = "TCP"
}

args = ["--model-id", "$(MODEL_ID)"]

env {
name = "MODEL_ID"
value = "mistralai/Mistral-7B-Instruct-v0.1"
value = "/model/Mistral-7B-Instruct-v0.1"
}

env {
Expand Down Expand Up @@ -118,6 +136,12 @@ resource "kubernetes_deployment" "inference_deployment" {
name = "data"
}

volume_mount {
mount_path = "/model"
name = "model-storage"
read_only = "true"
}

#liveness_probe {
#http_get {
#path = "/"
Expand Down Expand Up @@ -146,6 +170,11 @@ resource "kubernetes_deployment" "inference_deployment" {
empty_dir {}
}

volume {
name = "model-storage"
empty_dir {}
}

node_selector = merge({
"cloud.google.com/gke-accelerator" = "nvidia-l4"
}, var.autopilot_cluster ? {
Expand Down