diff --git a/dashboard/app/coverage_batch.go b/dashboard/app/batch_coverage.go similarity index 59% rename from dashboard/app/coverage_batch.go rename to dashboard/app/batch_coverage.go index 505fbe00d572..a91b9a3b8455 100644 --- a/dashboard/app/coverage_batch.go +++ b/dashboard/app/batch_coverage.go @@ -9,26 +9,16 @@ import ( "net/http" "strconv" - "cloud.google.com/go/batch/apiv1" "cloud.google.com/go/batch/apiv1/batchpb" "cloud.google.com/go/bigquery" "cloud.google.com/go/civil" "github.com/google/syzkaller/pkg/coveragedb" - "github.com/google/uuid" "google.golang.org/api/iterator" "google.golang.org/appengine/v2" "google.golang.org/appengine/v2/log" - "google.golang.org/protobuf/types/known/durationpb" ) -func initCoverageBatches() { - http.HandleFunc("/cron/batch_coverage", handleBatchCoverage) -} - -const ( - daysToMerge = 7 - batchTimeoutSeconds = 60 * 60 * 6 -) +const batchCoverageTimeoutSeconds = 60 * 60 * 6 func handleBatchCoverage(w http.ResponseWriter, r *http.Request) { ctx := appengine.NewContext(r) @@ -76,21 +66,24 @@ func handleBatchCoverage(w http.ResponseWriter, r *http.Request) { } periods = coveragedb.AtMostNLatestPeriods(periods, maxSteps) nsCovConfig := nsConfig.Coverage - if err := createScriptJob( - ctx, - nsCovConfig.BatchProject, - nsCovConfig.BatchServiceAccount, - batchScript(ns, repo, branch, periods, + serviceAccount := &batchpb.ServiceAccount{ + Email: nsCovConfig.BatchServiceAccount, + Scopes: nsCovConfig.BatchScopes, + } + if err := createScriptJob(ctx, nsCovConfig.BatchProject, "coverage-merge", + batchCoverageScript(ns, repo, branch, periods, nsCovConfig.JobInitScript, nsCovConfig.SyzEnvInitScript, nsCovConfig.DashboardClientName), - nsCovConfig.BatchScopes); err != nil { - log.Errorf(ctx, "failed to batchScript: %s", err.Error()) + batchCoverageTimeoutSeconds, + serviceAccount, + ); err != nil { + log.Errorf(ctx, "failed to batchCoverageScript: %s", err.Error()) } } } -func batchScript(ns, repo, branch string, periods []coveragedb.TimePeriod, +func batchCoverageScript(ns, repo, branch string, periods []coveragedb.TimePeriod, jobInitScript, syzEnvInitScript, clientName string) string { if clientName == "" { clientName = defaultDashboardClientName @@ -118,86 +111,6 @@ func batchScript(ns, repo, branch string, periods []coveragedb.TimePeriod, return script } -// from https://cloud.google.com/batch/docs/samples/batch-create-script-job -func createScriptJob(ctx context.Context, projectID, serviceAccount, script string, scopes []string) error { - region := "us-central1" - jobName := fmt.Sprintf("coverage-merge-%s", uuid.New().String()) - - batchClient, err := batch.NewClient(ctx) - if err != nil { - return fmt.Errorf("failed NewClient: %w", err) - } - defer batchClient.Close() - - taskGroups := []*batchpb.TaskGroup{ - { - TaskSpec: &batchpb.TaskSpec{ - Runnables: []*batchpb.Runnable{{ - Executable: &batchpb.Runnable_Script_{ - Script: &batchpb.Runnable_Script{Command: &batchpb.Runnable_Script_Text{ - Text: script, - }}, - }, - }}, - ComputeResource: &batchpb.ComputeResource{ - // CpuMilli is milliseconds per cpu-second. This means the task requires 2 whole CPUs. - CpuMilli: 4000, - MemoryMib: 12 * 1024, - }, - MaxRunDuration: &durationpb.Duration{ - Seconds: batchTimeoutSeconds, - }, - }, - }, - } - - // Policies are used to define on what kind of virtual machines the tasks will run on. - // In this case, we tell the system to use "e2-standard-4" machine type. - // Read more about machine types here: https://cloud.google.com/compute/docs/machine-types - allocationPolicy := &batchpb.AllocationPolicy{ - Instances: []*batchpb.AllocationPolicy_InstancePolicyOrTemplate{{ - PolicyTemplate: &batchpb.AllocationPolicy_InstancePolicyOrTemplate_Policy{ - Policy: &batchpb.AllocationPolicy_InstancePolicy{ - ProvisioningModel: batchpb.AllocationPolicy_SPOT, - MachineType: "c3-standard-4", - }, - }, - }}, - ServiceAccount: &batchpb.ServiceAccount{ - Email: serviceAccount, - Scopes: scopes, - }, - } - - logsPolicy := &batchpb.LogsPolicy{ - Destination: batchpb.LogsPolicy_CLOUD_LOGGING, - } - - // The job's parent is the region in which the job will run. - parent := fmt.Sprintf("projects/%s/locations/%s", projectID, region) - - job := batchpb.Job{ - TaskGroups: taskGroups, - AllocationPolicy: allocationPolicy, - LogsPolicy: logsPolicy, - } - - req := &batchpb.CreateJobRequest{ - Parent: parent, - JobId: jobName, - Job: &job, - } - - createdJob, err := batchClient.CreateJob(ctx, req) - if err != nil { - return fmt.Errorf("unable to create job: %w", err) - } - - log.Infof(ctx, "job created: %v\n", createdJob) - - return nil -} - func nsDataAvailable(ctx context.Context, ns string) ([]coveragedb.TimePeriod, []int64, error) { client, err := bigquery.NewClient(ctx, "syzkaller") if err != nil { diff --git a/dashboard/app/batch_main.go b/dashboard/app/batch_main.go new file mode 100644 index 000000000000..acf37ee8f0cd --- /dev/null +++ b/dashboard/app/batch_main.go @@ -0,0 +1,99 @@ +// Copyright 2017 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "context" + "fmt" + "net/http" + + "cloud.google.com/go/batch/apiv1" + "cloud.google.com/go/batch/apiv1/batchpb" + "github.com/google/uuid" + "google.golang.org/appengine/v2/log" + "google.golang.org/protobuf/types/known/durationpb" +) + +func initBatchProcessors() { + http.HandleFunc("/cron/batch_coverage", handleBatchCoverage) + http.HandleFunc("/cron/batch_reproexport", handleBatchReproExport) +} + +// from https://cloud.google.com/batch/docs/samples/batch-create-script-job +func createScriptJob(ctx context.Context, projectID, jobNamePrefix, script string, + timeout int64, sa *batchpb.ServiceAccount) error { + region := "us-central1" + jobName := fmt.Sprintf("%s-%s", jobNamePrefix, uuid.New().String()) + + batchClient, err := batch.NewClient(ctx) + if err != nil { + return fmt.Errorf("failed NewClient: %w", err) + } + defer batchClient.Close() + + taskGroups := []*batchpb.TaskGroup{ + { + TaskSpec: &batchpb.TaskSpec{ + Runnables: []*batchpb.Runnable{{ + Executable: &batchpb.Runnable_Script_{ + Script: &batchpb.Runnable_Script{Command: &batchpb.Runnable_Script_Text{ + Text: script, + }}, + }, + }}, + ComputeResource: &batchpb.ComputeResource{ + // CpuMilli is milliseconds per cpu-second. This means the task requires 2 whole CPUs. + CpuMilli: 4000, + MemoryMib: 12 * 1024, + }, + MaxRunDuration: &durationpb.Duration{ + Seconds: timeout, + }, + }, + }, + } + + // Policies are used to define on what kind of virtual machines the tasks will run on. + // In this case, we tell the system to use "e2-standard-4" machine type. + // Read more about machine types here: https://cloud.google.com/compute/docs/machine-types + allocationPolicy := &batchpb.AllocationPolicy{ + Instances: []*batchpb.AllocationPolicy_InstancePolicyOrTemplate{{ + PolicyTemplate: &batchpb.AllocationPolicy_InstancePolicyOrTemplate_Policy{ + Policy: &batchpb.AllocationPolicy_InstancePolicy{ + ProvisioningModel: batchpb.AllocationPolicy_SPOT, + MachineType: "c3-standard-4", + }, + }, + }}, + ServiceAccount: sa, + } + + logsPolicy := &batchpb.LogsPolicy{ + Destination: batchpb.LogsPolicy_CLOUD_LOGGING, + } + + // The job's parent is the region in which the job will run. + parent := fmt.Sprintf("projects/%s/locations/%s", projectID, region) + + job := batchpb.Job{ + TaskGroups: taskGroups, + AllocationPolicy: allocationPolicy, + LogsPolicy: logsPolicy, + } + + req := &batchpb.CreateJobRequest{ + Parent: parent, + JobId: jobName, + Job: &job, + } + + createdJob, err := batchClient.CreateJob(ctx, req) + if err != nil { + return fmt.Errorf("unable to create job: %w", err) + } + + log.Infof(ctx, "job created: %v\n", createdJob) + + return nil +} diff --git a/dashboard/app/batch_reproexport.go b/dashboard/app/batch_reproexport.go new file mode 100644 index 000000000000..8926f9fa0d5c --- /dev/null +++ b/dashboard/app/batch_reproexport.go @@ -0,0 +1,41 @@ +// Copyright 2024 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "net/http" + + "google.golang.org/appengine/v2" + "google.golang.org/appengine/v2/log" +) + +const exportTimeoutSeconds = 10000 * 2 // upstream has apx 7k reproducers, 1s each max (throttling) + +func handleBatchReproExport(w http.ResponseWriter, r *http.Request) { + ctx := appengine.NewContext(r) + reproExportPath := getConfig(ctx).ReproExportPath + if reproExportPath == "" { + return + } + if err := createScriptJob(ctx, "syzkaller", "export-repro", + exportReproScript(reproExportPath), + exportTimeoutSeconds, nil); err != nil { + log.Errorf(ctx, "createScriptJob: %s", err.Error()) + } +} + +func exportReproScript(archivePath string) string { + script := "\n" + + // "git clone --depth 1 --branch master --single-branch https://github.com/google/syzkaller\n" + + "git clone https://github.com/tarasmadan/syzkaller\n" + + "cd syzkaller\n" + + "git checkout syz_reprolist_by_namespace\n" + + "export CI=1\n" + + "./tools/syz-env \"" + + "go run ./tools/syz-reprolist/... -namespace upstream; " + + "tar -czvf reproducers.tar.gz ./repros/; " + + "gsutil -m cp reproducers.tar.gz " + archivePath + ";" + + "\"" + return script +} diff --git a/dashboard/app/config.go b/dashboard/app/config.go index 94602057213f..44a68f28c5ea 100644 --- a/dashboard/app/config.go +++ b/dashboard/app/config.go @@ -64,6 +64,8 @@ type GlobalConfig struct { DiscussionEmails []DiscussionEmailConfig // Incoming request throttling. Throttle ThrottleConfig + // Reproducers export path. + ReproExportPath string } // Per-namespace config. @@ -392,7 +394,7 @@ func installConfig(cfg *GlobalConfig) { initHTTPHandlers() initAPIHandlers() initKcidb() - initCoverageBatches() + initBatchProcessors() } var contextConfigKey = "Updated config (to be used during tests). Use only in tests!"