-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
dashboard/app: introduce batch_reproexport
This PR exports the latest reproducer for every bug. Reproducers are exported to the "bug_id/repro_id.c" files. This approach allows to add some metadata files or export more reproducers/bug later. All the files are then archived and uploaded to the preconfigured location.
- Loading branch information
1 parent
6959534
commit 1f6a60b
Showing
4 changed files
with
155 additions
and
100 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| // Copyright 2017 syzkaller project authors. All rights reserved. | ||
| // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "net/http" | ||
|
|
||
| "cloud.google.com/go/batch/apiv1" | ||
| "cloud.google.com/go/batch/apiv1/batchpb" | ||
| "github.com/google/uuid" | ||
| "google.golang.org/appengine/v2/log" | ||
| "google.golang.org/protobuf/types/known/durationpb" | ||
| ) | ||
|
|
||
| func initBatchProcessors() { | ||
| http.HandleFunc("/cron/batch_coverage", handleBatchCoverage) | ||
| http.HandleFunc("/cron/batch_reproexport", handleBatchReproExport) | ||
| } | ||
|
|
||
| // from https://cloud.google.com/batch/docs/samples/batch-create-script-job | ||
| func createScriptJob(ctx context.Context, projectID, jobNamePrefix, script string, | ||
| timeout int64, sa *batchpb.ServiceAccount) error { | ||
| region := "us-central1" | ||
| jobName := fmt.Sprintf("%s-%s", jobNamePrefix, uuid.New().String()) | ||
|
|
||
| batchClient, err := batch.NewClient(ctx) | ||
| if err != nil { | ||
| return fmt.Errorf("failed NewClient: %w", err) | ||
| } | ||
| defer batchClient.Close() | ||
|
|
||
| taskGroups := []*batchpb.TaskGroup{ | ||
| { | ||
| TaskSpec: &batchpb.TaskSpec{ | ||
| Runnables: []*batchpb.Runnable{{ | ||
| Executable: &batchpb.Runnable_Script_{ | ||
| Script: &batchpb.Runnable_Script{Command: &batchpb.Runnable_Script_Text{ | ||
| Text: script, | ||
| }}, | ||
| }, | ||
| }}, | ||
| ComputeResource: &batchpb.ComputeResource{ | ||
| // CpuMilli is milliseconds per cpu-second. This means the task requires 2 whole CPUs. | ||
| CpuMilli: 4000, | ||
| MemoryMib: 12 * 1024, | ||
| }, | ||
| MaxRunDuration: &durationpb.Duration{ | ||
| Seconds: timeout, | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| // Policies are used to define on what kind of virtual machines the tasks will run on. | ||
| // In this case, we tell the system to use "e2-standard-4" machine type. | ||
| // Read more about machine types here: https://cloud.google.com/compute/docs/machine-types | ||
| allocationPolicy := &batchpb.AllocationPolicy{ | ||
| Instances: []*batchpb.AllocationPolicy_InstancePolicyOrTemplate{{ | ||
| PolicyTemplate: &batchpb.AllocationPolicy_InstancePolicyOrTemplate_Policy{ | ||
| Policy: &batchpb.AllocationPolicy_InstancePolicy{ | ||
| ProvisioningModel: batchpb.AllocationPolicy_SPOT, | ||
| MachineType: "c3-standard-4", | ||
| }, | ||
| }, | ||
| }}, | ||
| ServiceAccount: sa, | ||
| } | ||
|
|
||
| logsPolicy := &batchpb.LogsPolicy{ | ||
| Destination: batchpb.LogsPolicy_CLOUD_LOGGING, | ||
| } | ||
|
|
||
| // The job's parent is the region in which the job will run. | ||
| parent := fmt.Sprintf("projects/%s/locations/%s", projectID, region) | ||
|
|
||
| job := batchpb.Job{ | ||
| TaskGroups: taskGroups, | ||
| AllocationPolicy: allocationPolicy, | ||
| LogsPolicy: logsPolicy, | ||
| } | ||
|
|
||
| req := &batchpb.CreateJobRequest{ | ||
| Parent: parent, | ||
| JobId: jobName, | ||
| Job: &job, | ||
| } | ||
|
|
||
| createdJob, err := batchClient.CreateJob(ctx, req) | ||
| if err != nil { | ||
| return fmt.Errorf("unable to create job: %w", err) | ||
| } | ||
|
|
||
| log.Infof(ctx, "job created: %v\n", createdJob) | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| // Copyright 2024 syzkaller project authors. All rights reserved. | ||
| // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "net/http" | ||
|
|
||
| "google.golang.org/appengine/v2" | ||
| "google.golang.org/appengine/v2/log" | ||
| ) | ||
|
|
||
| const exportTimeoutSeconds = 10000 * 2 // upstream has apx 7k reproducers, 1s each max (throttling) | ||
|
|
||
| func handleBatchReproExport(w http.ResponseWriter, r *http.Request) { | ||
| ctx := appengine.NewContext(r) | ||
| reproExportPath := getConfig(ctx).ReproExportPath | ||
| if reproExportPath == "" { | ||
| return | ||
| } | ||
| if err := createScriptJob(ctx, "syzkaller", "export-repro", | ||
| exportReproScript(reproExportPath), | ||
| exportTimeoutSeconds, nil); err != nil { | ||
| log.Errorf(ctx, "createScriptJob: %s", err.Error()) | ||
| } | ||
| } | ||
|
|
||
| func exportReproScript(archivePath string) string { | ||
| script := "\n" + | ||
| // "git clone --depth 1 --branch master --single-branch https://github.com/google/syzkaller\n" + | ||
| "git clone https://github.com/tarasmadan/syzkaller\n" + | ||
| "cd syzkaller\n" + | ||
| "git checkout syz_reprolist_by_namespace\n" + | ||
| "export CI=1\n" + | ||
| "./tools/syz-env \"" + | ||
| "go run ./tools/syz-reprolist/... -namespace upstream; " + | ||
| "tar -czvf reproducers.tar.gz ./repros/; " + | ||
| "gsutil -m cp reproducers.tar.gz " + archivePath + ";" + | ||
| "\"" | ||
| return script | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters