-
Notifications
You must be signed in to change notification settings - Fork 154
/
builders.go
2337 lines (2241 loc) · 78.4 KB
/
builders.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package dashboard contains shared configuration and logic used by various
// pieces of the Go continuous build system.
package dashboard
import (
"fmt"
"os"
"sort"
"strconv"
"strings"
"time"
"golang.org/x/build/buildenv"
"golang.org/x/build/maintner/maintnerd/maintapi/version"
"golang.org/x/build/types"
)
// Builders are the different build configurations.
// The keys are like "darwin-amd64" or "linux-386-387".
// This map should not be modified by other packages.
// Initialization happens below, via calls to addBuilder.
var Builders = map[string]*BuildConfig{}
// Hosts contains the names and configs of all the types of
// buildlets. They can be VMs, containers, or dedicated machines.
var Hosts = map[string]*HostConfig{
"host-linux-jessie": &HostConfig{
Notes: "Debian Jessie, our standard Linux container image.",
ContainerImage: "linux-x86-jessie:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-stretch": &HostConfig{
Notes: "Debian Stretch",
ContainerImage: "linux-x86-stretch:latest",
machineType: "n1-standard-4", // 4 vCPUs, 15 GB mem
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-stretch-morecpu": &HostConfig{
Notes: "Debian Stretch, but on n1-highcpu-8",
ContainerImage: "linux-x86-stretch:latest",
machineType: "n1-highcpu-16", // 16 vCPUs, 14.4 GB mem
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-stretch-vmx": &HostConfig{
Notes: "Debian Stretch w/ Nested Virtualization (VMX CPU bit) enabled, for testing",
ContainerImage: "linux-x86-stretch:latest",
NestedVirt: true,
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-armhf-cross": &HostConfig{
Notes: "Debian Jessie with armhf cross-compiler, built from env/crosscompile/linux-armhf-jessie",
ContainerImage: "linux-armhf-jessie:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
"host-linux-armel-cross": &HostConfig{
Notes: "Debian Jessie with armel cross-compiler, from env/crosscompile/linux-armel-stretch",
ContainerImage: "linux-armel-stretch:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
"host-linux-amd64-localdev": &HostConfig{
IsReverse: true,
ExpectNum: 0,
Notes: "for localhost development of buildlets/gomote/coordinator only",
SSHUsername: os.Getenv("USER"),
},
"host-nacl-kube": &HostConfig{
Notes: "Container with Native Client binaries.",
ContainerImage: "linux-x86-nacl:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
"host-js-wasm": &HostConfig{
Notes: "Container with node.js for testing js/wasm.",
ContainerImage: "js-wasm:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
"host-s390x-cross-kube": &HostConfig{
Notes: "Container with s390x cross-compiler.",
ContainerImage: "linux-s390x-stretch:latest",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
"host-linux-x86-alpine": &HostConfig{
Notes: "Alpine container",
ContainerImage: "linux-x86-alpine:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64-static",
env: []string{"GOROOT_BOOTSTRAP=/usr/lib/go"},
},
"host-linux-clang": &HostConfig{
Notes: "Container with clang.",
ContainerImage: "linux-x86-clang:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
"host-linux-sid": &HostConfig{
Notes: "Debian sid, updated occasionally.",
ContainerImage: "linux-x86-sid:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
"host-linux-fedora": &HostConfig{
Notes: "Fedora 30",
ContainerImage: "linux-x86-fedora:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/goboot"},
},
"host-linux-arm-scaleway": &HostConfig{
IsReverse: true,
HermeticReverse: true,
ExpectNum: 50,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go"},
ReverseAliases: []string{"linux-arm", "linux-arm-arm5"},
SSHUsername: "root",
},
"host-linux-arm5spacemonkey": &HostConfig{
IsReverse: true,
ExpectNum: 3,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go"},
ReverseAliases: []string{"linux-arm-arm5spacemonkey"},
OwnerGithub: "zeebo",
},
"host-openbsd-amd64-60": &HostConfig{
VMImage: "openbsd-amd64-60",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-amd64-60.tar.gz",
Notes: "OpenBSD 6.0; GCE VM is built from script in build/env/openbsd-amd64",
SSHUsername: "gopher",
},
"host-openbsd-386-60": &HostConfig{
VMImage: "openbsd-386-60",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-386",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-386-60.tar.gz",
Notes: "OpenBSD 6.0; GCE VM is built from script in build/env/openbsd-386",
SSHUsername: "gopher",
},
"host-openbsd-amd64-62": &HostConfig{
VMImage: "openbsd-amd64-62",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-amd64-go1_12.tar.gz",
Notes: "OpenBSD 6.2; GCE VM is built from script in build/env/openbsd-amd64",
SSHUsername: "gopher",
},
"host-openbsd-386-62": &HostConfig{
VMImage: "openbsd-386-62-a",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-386",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-386-go1_12.tar.gz",
Notes: "OpenBSD 6.2; GCE VM is built from script in build/env/openbsd-386",
SSHUsername: "gopher",
},
"host-openbsd-amd64-64": &HostConfig{
VMImage: "openbsd-amd64-64-190129a",
MinCPUPlatform: "Intel Skylake", // for better TSC? Maybe? see Issue 29223. builds faster at least.
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-amd64-64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-amd64-go1_12.tar.gz",
Notes: "OpenBSD 6.4 with hw.smt=1; GCE VM is built from script in build/env/openbsd-amd64",
SSHUsername: "gopher",
},
"host-openbsd-386-64": &HostConfig{
VMImage: "openbsd-386-64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-386-64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-386-go1_12.tar.gz",
Notes: "OpenBSD 6.4; GCE VM is built from script in build/env/openbsd-386",
SSHUsername: "gopher",
},
"host-freebsd-93-gce": &HostConfig{
VMImage: "freebsd-amd64-gce93",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-freebsd-10_3": &HostConfig{
VMImage: "freebsd-amd64-103-b",
Notes: "FreeBSD 10.3; GCE VM is built from script in build/env/freebsd-amd64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64", // TODO(bradfitz): why was this http instead of https?
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
env: []string{"CC=clang"},
SSHUsername: "gopher",
},
"host-freebsd-10_4": &HostConfig{
VMImage: "freebsd-amd64-104",
Notes: "FreeBSD 10.4; GCE VM is built from script in build/env/freebsd-amd64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-freebsd-11_1": &HostConfig{
VMImage: "freebsd-amd64-111-b",
Notes: "FreeBSD 11.1; GCE VM is built from script in build/env/freebsd-amd64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64", // TODO(bradfitz): why was this http instead of https?
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
env: []string{"CC=clang"},
SSHUsername: "gopher",
},
"host-freebsd-11_2": &HostConfig{
VMImage: "freebsd-amd64-112",
Notes: "FreeBSD 11.2; GCE VM is built from script in build/env/freebsd-amd64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-freebsd-12_0": &HostConfig{
VMImage: "freebsd-amd64-120-v1",
Notes: "FreeBSD 12.0; GCE VM is built from script in build/env/freebsd-amd64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-netbsd-amd64-8_0": &HostConfig{
VMImage: "netbsd-amd64-8-0-2018q1",
Notes: "NetBSD 8.0RC1; GCE VM is built from script in build/env/netbsd-amd64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.netbsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-netbsd-amd64-2da6b33.tar.gz",
SSHUsername: "root",
},
// Note: the netbsd-386 host hangs during the ../test phase of all.bash,
// so we don't use this for now. (See the netbsd-386-8 BuildConfig below.)
"host-netbsd-386-8_0": &HostConfig{
VMImage: "netbsd-i386-8-0-2018q1",
Notes: "NetBSD 8.0RC1; GCE VM is built from script in build/env/netbsd-386",
machineType: "n1-highcpu-4",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.netbsd-386",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-netbsd-386-0b3b511.tar.gz",
SSHUsername: "root",
},
"host-netbsd-arm-bsiegert": &HostConfig{
IsReverse: true,
ExpectNum: 1,
env: []string{"GOROOT_BOOTSTRAP=/usr/pkg/go112"},
OwnerGithub: "bsiegert",
},
"host-dragonfly-amd64-tdfbsd": &HostConfig{
IsReverse: true,
ExpectNum: 1,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go"},
ReverseAliases: []string{"dragonfly-amd64"},
OwnerGithub: "tdfbsd",
},
"host-freebsd-arm-paulzhol": &HostConfig{
IsReverse: true,
ExpectNum: 1,
Notes: "Cubiboard2 1Gb RAM dual-core Cortex-A7 (Allwinner A20), FreeBSD 11.1-RELEASE",
env: []string{"GOROOT_BOOTSTRAP=/usr/home/paulzhol/go1.4"},
ReverseAliases: []string{"freebsd-arm-paulzhol"},
OwnerGithub: "paulzhol",
},
"host-plan9-arm-0intro": &HostConfig{
IsReverse: true,
ExpectNum: 1,
Notes: "Raspberry Pi 3 Model B, Plan 9 from Bell Labs",
OwnerGithub: "0intro",
},
"host-plan9-amd64-0intro": &HostConfig{
IsReverse: true,
ExpectNum: 1,
OwnerGithub: "0intro",
},
"host-plan9-386-0intro": &HostConfig{
IsReverse: true,
ExpectNum: 1,
OwnerGithub: "0intro",
},
"host-plan9-386-gce": &HostConfig{
VMImage: "plan9-386-v7",
Notes: "Plan 9 from 0intro; GCE VM is built from script in build/env/plan9-386",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.plan9-386",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-plan9-386.tar.gz",
// We *were* using n1-standard-1 because Plan 9 can only
// reliably use a single CPU. Using 2 or 4 and we see
// test failures. See:
// https://golang.org/issue/8393
// https://golang.org/issue/9491
// n1-standard-1 has 3.6 GB of memory which WAS (see below)
// overkill (userspace probably only sees 2GB anyway),
// but it's the cheapest option. And plenty to keep
// our ~250 MB of inputs+outputs in its ramfs.
//
// But the docs says "For the n1 series of machine
// types, a virtual CPU is implemented as a single
// hyperthread on a 2.6GHz Intel Sandy Bridge Xeon or
// Intel Ivy Bridge Xeon (or newer) processor. This
// means that the n1-standard-2 machine type will see
// a whole physical core."
//
// ... so we used n1-highcpu-2 (1.80 RAM, still
// plenty), just so we can get 1 whole core for the
// single-core Plan 9. It will see 2 virtual cores and
// only use 1, but we hope that 1 will be more powerful
// and we'll stop timing out on tests.
machineType: "n1-highcpu-4",
env: []string{"GO_TEST_TIMEOUT_SCALE=3"},
},
"host-windows-amd64-2008": &HostConfig{
VMImage: "windows-amd64-server-2008r2-v7",
machineType: "n1-highcpu-4",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.windows-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-windows-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-windows-amd64-2012": &HostConfig{
VMImage: "windows-amd64-server-2012r2-v7",
machineType: "n1-highcpu-4",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.windows-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-windows-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-windows-amd64-2016": &HostConfig{
VMImage: "windows-amd64-server-2016-v7",
machineType: "n1-highcpu-4",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.windows-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-windows-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-windows-arm-iotcore": &HostConfig{
IsReverse: true,
ExpectNum: 1,
OwnerGithub: "jordanrh1",
env: []string{"GOROOT_BOOTSTRAP=C:\\Data\\Go"},
},
"host-darwin-10_8": &HostConfig{
IsReverse: true,
ExpectNum: 0,
Notes: "MacStadium OS X 10.8 VM under VMWare ESXi",
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/go1.4",
},
ReverseAliases: []string{"darwin-amd64-10_8"},
SSHUsername: "gopher",
HermeticReverse: false, // TODO: make it so, like 10.12
},
"host-darwin-10_10": &HostConfig{
IsReverse: true,
ExpectNum: 3,
Notes: "MacStadium OS X 10.10 VM under VMWare ESXi",
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/go1.4",
},
ReverseAliases: []string{"darwin-amd64-10_10"},
SSHUsername: "gopher",
HermeticReverse: false, // TODO: make it so, like 10.12
},
"host-darwin-10_11": &HostConfig{
IsReverse: true,
ExpectNum: 7,
Notes: "MacStadium OS X 10.11 VM under VMWare ESXi",
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/go1.4",
},
ReverseAliases: []string{"darwin-amd64-10_11"},
SSHUsername: "gopher",
HermeticReverse: false, // TODO: make it so, like 10.12
},
"host-darwin-10_12": &HostConfig{
IsReverse: true,
ExpectNum: 3,
Notes: "MacStadium OS X 10.12 VM under VMWare ESXi",
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/go1.4",
},
ReverseAliases: []string{"darwin-amd64-10_12"},
SSHUsername: "gopher",
HermeticReverse: true, // we destroy the VM when done & let cmd/makemac recreate
},
"host-darwin-10_14": &HostConfig{
IsReverse: true,
ExpectNum: 7,
Notes: "MacStadium macOS Mojave (10.14) VM under VMWare ESXi",
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/goboot", // Go 1.12.1
},
SSHUsername: "gopher",
HermeticReverse: true, // we destroy the VM when done & let cmd/makemac recreate
},
"host-linux-s390x": &HostConfig{
Notes: "run by IBM",
OwnerGithub: "mundaym",
IsReverse: true,
env: []string{"GOROOT_BOOTSTRAP=/var/buildlet/go-linux-s390x-bootstrap"},
ReverseAliases: []string{"linux-s390x-ibm"},
},
"host-linux-ppc64-osu": &HostConfig{
Notes: "Debian jessie; run by Go team on osuosl.org",
IsReverse: true,
ExpectNum: 5,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
ReverseAliases: []string{"linux-ppc64-buildlet"},
SSHUsername: "debian",
HermeticReverse: false, // TODO: use rundockerbuildlet like arm64
},
"host-linux-ppc64le-osu": &HostConfig{
Notes: "Debian jessie; run by Go team on osuosl.org",
IsReverse: true,
ExpectNum: 5,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
ReverseAliases: []string{"linux-ppc64le-buildlet"},
SSHUsername: "debian",
HermeticReverse: false, // TODO: use rundockerbuildlet like arm64
},
"host-linux-ppc64le-power9-osu": &HostConfig{
Notes: "Debian jessie; run by IBM",
OwnerGithub: "ceseo",
IsReverse: true,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
ReverseAliases: []string{"linux-ppc64le-power9osu"},
SSHUsername: "debian",
HermeticReverse: false, // TODO: use rundockerbuildlet like arm64
},
"host-linux-arm64-packet": &HostConfig{
Notes: "On 96 core packet.net host (Xenial) in Docker containers (Jessie); run by Go team. See x/build/env/linux-arm64/packet",
IsReverse: true,
HermeticReverse: true,
ExpectNum: 20,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
SSHUsername: "root",
},
"host-solaris-amd64": &HostConfig{
Notes: "run by Go team on Joyent, on a SmartOS 'infrastructure container'",
IsReverse: true,
ExpectNum: 5,
env: []string{"GOROOT_BOOTSTRAP=/root/go-solaris-amd64-bootstrap", "HOME=/root"},
ReverseAliases: []string{"solaris-amd64-smartosbuildlet"},
},
"host-illumos-amd64-joyent": &HostConfig{
Notes: "run by Go team on Joyent, on a SmartOS 'infrastructure container'",
IsReverse: true,
ExpectNum: 1,
env: []string{
"GOROOT_BOOTSTRAP=/root/goboot",
"HOME=/root",
"PATH=/usr/sbin:/usr/bin:/opt/local/bin", // gcc is in /opt/local/bin
},
},
"host-solaris-oracle-amd64-oraclerel": &HostConfig{
Notes: "Oracle Solaris amd64 Release System",
Owner: "", // TODO: find current owner
OwnerGithub: "", // TODO: find current owner
IsReverse: true,
ExpectNum: 1,
env: []string{"GOROOT_BOOTSTRAP=/opt/golang/go-solaris-amd64-bootstrap"},
},
"host-linux-mipsle-mengzhuo": &HostConfig{
Notes: "Loongson 3A Box hosted by Meng Zhuo",
OwnerGithub: "mengzhuo",
IsReverse: true,
ExpectNum: 1,
env: []string{
"GOROOT_BOOTSTRAP=/usr/lib/golang",
"GOMIPS64=hardfloat",
},
},
"host-darwin-amd64-zenly-ios": &HostConfig{
Notes: "MacBook Pro hosted by Zenly, running the ios reverse buildlet",
OwnerGithub: "znly",
IsReverse: true,
ExpectNum: 1,
env: []string{
"GOROOT_BOOTSTRAP=/usr/local/Cellar/1.10.3/libexec",
"GOHOSTARCH=amd64",
},
},
"host-darwin-arm64-corellium-ios": &HostConfig{
Notes: "Virtual iOS devices hosted by Zenly on Corellium",
OwnerGithub: "znly",
IsReverse: true,
ExpectNum: 3,
env: []string{
"GOROOT_BOOTSTRAP=/var/mobile/go-darwin-arm64-bootstrap",
},
},
"host-android-arm64-corellium-android": &HostConfig{
Notes: "Virtual Android devices hosted by Zenly on Corellium",
OwnerGithub: "znly",
IsReverse: true,
ExpectNum: 3,
env: []string{
"GOROOT_BOOTSTRAP=/data/data/com.termux/files/home/go-android-arm64-bootstrap",
},
},
"host-aix-ppc64-osuosl": &HostConfig{
Notes: "AIX 7.2 VM on OSU; run by Tony Reix",
OwnerGithub: "trex58",
IsReverse: true,
ExpectNum: 1,
env: []string{"GOROOT_BOOTSTRAP=/opt/freeware/lib/golang"},
},
"host-android-amd64-emu": &HostConfig{
Notes: "Debian Buster w/ Android SDK + emulator (use nested virt)",
ContainerImage: "android-amd64-emu:bff27c0c9263",
KonletVMImage: "android-amd64-emu",
NestedVirt: true,
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
}
func init() {
for key, c := range Hosts {
if key == "" {
panic("empty string key in Hosts")
}
if c.HostType == "" {
c.HostType = key
}
if c.HostType != key {
panic(fmt.Sprintf("HostType %q != key %q", c.HostType, key))
}
nSet := 0
if c.VMImage != "" {
nSet++
}
if c.ContainerImage != "" {
nSet++
}
if c.IsReverse {
nSet++
}
if nSet != 1 {
panic(fmt.Sprintf("exactly one of VMImage, ContainerImage, IsReverse must be set for host %q; got %v", key, nSet))
}
if c.buildletURLTmpl == "" && (c.VMImage != "" || c.ContainerImage != "") {
panic(fmt.Sprintf("missing buildletURLTmpl for host type %q", key))
}
}
}
// A HostConfig describes the available ways to obtain buildlets of
// different types. Some host configs can server multiple
// builders. For example, a host config of "host-linux-jessie" can
// serve linux-amd64, linux-amd64-race, linux-386, linux-386-387, etc.
type HostConfig struct {
// HostType is the unique name of this host config. It is also
// the key in the Hosts map.
HostType string
// buildletURLTmpl is the URL "template" ($BUCKET is auto-expanded)
// for the URL to the buildlet binary.
// This field is required for VM and Container builders. It's not
// needed for reverse buildlets because in that case, the buildlets
// are already running and their stage0 should know how to update it
// it automatically.
buildletURLTmpl string
// Exactly 1 of these must be set:
VMImage string // e.g. "openbsd-amd64-60"
ContainerImage string // e.g. "linux-buildlet-std:latest" (suffix after "gcr.io/<PROJ>/")
IsReverse bool // if true, only use the reverse buildlet pool
// GCE options, if VMImage != ""
machineType string // optional GCE instance type
RegularDisk bool // if true, use spinning disk instead of SSD
MinCPUPlatform string // optional; https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform
// ReverseOptions:
ExpectNum int // expected number of reverse buildlets of this type
HermeticReverse bool // whether reverse buildlet has fresh env per conn
// Container image options, if ContainerImage != "":
NestedVirt bool // container requires VMX nested virtualization
KonletVMImage string // optional VM image (containing konlet) to use instead of default
// Optional base env. GOROOT_BOOTSTRAP should go here if the buildlet
// has Go 1.4+ baked in somewhere.
env []string
// These template URLs may contain $BUCKET which is expanded to the
// relevant Cloud Storage bucket as specified by the build environment.
goBootstrapURLTmpl string // optional URL to a built Go 1.4+ tar.gz
Owner string // optional email of owner; "bradfitz@golang.org", empty means golang-dev
OwnerGithub string // optional GitHub username of owner
Notes string // notes for humans
SSHUsername string // username to ssh as, empty means not supported
// ReverseAliases lists alternate names for this buildlet
// config, for older clients doing a reverse dial into the
// coordinator from outside. This prevents us from updating
// 75+ dedicated machines/VMs atomically, switching them to
// the new "host-*" names.
// This is only applicable if IsReverse.
ReverseAliases []string
}
// A BuildConfig describes how to run a builder.
type BuildConfig struct {
// Name is the unique name of the builder, in the form of
// "GOOS-GOARCH" or "GOOS-GOARCH-suffix". For example,
// "darwin-386", "linux-386-387", "linux-amd64-race". Some
// suffixes are well-known and carry special meaning, such as
// "-race".
Name string
// HostType is the required key into the Hosts map, describing
// the type of host this build will run on.
// For example, "host-linux-jessie".
HostType string
Notes string // notes for humans
// tryBot optionally specifies a policy func for whether trybots are enabled.
// nil means off. Even if tryBot returns true, BuildConfig.BuildsRepo must also
// return true. See the implementation of BuildConfig.BuildsRepoTryBot.
// The proj is "go", "net", etc. The branch is proj's branch.
// The goBranch is the same as branch for proj "go", else it's the go branch
// ("master, "release-branch.go1.12", etc).
tryBot func(proj, branch, goBranch string) bool
tryOnly bool // only used for trybots, and not regular builds
CompileOnly bool // if true, compile tests, but don't run them
FlakyNet bool // network tests are flaky (try anyway, but ignore some failures)
// buildsRepo optionally specifies whether this
// builder does builds (of any type) for the given repo ("go",
// "net", etc) and its branch ("master", "release-branch.go1.12").
// If nil, a default policy is used. (see buildsRepoAtAll for details)
// goBranch is the branch of "go" to build against. If repo == "go",
// goBranch == branch.
buildsRepo func(repo, branch, goBranch string) bool
// MinimumGoVersion optionally specifies the minimum Go version
// this builder is allowed to use. It can be useful for skipping
// builders that are too new and no longer support some supported
// Go versions. It doesn't need to be set for builders that support
// all supported Go versions.
//
// Note: This field currently has effect on trybot runs only.
//
// TODO: unexport this and make buildsRepoAtAll return false on too-old
// of repos. The callers in coordinator will need updating.
MinimumGoVersion types.MajorMinor
// MaxAtOnce optionally specifies a cap of how many builds of
// this type can run at once. Zero means unlimited. This is a
// temporary measure until the build scheduler
// (golang.org/issue/19178) is done, at which point this field
// should be deleted.
MaxAtOnce int
// SkipSnapshot, if true, means to not fetch a tarball
// snapshot of the world post-make.bash from the buildlet (and
// thus to not write it to Google Cloud Storage). This is
// incompatible with sharded tests, and should only be used
// for very slow builders or networks, unable to transfer
// the tarball in under ~5 minutes.
SkipSnapshot bool
// RunBench causes the coordinator to run benchmarks on this buildlet type.
RunBench bool
// StopAfterMake causes the build to stop after the make
// script completes, returning its result as the result of the
// whole build. It does not run or compile any of the tests,
// nor does it write a snapshot of the world to cloud
// storage. This option is only supported for builders whose
// BuildConfig.SplitMakeRun returns true.
StopAfterMake bool
// needsGoProxy is whether this builder should have GOPROXY set.
// Currently this is only for the longtest builder, which needs
// to run cmd/go tests fetching from the network.
needsGoProxy bool
// InstallRacePackages controls which packages to "go install
// -race <pkgs>" after running make.bash (or equivalent). If
// the builder ends in "-race", the default if non-nil is just
// "std".
InstallRacePackages []string
// GoDeps is a list of of git sha1 commits that must be in the
// commit to be tested's history. If absent, this builder is
// not run for that commit.
GoDeps []string
// shouldRunDistTest optionally specifies a function to
// override the BuildConfig.ShouldRunDistTest method's
// default behavior.
shouldRunDistTest func(distTest string, isTry bool) bool
// numTestHelpers is the number of _additional_ buildlets
// past the first one to help out with sharded tests.
// For trybots, the numTryHelpers value is used, unless it's
// zero, in which case numTestHelpers is used.
numTestHelpers int
numTryTestHelpers int // for trybots. if 0, numTesthelpers is used
env []string // extra environment ("key=value") pairs
allScriptArgs []string
testHostConf *HostConfig // override HostConfig for testing, at least for now
}
// Env returns the environment variables this builder should run with.
func (c *BuildConfig) Env() []string {
env := []string{"GO_BUILDER_NAME=" + c.Name}
if c.FlakyNet {
env = append(env, "GO_BUILDER_FLAKY_NET=1")
}
env = append(env, c.hostConf().env...)
return append(env, c.env...)
}
// ModulesEnv returns the extra module-specific environment variables
// to append to this builder as a function of the repo being built
// ("go", "oauth2", "net", etc).
func (c *BuildConfig) ModulesEnv(repo string) (env []string) {
if c.IsReverse() && repo != "go" {
env = append(env, "GOPROXY=https://proxy.golang.org")
}
switch repo {
case "go":
if !c.OutboundNetworkAllowed() {
env = append(env, "GOPROXY=off")
}
case "oauth2", "build", "perf", "website":
env = append(env, "GO111MODULE=on")
}
return env
}
// ShouldTestPackageInGOPATHMode is used to control whether the package
// with the specified import path should be tested in GOPATH mode.
//
// When running tests for all golang.org/* repositories in GOPATH mode,
// this method is called repeatedly with the full import path of each
// package that is found and is being considered for testing in GOPATH
// mode. It's not used and has no effect on import paths in the main
// "go" repository. It has no effect on tests done in module mode.
//
// When considering making changes here, keep the release policy in mind:
//
// https://golang.org/doc/devel/release.html#policy
//
func (*BuildConfig) ShouldTestPackageInGOPATHMode(importPath string) bool {
if importPath == "golang.org/x/tools/gopls" ||
strings.HasPrefix(importPath, "golang.org/x/tools/gopls/") {
// Don't test golang.org/x/tools/gopls/... in GOPATH mode.
return false
}
if importPath == "golang.org/x/net/http2/h2demo" {
// Don't test golang.org/x/net/http2/h2demo in GOPATH mode.
//
// It was never tested before golang.org/issue/34361 because it
// had a +build h2demo constraint. But that build constraint is
// being removed, so explicitly skip testing it in GOPATH mode.
//
// The package is supported only in module mode now, since
// it requires third-party dependencies.
return false
}
// Test everything else in GOPATH mode as usual.
return true
}
func (c *BuildConfig) IsReverse() bool { return c.hostConf().IsReverse }
func (c *BuildConfig) IsContainer() bool { return c.hostConf().IsContainer() }
func (c *HostConfig) IsContainer() bool { return c.ContainerImage != "" }
func (c *BuildConfig) IsVM() bool { return c.hostConf().IsVM() }
func (c *HostConfig) IsVM() bool { return c.VMImage != "" }
func (c *BuildConfig) GOOS() string { return c.Name[:strings.Index(c.Name, "-")] }
func (c *BuildConfig) GOARCH() string {
arch := c.Name[strings.Index(c.Name, "-")+1:]
i := strings.Index(arch, "-")
if i == -1 {
return arch
}
return arch[:i]
}
// FilePathJoin is mostly like filepath.Join (without the cleaning) except
// it uses the path separator of c.GOOS instead of the host system's.
func (c *BuildConfig) FilePathJoin(x ...string) string {
if c.GOOS() == "windows" {
return strings.Join(x, "\\")
}
return strings.Join(x, "/")
}
// DistTestsExecTimeout returns how long the coordinator should wait
// for a cmd/dist test execution to run the provided dist test names.
func (c *BuildConfig) DistTestsExecTimeout(distTests []string) time.Duration {
// TODO: consider using distTests? We never did before, but
// now we have the TestStats in the coordinator. Pass in a
// *buildstats.TestStats and use historical data times some
// fudge factor? For now just use the old 20 minute limit
// we've used since 2014, but scale it by the
// GO_TEST_TIMEOUT_SCALE for the super slow builders which
// struggle with, say, the cgo tests. (which should be broken
// up into separate dist tests or shards, like the test/ dir
// was)
d := 20 * time.Minute
d *= time.Duration(c.timeoutScale())
return d
}
// timeoutScale returns this builder's GO_TEST_TIMEOUT_SCALE value, or 1.
func (c *BuildConfig) timeoutScale() int {
const pfx = "GO_TEST_TIMEOUT_SCALE="
for _, env := range [][]string{c.env, c.hostConf().env} {
for _, kv := range env {
if strings.HasPrefix(kv, pfx) {
if n, err := strconv.Atoi(kv[len(pfx):]); err == nil && n > 0 {
return n
}
}
}
}
return 1
}
func (c *BuildConfig) hostConf() *HostConfig {
if c.testHostConf != nil {
return c.testHostConf
}
if c, ok := Hosts[c.HostType]; ok {
return c
}
panic(fmt.Sprintf("missing buildlet config for buildlet %q", c.Name))
}
// GoBootstrapURL returns the URL of a built Go 1.4+ tar.gz for the
// build configuration type c, or empty string if there isn't one.
func (c *BuildConfig) GoBootstrapURL(e *buildenv.Environment) string {
return strings.Replace(c.hostConf().goBootstrapURLTmpl, "$BUCKET", e.BuildletBucket, 1)
}
// BuildletBinaryURL returns the public URL of this builder's buildlet.
func (c *HostConfig) BuildletBinaryURL(e *buildenv.Environment) string {
tmpl := c.buildletURLTmpl
return strings.Replace(tmpl, "$BUCKET", e.BuildletBucket, 1)
}
func (c *BuildConfig) IsRace() bool {
return strings.HasSuffix(c.Name, "-race")
}
func (c *BuildConfig) IsLongTest() bool {
return strings.HasSuffix(c.Name, "-longtest")
}
// OutboundNetworkAllowed reports whether this builder should be
// allowed to make outbound network requests. This is only enforced
// on some builders. (Currently most Linux ones)
func (c *BuildConfig) OutboundNetworkAllowed() bool {
return c.IsLongTest()
}
func (c *BuildConfig) GoInstallRacePackages() []string {
if c.InstallRacePackages != nil {
return append([]string(nil), c.InstallRacePackages...)
}
if c.IsRace() {
return []string{"std"}
}
return nil
}
// AllScript returns the relative path to the operating system's script to
// do the build and run its standard set of tests.
// Example values are "src/all.bash", "src/all.bat", "src/all.rc".
func (c *BuildConfig) AllScript() string {
if c.Name == "" {
panic("bogus BuildConfig")
}
if c.IsRace() {
if strings.HasPrefix(c.Name, "windows-") {
return "src/race.bat"
}
return "src/race.bash"
}
if strings.HasPrefix(c.Name, "windows-") {
return "src/all.bat"
}
if strings.HasPrefix(c.Name, "plan9-") {
return "src/all.rc"
}
if strings.HasPrefix(c.Name, "nacl-") {
return "src/nacltest.bash"
}
if strings.HasPrefix(c.Name, "darwin-arm") && !strings.Contains(c.Name, "corellium") {
return "src/iostest.bash"
}
if strings.HasPrefix(c.Name, "misc-compile") {
return "src/buildall.bash"
}
return "src/all.bash"
}
// SplitMakeRun reports whether the coordinator should first compile
// (using c.MakeScript), then snapshot, then run the tests (ideally
// sharded) using cmd/dist test.
// Eventually this function should always return true (and then be deleted)
// but for now we've only set up the scripts and verified that the main
// configurations work.
func (c *BuildConfig) SplitMakeRun() bool {
switch c.AllScript() {
case "src/all.bash", "src/all.bat",
"src/race.bash", "src/race.bat",
"src/all.rc",
"src/nacltest.bash":
// These we've verified to work.
return true
}
// TODO(bradfitz): make iostest.bash work too. And
// buildall.bash should really just be N small container jobs
// instead of a "buildall.bash". Then we can delete this whole
// method.
return false
}
func (c *BuildConfig) IsTryOnly() bool { return c.tryOnly }
func (c *BuildConfig) NeedsGoProxy() bool { return c.needsGoProxy }
// BuildsRepoPostSubmit reports whether the build configuration type c
// should build the given repo ("go", "net", etc) and branch
// ("master", "release-branch.go1.12") as a post-submit build
// that shows up on https://build.golang.org/.
func (c *BuildConfig) BuildsRepoPostSubmit(repo, branch, goBranch string) bool {
if c.tryOnly {
return false
}
return c.buildsRepoAtAll(repo, branch, goBranch)
}
// BuildsRepoTryBot reports whether the build configuration type c
// should build the given repo ("go", "net", etc) and branch
// ("master", "release-branch.go1.12") as a trybot.
func (c *BuildConfig) BuildsRepoTryBot(repo, branch, goBranch string) bool {
return c.tryBot != nil && c.tryBot(repo, branch, goBranch) && c.buildsRepoAtAll(repo, branch, goBranch)
}
// ShouldRunDistTest reports whether the named cmd/dist test should be
// run for this build config. The isTry parameter is whether this is
// for a trybot (pre-submit) run.
//
// In general, this returns true, unless a builder has configured it
// otherwise. Certain portable, slow tests are only run on fast builders in
// trybot mode.
func (c *BuildConfig) ShouldRunDistTest(distTest string, isTry bool) bool {
if c.shouldRunDistTest != nil {
return c.shouldRunDistTest(distTest, isTry)
}
if distTest == "api" {
// This test is slow and has the same behavior
// everywhere, so only run it on our fastest buidler
// (linux-amd64) when in trybot mode.
return !isTry || c.Name == "linux-amd64"
}
return true
}
// buildsRepoAtAll reports whether we should do builds of the provided
// repo ("go", "sys", "net", etc). This applies to both post-submit
// and trybot builds. Use BuildsRepoPostSubmit for only post-submit
// or BuildsRepoTryBot for trybots.
//
// The branch is the branch of repo ("master",
// "release-branch.go1.12", etc); it is required. The goBranch is the
// branch of Go itself. It's required if repo != "go". When repo ==
// "go", the goBranch defaults to the value of branch.
func (c *BuildConfig) buildsRepoAtAll(repo, branch, goBranch string) bool {
if goBranch == "" {
if repo == "go" {
goBranch = branch
} else {
panic("missing goBranch")
}