-
Notifications
You must be signed in to change notification settings - Fork 410
/
update.go
2989 lines (2649 loc) · 111 KB
/
update.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package daemon
import (
"bytes"
"context"
"errors"
"fmt"
"io/fs"
"os"
"os/exec"
"os/user"
"path/filepath"
"reflect"
goruntime "runtime"
"strconv"
"strings"
"syscall"
"time"
"github.com/clarketm/json"
ign3types "github.com/coreos/ignition/v2/config/v3_4/types"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
kubeErrs "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog/v2"
features "github.com/openshift/api/features"
mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
mcfgalphav1 "github.com/openshift/api/machineconfiguration/v1alpha1"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
opv1 "github.com/openshift/api/operator/v1"
"github.com/openshift/machine-config-operator/pkg/apihelpers"
ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
"github.com/openshift/machine-config-operator/pkg/daemon/constants"
pivottypes "github.com/openshift/machine-config-operator/pkg/daemon/pivot/types"
pivotutils "github.com/openshift/machine-config-operator/pkg/daemon/pivot/utils"
"github.com/openshift/machine-config-operator/pkg/daemon/runtimeassets"
"github.com/openshift/machine-config-operator/pkg/upgrademonitor"
)
const (
// defaultDirectoryPermissions houses the default mode to use when no directory permissions are provided
defaultDirectoryPermissions os.FileMode = 0o755
// defaultFilePermissions houses the default mode to use when no file permissions are provided
defaultFilePermissions os.FileMode = 0o644
// fipsFile is the file to check if FIPS is enabled
fipsFile = "/proc/sys/crypto/fips_enabled"
extensionsRepo = "/etc/yum.repos.d/coreos-extensions.repo"
osExtensionsContentBaseDir = "/run/mco-extensions/"
// These are the actions for a node to take after applying config changes. (e.g. a new machineconfig is applied)
// "None" means no special action needs to be taken
// This happens for example when ssh keys or the pull secret (/var/lib/kubelet/config.json) is changed
postConfigChangeActionNone = "none"
// The "reload crio" action will run "systemctl reload crio"
postConfigChangeActionReloadCrio = "reload crio"
// The "restart crio" action will run "systemctl restart crio"
postConfigChangeActionRestartCrio = "restart crio"
// Rebooting is still the default scenario for any other change
postConfigChangeActionReboot = "reboot"
)
func getNodeRef(node *corev1.Node) *corev1.ObjectReference {
return &corev1.ObjectReference{
Kind: "Node",
Name: node.GetName(),
UID: node.GetUID(),
}
}
func restartService(name string) error {
return runCmdSync("systemctl", "restart", name)
}
func reloadService(name string) error {
return runCmdSync("systemctl", "reload", name)
}
func reloadDaemon() error {
return runCmdSync("systemctl", constants.DaemonReloadCommand)
}
func (dn *Daemon) finishRebootlessUpdate() error {
// Get current state of node, in case of an error reboot
state, err := dn.getStateAndConfigs()
if err != nil {
return fmt.Errorf("could not apply update: error processing state and configs. Error: %w", err)
}
var inDesiredConfig bool
var missingODC bool
if missingODC, inDesiredConfig, err = dn.updateConfigAndState(state); err != nil {
return fmt.Errorf("could not apply update: setting node's state to Done failed. Error: %w", err)
}
if missingODC {
return fmt.Errorf("error updating state.currentconfig from on-disk currentconfig")
}
if inDesiredConfig {
// (re)start the config drift monitor since rebooting isn't needed.
dn.startConfigDriftMonitor()
return nil
}
// currentConfig != desiredConfig, kick off an update
return dn.triggerUpdateWithMachineConfig(state.currentConfig, state.desiredConfig, true)
}
func (dn *Daemon) executeReloadServiceNodeDisruptionAction(serviceName string, reloadErr error) error {
if reloadErr != nil {
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeWarning, "FailedServiceReload", fmt.Sprintf("Reloading service %s failed. Error: %v", serviceName, reloadErr))
}
return fmt.Errorf("could not apply update: reloading %s configuration failed. Error: %w", serviceName, reloadErr)
}
err := upgrademonitor.GenerateAndApplyMachineConfigNodes(
&upgrademonitor.Condition{State: mcfgalphav1.MachineConfigNodeUpdatePostActionComplete, Reason: string(mcfgalphav1.MachineConfigNodeUpdateReloaded), Message: fmt.Sprintf("Node has reloaded service %s", serviceName)},
&upgrademonitor.Condition{State: mcfgalphav1.MachineConfigNodeUpdateReloaded, Reason: fmt.Sprintf("%s%s", string(mcfgalphav1.MachineConfigNodeUpdatePostActionComplete), string(mcfgalphav1.MachineConfigNodeUpdateReloaded)), Message: fmt.Sprintf("Upgrade required a service %s reload. Completed this this as a post update action.", serviceName)},
metav1.ConditionTrue,
metav1.ConditionTrue,
dn.node,
dn.mcfgClient,
dn.featureGatesAccessor,
)
if err != nil {
klog.Errorf("Error making MCN for Reloading success: %v", err)
}
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeNormal, "ServiceReload", "Config changes do not require reboot. Service %s was reloaded.", serviceName)
}
logSystem("%s service reloaded successfully!", serviceName)
return nil
}
// performPostConfigChangeNodeDisruptionAction takes action based on the cluster's Node disruption policies.
// For non-reboot action, it applies configuration, updates node's config and state.
// In the end uncordon node to schedule workload.
// If at any point an error occurs, we reboot the node so that node has correct configuration.
func (dn *Daemon) performPostConfigChangeNodeDisruptionAction(postConfigChangeActions []opv1.NodeDisruptionPolicyStatusAction, configName string) error {
for _, action := range postConfigChangeActions {
// Drain is already completed at this stage and essentially a no-op for this loop, so no need to log that.
if action.Type == opv1.DrainStatusAction {
continue
}
logSystem("Performing post config change action: %v for config %s", action.Type, configName)
switch action.Type {
case opv1.RebootStatusAction:
err := upgrademonitor.GenerateAndApplyMachineConfigNodes(
&upgrademonitor.Condition{State: mcfgalphav1.MachineConfigNodeUpdatePostActionComplete, Reason: string(mcfgalphav1.MachineConfigNodeUpdateRebooted), Message: fmt.Sprintf("Node will reboot into config %s", configName)},
&upgrademonitor.Condition{State: mcfgalphav1.MachineConfigNodeUpdateRebooted, Reason: fmt.Sprintf("%s%s", string(mcfgalphav1.MachineConfigNodeUpdatePostActionComplete), string(mcfgalphav1.MachineConfigNodeUpdateRebooted)), Message: "Upgrade requires a reboot. Currently doing this as the post update action."},
metav1.ConditionUnknown,
metav1.ConditionUnknown,
dn.node,
dn.mcfgClient,
dn.featureGatesAccessor,
)
if err != nil {
klog.Errorf("Error making MCN for rebooting: %v", err)
}
logSystem("Rebooting node")
return dn.reboot(fmt.Sprintf("Node will reboot into config %s", configName))
case opv1.NoneStatusAction:
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeNormal, "SkipReboot", "Config changes do not require reboot.")
}
err := upgrademonitor.GenerateAndApplyMachineConfigNodes(
&upgrademonitor.Condition{State: mcfgalphav1.MachineConfigNodeUpdatePostActionComplete, Reason: "None", Message: "Changes do not require a reboot"},
nil,
metav1.ConditionTrue,
metav1.ConditionFalse,
dn.node,
dn.mcfgClient,
dn.featureGatesAccessor,
)
if err != nil {
klog.Errorf("Error making MCN for no post config change action: %v", err)
}
logSystem("Node has Desired Config %s, skipping reboot", configName)
case opv1.RestartStatusAction:
serviceName := string(action.Restart.ServiceName)
if err := restartService(serviceName); err != nil {
// On RHEL nodes, this service is not available and will error out.
// In those cases, directly run the command instead of using the service
if serviceName == constants.UpdateCATrustServiceName {
logSystem("Error executing %s unit, falling back to command", serviceName)
cmd := exec.Command(constants.UpdateCATrustCommand)
var stderr bytes.Buffer
cmd.Stdout = os.Stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeWarning, "FailedServiceRestart", fmt.Sprintf("Restarting %s service failed. Error: %v", serviceName, err))
}
return fmt.Errorf("error running %s: %s: %w", constants.UpdateCATrustCommand, stderr.String(), err)
}
} else {
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeWarning, "FailedServiceRestart", fmt.Sprintf("Restarting %s service failed. Error: %v", serviceName, err))
}
return fmt.Errorf("could not apply update: restarting %s service failed. Error: %w", serviceName, err)
}
}
// TODO: Add a new MCN Condition to the API for service restarts?
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeNormal, "ServiceRestart", "Config changes do not require reboot. Service %s was restarted.", serviceName)
}
logSystem("%s service restarted successfully!", serviceName)
case opv1.ReloadStatusAction:
// Execute a generic service reload defined by the action object
serviceName := string(action.Reload.ServiceName)
if err := dn.executeReloadServiceNodeDisruptionAction(serviceName, reloadService(serviceName)); err != nil {
return err
}
case opv1.SpecialStatusAction:
// The special action type requires a CRIO reload
if err := dn.executeReloadServiceNodeDisruptionAction(constants.CRIOServiceName, reloadService(constants.CRIOServiceName)); err != nil {
return err
}
case opv1.DaemonReloadStatusAction:
// Execute daemon-reload
if err := dn.executeReloadServiceNodeDisruptionAction(constants.DaemonReloadCommand, reloadDaemon()); err != nil {
return err
}
}
}
// We are here, which means a reboot was not needed to apply the configuration.
return dn.finishRebootlessUpdate()
}
// performPostConfigChangeAction takes action based on what postConfigChangeAction has been asked.
// For non-reboot action, it applies configuration, updates node's config and state.
// In the end uncordon node to schedule workload.
// If at any point an error occurs, we reboot the node so that node has correct configuration.
func (dn *Daemon) performPostConfigChangeAction(postConfigChangeActions []string, configName string) error {
if ctrlcommon.InSlice(postConfigChangeActionReboot, postConfigChangeActions) {
err := upgrademonitor.GenerateAndApplyMachineConfigNodes(
&upgrademonitor.Condition{State: mcfgalphav1.MachineConfigNodeUpdatePostActionComplete, Reason: string(mcfgalphav1.MachineConfigNodeUpdateRebooted), Message: fmt.Sprintf("Node will reboot into config %s", configName)},
&upgrademonitor.Condition{State: mcfgalphav1.MachineConfigNodeUpdateRebooted, Reason: fmt.Sprintf("%s%s", string(mcfgalphav1.MachineConfigNodeUpdatePostActionComplete), string(mcfgalphav1.MachineConfigNodeUpdateRebooted)), Message: "Upgrade requires a reboot. Currently doing this as the post update action."},
metav1.ConditionUnknown,
metav1.ConditionUnknown,
dn.node,
dn.mcfgClient,
dn.featureGatesAccessor,
)
if err != nil {
klog.Errorf("Error making MCN for rebooting: %v", err)
}
logSystem("Rebooting node")
return dn.reboot(fmt.Sprintf("Node will reboot into config %s", configName))
}
if ctrlcommon.InSlice(postConfigChangeActionNone, postConfigChangeActions) {
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeNormal, "SkipReboot", "Config changes do not require reboot.")
}
err := upgrademonitor.GenerateAndApplyMachineConfigNodes(
&upgrademonitor.Condition{State: mcfgalphav1.MachineConfigNodeUpdatePostActionComplete, Reason: "None", Message: fmt.Sprintf("Changes do not require a reboot")},
nil,
metav1.ConditionTrue,
metav1.ConditionFalse,
dn.node,
dn.mcfgClient,
dn.featureGatesAccessor,
)
if err != nil {
klog.Errorf("Error making MCN for no post config change action: %v", err)
}
logSystem("Node has Desired Config %s, skipping reboot", configName)
}
if ctrlcommon.InSlice(postConfigChangeActionReloadCrio, postConfigChangeActions) {
serviceName := constants.CRIOServiceName
if err := reloadService(serviceName); err != nil {
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeWarning, "FailedServiceReload", fmt.Sprintf("Reloading %s service failed. Error: %v", serviceName, err))
}
return fmt.Errorf("could not apply update: reloading %s configuration failed. Error: %w", serviceName, err)
}
err := upgrademonitor.GenerateAndApplyMachineConfigNodes(
&upgrademonitor.Condition{State: mcfgalphav1.MachineConfigNodeUpdatePostActionComplete, Reason: string(mcfgalphav1.MachineConfigNodeUpdateReloaded), Message: "Node has reloaded CRIO"},
&upgrademonitor.Condition{State: mcfgalphav1.MachineConfigNodeUpdateReloaded, Reason: fmt.Sprintf("%s%s", string(mcfgalphav1.MachineConfigNodeUpdatePostActionComplete), string(mcfgalphav1.MachineConfigNodeUpdateReloaded)), Message: "Upgrade required a CRIO reload. Completed this this as the post update action."},
metav1.ConditionTrue,
metav1.ConditionTrue,
dn.node,
dn.mcfgClient,
dn.featureGatesAccessor,
)
if err != nil {
klog.Errorf("Error making MCN for Reloading success: %v", err)
}
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeNormal, "SkipReboot", "Config changes do not require reboot. Service %s was reloaded.", serviceName)
}
logSystem("%s config reloaded successfully! Desired config %s has been applied, skipping reboot", serviceName, configName)
}
if ctrlcommon.InSlice(postConfigChangeActionRestartCrio, postConfigChangeActions) {
cmd := exec.Command(constants.UpdateCATrustCommand)
var stderr bytes.Buffer
cmd.Stdout = os.Stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("error running %s: %s: %w", constants.UpdateCATrustCommand, string(stderr.Bytes()), err)
}
serviceName := constants.CRIOServiceName
if err := restartService(serviceName); err != nil {
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeWarning, "FailedServiceReload", fmt.Sprintf("Reloading %s service failed. Error: %v", serviceName, err))
}
return fmt.Errorf("could not apply update: reloading %s configuration failed. Error: %w", serviceName, err)
}
logSystem("%s config restarted successfully! Desired config %s has been applied, skipping reboot", serviceName, configName)
}
// We are here, which means a reboot was not needed to apply the configuration.
return dn.finishRebootlessUpdate()
}
func setRunningKargsWithCmdline(config *mcfgv1.MachineConfig, requestedKargs []string, cmdline []byte) error {
splits := splitKernelArguments(strings.TrimSpace(string(cmdline)))
config.Spec.KernelArguments = nil
for _, split := range splits {
for _, reqKarg := range requestedKargs {
if reqKarg == split {
config.Spec.KernelArguments = append(config.Spec.KernelArguments, reqKarg)
break
}
}
}
return nil
}
func setRunningKargs(config *mcfgv1.MachineConfig, requestedKargs []string) error {
rpmostreeKargsBytes, err := runGetOut("rpm-ostree", "kargs")
if err != nil {
return err
}
return setRunningKargsWithCmdline(config, requestedKargs, rpmostreeKargsBytes)
}
func canonicalizeEmptyMC(config *mcfgv1.MachineConfig) *mcfgv1.MachineConfig {
if config != nil {
return config
}
newIgnCfg := ctrlcommon.NewIgnConfig()
rawNewIgnCfg, err := json.Marshal(newIgnCfg)
if err != nil {
// This should never happen
panic(err)
}
return &mcfgv1.MachineConfig{
ObjectMeta: metav1.ObjectMeta{Name: "mco-empty-mc"},
Spec: mcfgv1.MachineConfigSpec{
Config: runtime.RawExtension{
Raw: rawNewIgnCfg,
},
},
}
}
// return true if the machineConfigDiff is not empty
func (dn *Daemon) compareMachineConfig(oldConfig, newConfig *mcfgv1.MachineConfig) (bool, error) {
oldConfig = canonicalizeEmptyMC(oldConfig)
oldConfigName := oldConfig.GetName()
newConfigName := newConfig.GetName()
mcDiff, err := newMachineConfigDiff(oldConfig, newConfig)
if err != nil {
return true, fmt.Errorf("error creating machineConfigDiff for comparison: %w", err)
}
if mcDiff.isEmpty() {
logSystem("No changes from %s to %s", oldConfigName, newConfigName)
return false, nil
}
logSystem("Changes detected from %s to %s: %+v", oldConfigName, newConfigName, mcDiff)
return true, nil
}
// addExtensionsRepo adds a repo into /etc/yum.repos.d/ which we use later to
// install extensions (additional packages).
func addExtensionsRepo(extensionsImageContentDir string) error {
repoContent := "[coreos-extensions]\nenabled=1\nmetadata_expire=1m\nbaseurl=" + extensionsImageContentDir + "/usr/share/rpm-ostree/extensions/\ngpgcheck=0\nskip_if_unavailable=False\n"
return writeFileAtomicallyWithDefaults(extensionsRepo, []byte(repoContent))
}
// podmanRemove kills and removes a container
func podmanRemove(cid string) {
// Ignore errors here
exec.Command("podman", "kill", cid).Run()
exec.Command("podman", "rm", "-f", cid).Run()
}
// return true if the image is present
func isImagePresent(imgURL string) (bool, error) {
// search the image
var imageSearch []byte
imageSearch, err := runGetOut("podman", "images", "-q", "--filter", fmt.Sprintf("reference=%s", imgURL))
if err != nil {
return false, fmt.Errorf("error searching the image: %w", err)
}
if strings.TrimSpace(string(imageSearch)) == "" {
return false, nil
}
return true, nil
}
func podmanCopy(imgURL, osImageContentDir string) (err error) {
// arguments used in external commands
var args []string
// make sure that osImageContentDir doesn't exist
os.RemoveAll(osImageContentDir)
// Check if the image is present
imagePresent, err := isImagePresent(imgURL)
if err != nil {
return
}
// Pull the container image
if !imagePresent {
var authArgs []string
if _, err := os.Stat(kubeletAuthFile); err == nil {
authArgs = append(authArgs, "--authfile", kubeletAuthFile)
}
args = []string{"pull", "-q"}
args = append(args, authArgs...)
args = append(args, imgURL)
_, err = pivotutils.RunExtBackground(numRetriesNetCommands, "podman", args...)
if err != nil {
return
}
}
// create a container
var cidBuf []byte
containerName := pivottypes.PivotNamePrefix + string(uuid.NewUUID())
cidBuf, err = runGetOut("podman", "create", "--net=none", "--annotation=org.openshift.machineconfigoperator.pivot=true", "--name", containerName, imgURL)
if err != nil {
return
}
// only delete created container, we will delete container image later as we may need it for podmanInspect()
defer podmanRemove(containerName)
// copy the content from create container locally into a temp directory under /run/
cid := strings.TrimSpace(string(cidBuf))
args = []string{"cp", fmt.Sprintf("%s:/", cid), osImageContentDir}
_, err = pivotutils.RunExtBackground(numRetriesNetCommands, "podman", args...)
if err != nil {
return
}
// Set selinux context to var_run_t to avoid selinux denial
args = []string{"-R", "-t", "var_run_t", osImageContentDir}
err = runCmdSync("chcon", args...)
if err != nil {
err = fmt.Errorf("changing selinux context on path %s: %w", osImageContentDir, err)
return
}
return
}
// ExtractExtensionsImage extracts the OS extensions content in a temporary directory under /run/machine-os-extensions
// and returns the path on successful extraction
func ExtractExtensionsImage(imgURL string) (osExtensionsImageContentDir string, err error) {
if err = os.MkdirAll(osExtensionsContentBaseDir, 0o755); err != nil {
err = fmt.Errorf("error creating directory %s: %w", osExtensionsContentBaseDir, err)
return
}
if osExtensionsImageContentDir, err = os.MkdirTemp(osExtensionsContentBaseDir, "os-extensions-content-"); err != nil {
return
}
// Extract the image using `podman cp`
return osExtensionsImageContentDir, podmanCopy(imgURL, osExtensionsImageContentDir)
}
// Remove pending deployment on OSTree based system
func removePendingDeployment() error {
return runRpmOstree("cleanup", "-p")
}
// applyOSChanges extracts the OS image and adds coreos-extensions repo if we have either OS update or package layering to perform
func (dn *CoreOSDaemon) applyOSChanges(mcDiff machineConfigDiff, oldConfig, newConfig *mcfgv1.MachineConfig) (retErr error) {
// We previously did not emit this event when kargs changed, so we still don't
if mcDiff.osUpdate || mcDiff.extensions || mcDiff.kernelType {
// We emitted this event before, so keep it
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeNormal, "InClusterUpgrade", fmt.Sprintf("Updating from oscontainer %s", newConfig.Spec.OSImageURL))
}
}
// Only check the image type and execute OS changes if:
// - machineconfig changed
// - we're staying on a realtime kernel ( need to run rpm-ostree update )
// - we have extensions ( need to run rpm-ostree update )
// We have at least one customer that removes the pull secret from the cluster to "shrinkwrap" it for distribution and we want
// to make sure we don't break that use case, but realtime kernel update and extensions update always ran
// if they were in use, so we also need to preserve that behavior.
// https://issues.redhat.com/browse/OCPBUGS-4049
if mcDiff.osUpdate || mcDiff.extensions || mcDiff.kernelType || mcDiff.kargs ||
canonicalizeKernelType(newConfig.Spec.KernelType) == ctrlcommon.KernelTypeRealtime ||
canonicalizeKernelType(newConfig.Spec.KernelType) == ctrlcommon.KernelType64kPages ||
len(newConfig.Spec.Extensions) > 0 {
// Throw started/staged events only if there is any update required for the OS
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeNormal, "OSUpdateStarted", mcDiff.osChangesString())
}
if err := dn.applyLayeredOSChanges(mcDiff, oldConfig, newConfig); err != nil {
return err
}
if dn.nodeWriter != nil {
var nodeName string
var nodeObjRef corev1.ObjectReference
if dn.node != nil {
nodeName = dn.node.ObjectMeta.GetName()
nodeObjRef = corev1.ObjectReference{
Kind: "Node",
Name: dn.node.GetName(),
UID: dn.node.GetUID(),
}
}
// We send out the event OSUpdateStaged synchronously to ensure it is recorded.
// Remove this when we can ensure all events are sent before exiting.
t := metav1.NewTime(time.Now())
event := &corev1.Event{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%v.%x", nodeName, t.UnixNano()),
Namespace: metav1.NamespaceDefault,
},
InvolvedObject: nodeObjRef,
Reason: "OSUpdateStaged",
Type: corev1.EventTypeNormal,
Message: "Changes to OS staged",
FirstTimestamp: t,
LastTimestamp: t,
Count: 1,
Source: corev1.EventSource{Component: "machineconfigdaemon", Host: dn.name},
}
// its ok to create a unique event for this low volume event
if _, err := dn.kubeClient.CoreV1().Events(metav1.NamespaceDefault).Create(context.TODO(),
event, metav1.CreateOptions{}); err != nil {
klog.Errorf("Failed to create event with reason 'OSUpdateStaged': %v", err)
}
}
}
return nil
}
func calculatePostConfigChangeActionFromMCDiffs(diffFileSet []string) (actions []string) {
filesPostConfigChangeActionNone := []string{
caBundleFilePath,
constants.KubeletAuthFile,
}
directoriesPostConfigChangeActionNone := []string{
constants.OpenShiftNMStateConfigDir,
}
filesPostConfigChangeActionReloadCrio := []string{
constants.ContainerRegistryConfPath,
constants.GPGNoRebootPath,
constants.ContainerRegistryPolicyPath,
}
filesPostConfigChangeActionRestartCrio := []string{
constants.UserCABundlePath,
}
dirsPostConfigChangeActionReloadCrio := []string{
constants.SigstoreRegistriesConfigDir,
}
actions = []string{postConfigChangeActionNone}
for _, path := range diffFileSet {
switch {
case ctrlcommon.InSlice(path, filesPostConfigChangeActionNone):
continue
case ctrlcommon.InSlice(path, filesPostConfigChangeActionReloadCrio),
ctrlcommon.InSlice(filepath.Dir(path), dirsPostConfigChangeActionReloadCrio):
// Don't override a restart CRIO action
if !ctrlcommon.InSlice(postConfigChangeActionRestartCrio, actions) {
actions = []string{postConfigChangeActionReloadCrio}
}
case ctrlcommon.InSlice(path, filesPostConfigChangeActionRestartCrio):
actions = []string{postConfigChangeActionRestartCrio}
case ctrlcommon.InSlice(filepath.Dir(path), directoriesPostConfigChangeActionNone):
continue
default:
actions = []string{postConfigChangeActionReboot}
return actions
}
}
return actions
}
// calculatePostConfigChangeNodeDisruptionActionFromMCDiffs takes action based on the cluster's Node disruption policies.
func calculatePostConfigChangeNodeDisruptionActionFromMCDiffs(diffSSH bool, diffFileSet, diffUnitSet []string, clusterPolicies opv1.NodeDisruptionPolicyClusterStatus) []opv1.NodeDisruptionPolicyStatusAction {
actions := []opv1.NodeDisruptionPolicyStatusAction{}
// Step through all file based policies, and build out the actions object
for _, diffPath := range diffFileSet {
pathFound, actionsFound := ctrlcommon.FindClosestFilePolicyPathMatch(diffPath, clusterPolicies.Files)
if pathFound {
klog.Infof("NodeDisruptionPolicy %v found for diff file %s", actionsFound, diffPath)
actions = append(actions, actionsFound...)
} else {
// If this file path has no policy defined, default to reboot
klog.V(4).Infof("no policy found for diff path %s", diffPath)
return []opv1.NodeDisruptionPolicyStatusAction{{
Type: opv1.RebootStatusAction,
}}
}
}
// Step through all unit based policies, and build out the actions object
for _, diffUnit := range diffUnitSet {
unitFound := false
for _, policyUnit := range clusterPolicies.Units {
klog.V(4).Infof("comparing policy unit name %s to diff unit name %s", string(policyUnit.Name), diffUnit)
if string(policyUnit.Name) == diffUnit {
klog.Infof("NodeDisruptionPolicy %v found for diff unit %s!", policyUnit.Actions, diffUnit)
actions = append(actions, policyUnit.Actions...)
unitFound = true
break
}
}
if !unitFound {
// If this unit has no policy defined, default to reboot
klog.V(4).Infof("no policy found for diff unit %s", diffUnit)
return []opv1.NodeDisruptionPolicyStatusAction{{
Type: opv1.RebootStatusAction,
}}
}
}
// SSH only has one possible policy(and there is a default), so blindly add that if there is an SSH diff
if diffSSH {
klog.Infof("SSH diff detected, applying SSH policy %v", clusterPolicies.SSHKey.Actions)
actions = append(actions, clusterPolicies.SSHKey.Actions...)
}
// If any of the actions need a reboot, then just return a single Reboot action
if apihelpers.CheckNodeDisruptionActionsForTargetActions(actions, opv1.RebootStatusAction) {
return []opv1.NodeDisruptionPolicyStatusAction{{
Type: opv1.RebootStatusAction,
}}
}
// If there is a "None" action in conjunction with other kinds of actions, strip out the "None" action elements as it is redundant
if apihelpers.CheckNodeDisruptionActionsForTargetActions(actions, opv1.NoneStatusAction) {
if apihelpers.CheckNodeDisruptionActionsForTargetActions(actions, opv1.DrainStatusAction, opv1.ReloadStatusAction, opv1.RestartStatusAction, opv1.DaemonReloadStatusAction, opv1.SpecialStatusAction) {
finalActions := []opv1.NodeDisruptionPolicyStatusAction{}
for _, action := range actions {
if action.Type != opv1.NoneStatusAction {
finalActions = append(finalActions, action)
}
}
return finalActions
}
// If we're here, this means that the action list has only "None" actions; return a single "None" Action
return []opv1.NodeDisruptionPolicyStatusAction{{
Type: opv1.NoneStatusAction,
}}
}
// If we're here, return as is - this means action list had zero "None" actions in the list
return actions
}
func calculatePostConfigChangeAction(diff *machineConfigDiff, diffFileSet []string) ([]string, error) {
// If a machine-config-daemon-force file is present, it means the user wants to
// move to desired state without additional validation. We will reboot the node in
// this case regardless of what MachineConfig diff is.
if _, err := os.Stat(constants.MachineConfigDaemonForceFile); err == nil {
if err := os.Remove(constants.MachineConfigDaemonForceFile); err != nil {
return []string{}, fmt.Errorf("failed to remove force validation file: %w", err)
}
klog.Infof("Setting post config change action to postConfigChangeActionReboot; %s present", constants.MachineConfigDaemonForceFile)
return []string{postConfigChangeActionReboot}, nil
}
if diff.osUpdate || diff.kargs || diff.fips || diff.units || diff.kernelType || diff.extensions {
// must reboot
return []string{postConfigChangeActionReboot}, nil
}
// Calculate actions based on file, unit and ssh diffs
return calculatePostConfigChangeActionFromMCDiffs(diffFileSet), nil
}
// calculatePostConfigChangeNodeDisruptionAction takes action based on the cluster's Node disruption policies.
func (dn *Daemon) calculatePostConfigChangeNodeDisruptionAction(diff *machineConfigDiff, diffFileSet, diffUnitSet []string) ([]opv1.NodeDisruptionPolicyStatusAction, error) {
var mcop *opv1.MachineConfiguration
var pollErr error
// Wait for mcop.Status.NodeDisruptionPolicyStatus to populate, otherwise error out. This shouldn't take very long
// as this is done by the operator sync loop, but may be extended if transitioning to TechPreview as the operator restarts,
if err := wait.PollUntilContextTimeout(context.TODO(), 5*time.Second, 2*time.Minute, true, func(_ context.Context) (bool, error) {
mcop, pollErr = dn.mcopClient.OperatorV1().MachineConfigurations().Get(context.TODO(), ctrlcommon.MCOOperatorKnobsObjectName, metav1.GetOptions{})
if pollErr != nil {
klog.Errorf("calculating NodeDisruptionPolicies: MachineConfiguration/cluster has not been created yet")
pollErr = fmt.Errorf("MachineConfiguration/cluster has not been created yet")
return false, nil
}
// Ensure status.ObservedGeneration matches the last generation of MachineConfiguration
if mcop.Generation != mcop.Status.ObservedGeneration {
klog.Errorf("calculating NodeDisruptionPolicies: NodeDisruptionPolicyStatus is not up to date.")
pollErr = fmt.Errorf("NodeDisruptionPolicyStatus is not up to date")
return false, nil
}
return true, nil
}); err != nil {
klog.Errorf("NodeDisruptionPolicyStatus was not ready: %v", pollErr)
return nil, fmt.Errorf("NodeDisruptionPolicyStatus was not ready: %v", pollErr)
}
// Continue policy calculation if no errors were encountered in fetching the policy.
// If a machine-config-daemon-force file is present, it means the user wants to
// move to desired state without additional validation. We will reboot the node in
// this case regardless of what MachineConfig diff is.
klog.Infof("Calculating node disruption actions")
if _, err := os.Stat(constants.MachineConfigDaemonForceFile); err == nil {
if err = os.Remove(constants.MachineConfigDaemonForceFile); err != nil {
return []opv1.NodeDisruptionPolicyStatusAction{}, fmt.Errorf("failed to remove force validation file: %w", err)
}
klog.Infof("Setting post config change node disruption action to Reboot; %s present", constants.MachineConfigDaemonForceFile)
return []opv1.NodeDisruptionPolicyStatusAction{{
Type: opv1.RebootStatusAction,
}}, nil
}
if diff.osUpdate || diff.kargs || diff.fips || diff.kernelType || diff.extensions {
// must reboot
return []opv1.NodeDisruptionPolicyStatusAction{{
Type: opv1.RebootStatusAction,
}}, nil
}
if !diff.files && !diff.units && !diff.passwd {
// This is a diff which requires no actions
klog.Infof("No changes in files, units or SSH keys, no NodeDisruptionPolicies are in effect")
return []opv1.NodeDisruptionPolicyStatusAction{{
Type: opv1.NoneStatusAction,
}}, nil
}
// Calculate actions based on file, unit and ssh diffs
nodeDisruptionActions := calculatePostConfigChangeNodeDisruptionActionFromMCDiffs(diff.passwd, diffFileSet, diffUnitSet, mcop.Status.NodeDisruptionPolicyStatus.ClusterPolicies)
// Print out node disruption actions for debug purposes
klog.Infof("Calculated node disruption actions:")
for _, action := range nodeDisruptionActions {
switch action.Type {
case opv1.ReloadStatusAction:
klog.Infof("%v - %v", action.Type, action.Reload.ServiceName)
case opv1.RestartStatusAction:
klog.Infof("%v - %v", action.Type, action.Restart.ServiceName)
default:
klog.Infof("%v", action.Type)
}
}
return nodeDisruptionActions, nil
}
// This is another update function implementation for the special case of
// on-cluster built images. It is necessary to perform certain steps
// post-reboot since rpm-ostree will not write contents to the /home/core
// directory nor certain portions of the /etc directory.
//
// This function should be consolidated with dn.update() and dn.updateHypershift(). See: https://issues.redhat.com/browse/MCO-810 for further discussion.
//
//nolint:gocyclo
func (dn *Daemon) updateOnClusterBuild(oldConfig, newConfig *mcfgv1.MachineConfig, oldImage, newImage string, skipCertificateWrite bool) (retErr error) {
oldConfig = canonicalizeEmptyMC(oldConfig)
if dn.nodeWriter != nil {
state, err := getNodeAnnotationExt(dn.node, constants.MachineConfigDaemonStateAnnotationKey, true)
if err != nil {
return err
}
if state != constants.MachineConfigDaemonStateDegraded && state != constants.MachineConfigDaemonStateUnreconcilable {
if err := dn.nodeWriter.SetWorking(); err != nil {
return fmt.Errorf("error setting node's state to Working: %w", err)
}
}
}
dn.catchIgnoreSIGTERM()
defer func() {
// now that we do rebootless updates, we need to turn off our SIGTERM protection
// regardless of how we leave the "update loop"
dn.CancelSIGTERM()
}()
oldConfigName := oldConfig.GetName()
newConfigName := newConfig.GetName()
oldIgnConfig, err := ctrlcommon.ParseAndConvertConfig(oldConfig.Spec.Config.Raw)
if err != nil {
return fmt.Errorf("parsing old Ignition config failed: %w", err)
}
newIgnConfig, err := ctrlcommon.ParseAndConvertConfig(newConfig.Spec.Config.Raw)
if err != nil {
return fmt.Errorf("parsing new Ignition config failed: %w", err)
}
klog.Infof("Checking Reconcilable for config %v to %v", oldConfigName, newConfigName)
// Make sure we can actually reconcile this state. In the future, this check should be moved to the BuildController and performed prior to the build occurring. This addresses the following bugs:
// - https://issues.redhat.com/browse/OCPBUGS-18670
// - https://issues.redhat.com/browse/OCPBUGS-18535
// -
diff, reconcilableError := reconcilable(oldConfig, newConfig)
if reconcilableError != nil {
wrappedErr := fmt.Errorf("can't reconcile config %s with %s: %w", oldConfigName, newConfigName, reconcilableError)
if dn.nodeWriter != nil {
dn.nodeWriter.Eventf(corev1.EventTypeWarning, "FailedToReconcile", wrappedErr.Error())
}
return &unreconcilableErr{wrappedErr}
}
if oldImage == newImage && newImage != "" {
if oldImage == "" {
logSystem("Starting transition to %q", newImage)
} else {
logSystem("Starting transition from %q to %q", oldImage, newImage)
}
}
if err := dn.performDrain(); err != nil {
return err
}
// If the new image pullspec is already on disk, do not attempt to re-apply
// it. rpm-ostree will throw an error as a result.
// See: https://issues.redhat.com/browse/OCPBUGS-18414.
if oldImage != newImage {
// If the new image field is empty, set it to the OS image URL value
// provided by the MachineConfig to do a rollback.
if newImage == "" {
klog.Infof("%s empty, reverting to osImageURL %s from MachineConfig %s", constants.DesiredImageAnnotationKey, newConfig.Spec.OSImageURL, newConfig.Name)
newImage = newConfig.Spec.OSImageURL
}
if err := dn.updateLayeredOSToPullspec(newImage); err != nil {
return err
}
} else {
klog.Infof("Image pullspecs equal, skipping rpm-ostree rebase")
}
// If the new OS image equals the OS image URL value, this means we're in a
// revert-from-layering situation. This also means we can return early after
// taking a different path.
if newImage == newConfig.Spec.OSImageURL {
return dn.finalizeRevertToNonLayering(newConfig)
}
// update files on disk that need updating
if err := dn.updateFiles(oldIgnConfig, newIgnConfig, skipCertificateWrite); err != nil {
return err
}
defer func() {
if retErr != nil {
if err := dn.updateFiles(newIgnConfig, oldIgnConfig, skipCertificateWrite); err != nil {
errs := kubeErrs.NewAggregate([]error{err, retErr})
retErr = fmt.Errorf("error rolling back files writes: %w", errs)
return
}
}
}()
// update file permissions
if err := dn.updateKubeConfigPermission(); err != nil {
return err
}
// only update passwd if it has changed (do not nullify)
// we do not need to include SetPasswordHash in this, since only updateSSHKeys has issues on firstboot.
// For on-cluster builds, this needs to be performed here instead of during
// the image build process. This is bceause rpm-ostree will not touch files
// in /home/core. See: https://issues.redhat.com/browse/OCPBUGS-18458
if diff.passwd {
if err := dn.updateSSHKeys(newIgnConfig.Passwd.Users, oldIgnConfig.Passwd.Users); err != nil {
return err
}
defer func() {
if retErr != nil {
if err := dn.updateSSHKeys(newIgnConfig.Passwd.Users, oldIgnConfig.Passwd.Users); err != nil {
errs := kubeErrs.NewAggregate([]error{err, retErr})
retErr = fmt.Errorf("error rolling back SSH keys updates: %w", errs)
return
}
}
}()
}
// Set password hash
// See: https://issues.redhat.com/browse/OCPBUGS-18456
if err := dn.SetPasswordHash(newIgnConfig.Passwd.Users, oldIgnConfig.Passwd.Users); err != nil {
return err
}
defer func() {
if retErr != nil {
if err := dn.SetPasswordHash(newIgnConfig.Passwd.Users, oldIgnConfig.Passwd.Users); err != nil {
errs := kubeErrs.NewAggregate([]error{err, retErr})
retErr = fmt.Errorf("error rolling back password hash updates: %w", errs)
return
}
}
}()
// Update the kernal args if there is a difference
if diff.kargs && dn.os.IsCoreOSVariant() {
coreOSDaemon := CoreOSDaemon{dn}
if err := coreOSDaemon.updateKernelArguments(oldConfig.Spec.KernelArguments, newConfig.Spec.KernelArguments); err != nil {
return err
}
}
// Ideally we would want to update kernelArguments only via MachineConfigs.
// We are keeping this to maintain compatibility and OKD requirement.
if err := UpdateTuningArgs(KernelTuningFile, CmdLineFile); err != nil {
return err
}
odc := &onDiskConfig{
currentImage: newImage,
currentConfig: newConfig,
}
if err := dn.storeCurrentConfigOnDisk(odc); err != nil {
return err
}
defer func() {
if retErr != nil {
odc.currentConfig = oldConfig
odc.currentImage = oldImage
if err := dn.storeCurrentConfigOnDisk(odc); err != nil {
errs := kubeErrs.NewAggregate([]error{err, retErr})
retErr = fmt.Errorf("error rolling back current config on disk: %w", errs)
return
}
}
}()
return dn.reboot(fmt.Sprintf("Node will reboot into image %s / MachineConfig %s", newImage, newConfigName))
}
// Finalizes the revert process by enabling a special systemd unit prior to
// rebooting the node.
//
// After we write the original factory image to the node, none of the files
// specified in the MachineConfig will be written due to how rpm-ostree handles
// file writes. Because those files are owned by the layered container image,
// they are not present after reboot; even if we were to write them to the node
// before rebooting. Consequently, after reverting back to the original image,
// the node will lose contact with the control plane and the easiest way to
// reestablish contact is to rebootstrap the node.
//
// By comparison, if we write a file that is _not_ owned by the layered
// container image, this file will persist after reboot. So what we do is write