From f4ccb5a781022fccd413ca719c3aaafd7668daec Mon Sep 17 00:00:00 2001 From: Sean Malloy Date: Thu, 27 Aug 2020 23:07:46 -0500 Subject: [PATCH] Begin Refactoring RemovePodsViolatingTopologySpreadConstraint --- go.mod | 1 - pkg/api/types.go | 32 - pkg/api/v1alpha1/types.go | 46 +- pkg/api/v1alpha1/zz_generated.conversion.go | 69 - pkg/api/v1alpha1/zz_generated.deepcopy.go | 52 - pkg/api/zz_generated.deepcopy.go | 52 - pkg/descheduler/descheduler.go | 16 +- ...straint.go => topologyspreadconstraint.go} | 185 ++- vendor/github.com/golang/glog/LICENSE | 191 --- vendor/github.com/golang/glog/README | 44 - vendor/github.com/golang/glog/glog.go | 1180 ----------------- vendor/github.com/golang/glog/glog_file.go | 124 -- vendor/modules.txt | 3 - 13 files changed, 145 insertions(+), 1850 deletions(-) rename pkg/descheduler/strategies/{topologyspreadyconstraint.go => topologyspreadconstraint.go} (57%) delete mode 100644 vendor/github.com/golang/glog/LICENSE delete mode 100644 vendor/github.com/golang/glog/README delete mode 100644 vendor/github.com/golang/glog/glog.go delete mode 100644 vendor/github.com/golang/glog/glog_file.go diff --git a/go.mod b/go.mod index 4223fbebbb..12fd1999c7 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module sigs.k8s.io/descheduler go 1.14 require ( - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 k8s.io/api v0.19.0-rc.4 diff --git a/pkg/api/types.go b/pkg/api/types.go index 24e0cbdb08..2044e384a0 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -72,9 +72,6 @@ type StrategyParameters struct { Namespaces *Namespaces ThresholdPriority *int32 ThresholdPriorityClassName string - // TopologySpreadConstraints describes how a group of pods should be spread across topology - // domains. Descheduler will use these constraints to decide which pods to evict. - NamespacedTopologySpreadConstraints []NamespacedTopologySpreadConstraint } type Percentage float64 @@ -94,32 +91,3 @@ type PodsHavingTooManyRestarts struct { type RemoveDuplicates struct { ExcludeOwnerKinds []string } - -type NamespacedTopologySpreadConstraint struct { - Namespace string - TopologySpreadConstraints []TopologySpreadConstraint -} - -type TopologySpreadConstraint struct { - // MaxSkew describes the degree to which pods may be unevenly distributed. - // It's the maximum permitted difference between the number of matching pods in - // any two topology domains of a given topology type. - // For example, in a 3-zone cluster, currently pods with the same labelSelector - // are "spread" such that zone1 and zone2 each have one pod, but not zone3. - // - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; - // scheduling it onto zone1(zone2) would make the ActualSkew(2) violate MaxSkew(1) - // - if MaxSkew is 2, incoming pod can be scheduled to any zone. - // It's a required value. Default value is 1 and 0 is not allowed. - MaxSkew int32 - // TopologyKey is the key of node labels. Nodes that have a label with this key - // and identical values are considered to be in the same topology. - // We consider each as a "bucket", and try to put balanced number - // of pods into each bucket. - // It's a required field. - TopologyKey string - // LabelSelector is used to find matching pods. - // Pods that match this label selector are counted to determine the number of pods - // in their corresponding topology domain. - // +optional - LabelSelector *metav1.LabelSelector -} diff --git a/pkg/api/v1alpha1/types.go b/pkg/api/v1alpha1/types.go index ff28081786..eca22681cd 100644 --- a/pkg/api/v1alpha1/types.go +++ b/pkg/api/v1alpha1/types.go @@ -62,15 +62,14 @@ type Namespaces struct { // Besides Namespaces ThresholdPriority and ThresholdPriorityClassName only one of its members may be specified type StrategyParameters struct { - NodeResourceUtilizationThresholds *NodeResourceUtilizationThresholds `json:"nodeResourceUtilizationThresholds,omitempty"` - NodeAffinityType []string `json:"nodeAffinityType,omitempty"` - PodsHavingTooManyRestarts *PodsHavingTooManyRestarts `json:"podsHavingTooManyRestarts,omitempty"` - MaxPodLifeTimeSeconds *uint `json:"maxPodLifeTimeSeconds,omitempty"` - RemoveDuplicates *RemoveDuplicates `json:"removeDuplicates,omitempty"` - Namespaces *Namespaces `json:"namespaces"` - ThresholdPriority *int32 `json:"thresholdPriority"` - ThresholdPriorityClassName string `json:"thresholdPriorityClassName"` - NamespacedTopologySpreadConstraints []NamespacedTopologySpreadConstraint + NodeResourceUtilizationThresholds *NodeResourceUtilizationThresholds `json:"nodeResourceUtilizationThresholds,omitempty"` + NodeAffinityType []string `json:"nodeAffinityType,omitempty"` + PodsHavingTooManyRestarts *PodsHavingTooManyRestarts `json:"podsHavingTooManyRestarts,omitempty"` + MaxPodLifeTimeSeconds *uint `json:"maxPodLifeTimeSeconds,omitempty"` + RemoveDuplicates *RemoveDuplicates `json:"removeDuplicates,omitempty"` + Namespaces *Namespaces `json:"namespaces"` + ThresholdPriority *int32 `json:"thresholdPriority"` + ThresholdPriorityClassName string `json:"thresholdPriorityClassName"` } type Percentage float64 @@ -90,32 +89,3 @@ type PodsHavingTooManyRestarts struct { type RemoveDuplicates struct { ExcludeOwnerKinds []string `json:"excludeOwnerKinds,omitempty"` } - -type NamespacedTopologySpreadConstraint struct { - Namespace string - TopologySpreadConstraints []TopologySpreadConstraint -} - -type TopologySpreadConstraint struct { - // MaxSkew describes the degree to which pods may be unevenly distributed. - // It's the maximum permitted difference between the number of matching pods in - // any two topology domains of a given topology type. - // For example, in a 3-zone cluster, currently pods with the same labelSelector - // are "spread" such that zone1 and zone2 each have one pod, but not zone3. - // - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; - // scheduling it onto zone1(zone2) would make the ActualSkew(2) violate MaxSkew(1) - // - if MaxSkew is 2, incoming pod can be scheduled to any zone. - // It's a required value. Default value is 1 and 0 is not allowed. - MaxSkew int32 - // TopologyKey is the key of node labels. Nodes that have a label with this key - // and identical values are considered to be in the same topology. - // We consider each as a "bucket", and try to put balanced number - // of pods into each bucket. - // It's a required field. - TopologyKey string - // LabelSelector is used to find matching pods. - // Pods that match this label selector are counted to determine the number of pods - // in their corresponding topology domain. - // +optional - LabelSelector *metav1.LabelSelector -} diff --git a/pkg/api/v1alpha1/zz_generated.conversion.go b/pkg/api/v1alpha1/zz_generated.conversion.go index 1c38e3eb98..7c1d245307 100644 --- a/pkg/api/v1alpha1/zz_generated.conversion.go +++ b/pkg/api/v1alpha1/zz_generated.conversion.go @@ -23,7 +23,6 @@ package v1alpha1 import ( unsafe "unsafe" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" api "sigs.k8s.io/descheduler/pkg/api" @@ -56,16 +55,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*NamespacedTopologySpreadConstraint)(nil), (*api.NamespacedTopologySpreadConstraint)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_NamespacedTopologySpreadConstraint_To_api_NamespacedTopologySpreadConstraint(a.(*NamespacedTopologySpreadConstraint), b.(*api.NamespacedTopologySpreadConstraint), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*api.NamespacedTopologySpreadConstraint)(nil), (*NamespacedTopologySpreadConstraint)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_api_NamespacedTopologySpreadConstraint_To_v1alpha1_NamespacedTopologySpreadConstraint(a.(*api.NamespacedTopologySpreadConstraint), b.(*NamespacedTopologySpreadConstraint), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*Namespaces)(nil), (*api.Namespaces)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_Namespaces_To_api_Namespaces(a.(*Namespaces), b.(*api.Namespaces), scope) }); err != nil { @@ -116,16 +105,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*TopologySpreadConstraint)(nil), (*api.TopologySpreadConstraint)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_TopologySpreadConstraint_To_api_TopologySpreadConstraint(a.(*TopologySpreadConstraint), b.(*api.TopologySpreadConstraint), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*api.TopologySpreadConstraint)(nil), (*TopologySpreadConstraint)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_api_TopologySpreadConstraint_To_v1alpha1_TopologySpreadConstraint(a.(*api.TopologySpreadConstraint), b.(*TopologySpreadConstraint), scope) - }); err != nil { - return err - } return nil } @@ -179,28 +158,6 @@ func Convert_api_DeschedulerStrategy_To_v1alpha1_DeschedulerStrategy(in *api.Des return autoConvert_api_DeschedulerStrategy_To_v1alpha1_DeschedulerStrategy(in, out, s) } -func autoConvert_v1alpha1_NamespacedTopologySpreadConstraint_To_api_NamespacedTopologySpreadConstraint(in *NamespacedTopologySpreadConstraint, out *api.NamespacedTopologySpreadConstraint, s conversion.Scope) error { - out.Namespace = in.Namespace - out.TopologySpreadConstraints = *(*[]api.TopologySpreadConstraint)(unsafe.Pointer(&in.TopologySpreadConstraints)) - return nil -} - -// Convert_v1alpha1_NamespacedTopologySpreadConstraint_To_api_NamespacedTopologySpreadConstraint is an autogenerated conversion function. -func Convert_v1alpha1_NamespacedTopologySpreadConstraint_To_api_NamespacedTopologySpreadConstraint(in *NamespacedTopologySpreadConstraint, out *api.NamespacedTopologySpreadConstraint, s conversion.Scope) error { - return autoConvert_v1alpha1_NamespacedTopologySpreadConstraint_To_api_NamespacedTopologySpreadConstraint(in, out, s) -} - -func autoConvert_api_NamespacedTopologySpreadConstraint_To_v1alpha1_NamespacedTopologySpreadConstraint(in *api.NamespacedTopologySpreadConstraint, out *NamespacedTopologySpreadConstraint, s conversion.Scope) error { - out.Namespace = in.Namespace - out.TopologySpreadConstraints = *(*[]TopologySpreadConstraint)(unsafe.Pointer(&in.TopologySpreadConstraints)) - return nil -} - -// Convert_api_NamespacedTopologySpreadConstraint_To_v1alpha1_NamespacedTopologySpreadConstraint is an autogenerated conversion function. -func Convert_api_NamespacedTopologySpreadConstraint_To_v1alpha1_NamespacedTopologySpreadConstraint(in *api.NamespacedTopologySpreadConstraint, out *NamespacedTopologySpreadConstraint, s conversion.Scope) error { - return autoConvert_api_NamespacedTopologySpreadConstraint_To_v1alpha1_NamespacedTopologySpreadConstraint(in, out, s) -} - func autoConvert_v1alpha1_Namespaces_To_api_Namespaces(in *Namespaces, out *api.Namespaces, s conversion.Scope) error { out.Include = *(*[]string)(unsafe.Pointer(&in.Include)) out.Exclude = *(*[]string)(unsafe.Pointer(&in.Exclude)) @@ -298,7 +255,6 @@ func autoConvert_v1alpha1_StrategyParameters_To_api_StrategyParameters(in *Strat out.Namespaces = (*api.Namespaces)(unsafe.Pointer(in.Namespaces)) out.ThresholdPriority = (*int32)(unsafe.Pointer(in.ThresholdPriority)) out.ThresholdPriorityClassName = in.ThresholdPriorityClassName - out.NamespacedTopologySpreadConstraints = *(*[]api.NamespacedTopologySpreadConstraint)(unsafe.Pointer(&in.NamespacedTopologySpreadConstraints)) return nil } @@ -316,7 +272,6 @@ func autoConvert_api_StrategyParameters_To_v1alpha1_StrategyParameters(in *api.S out.Namespaces = (*Namespaces)(unsafe.Pointer(in.Namespaces)) out.ThresholdPriority = (*int32)(unsafe.Pointer(in.ThresholdPriority)) out.ThresholdPriorityClassName = in.ThresholdPriorityClassName - out.NamespacedTopologySpreadConstraints = *(*[]NamespacedTopologySpreadConstraint)(unsafe.Pointer(&in.NamespacedTopologySpreadConstraints)) return nil } @@ -324,27 +279,3 @@ func autoConvert_api_StrategyParameters_To_v1alpha1_StrategyParameters(in *api.S func Convert_api_StrategyParameters_To_v1alpha1_StrategyParameters(in *api.StrategyParameters, out *StrategyParameters, s conversion.Scope) error { return autoConvert_api_StrategyParameters_To_v1alpha1_StrategyParameters(in, out, s) } - -func autoConvert_v1alpha1_TopologySpreadConstraint_To_api_TopologySpreadConstraint(in *TopologySpreadConstraint, out *api.TopologySpreadConstraint, s conversion.Scope) error { - out.MaxSkew = in.MaxSkew - out.TopologyKey = in.TopologyKey - out.LabelSelector = (*v1.LabelSelector)(unsafe.Pointer(in.LabelSelector)) - return nil -} - -// Convert_v1alpha1_TopologySpreadConstraint_To_api_TopologySpreadConstraint is an autogenerated conversion function. -func Convert_v1alpha1_TopologySpreadConstraint_To_api_TopologySpreadConstraint(in *TopologySpreadConstraint, out *api.TopologySpreadConstraint, s conversion.Scope) error { - return autoConvert_v1alpha1_TopologySpreadConstraint_To_api_TopologySpreadConstraint(in, out, s) -} - -func autoConvert_api_TopologySpreadConstraint_To_v1alpha1_TopologySpreadConstraint(in *api.TopologySpreadConstraint, out *TopologySpreadConstraint, s conversion.Scope) error { - out.MaxSkew = in.MaxSkew - out.TopologyKey = in.TopologyKey - out.LabelSelector = (*v1.LabelSelector)(unsafe.Pointer(in.LabelSelector)) - return nil -} - -// Convert_api_TopologySpreadConstraint_To_v1alpha1_TopologySpreadConstraint is an autogenerated conversion function. -func Convert_api_TopologySpreadConstraint_To_v1alpha1_TopologySpreadConstraint(in *api.TopologySpreadConstraint, out *TopologySpreadConstraint, s conversion.Scope) error { - return autoConvert_api_TopologySpreadConstraint_To_v1alpha1_TopologySpreadConstraint(in, out, s) -} diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 4c4acf537a..79ad51ade5 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,6 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -93,29 +92,6 @@ func (in *DeschedulerStrategy) DeepCopy() *DeschedulerStrategy { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NamespacedTopologySpreadConstraint) DeepCopyInto(out *NamespacedTopologySpreadConstraint) { - *out = *in - if in.TopologySpreadConstraints != nil { - in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints - *out = make([]TopologySpreadConstraint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespacedTopologySpreadConstraint. -func (in *NamespacedTopologySpreadConstraint) DeepCopy() *NamespacedTopologySpreadConstraint { - if in == nil { - return nil - } - out := new(NamespacedTopologySpreadConstraint) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Namespaces) DeepCopyInto(out *Namespaces) { *out = *in @@ -291,13 +267,6 @@ func (in *StrategyParameters) DeepCopyInto(out *StrategyParameters) { *out = new(int32) **out = **in } - if in.NamespacedTopologySpreadConstraints != nil { - in, out := &in.NamespacedTopologySpreadConstraints, &out.NamespacedTopologySpreadConstraints - *out = make([]NamespacedTopologySpreadConstraint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } return } @@ -310,24 +279,3 @@ func (in *StrategyParameters) DeepCopy() *StrategyParameters { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TopologySpreadConstraint) DeepCopyInto(out *TopologySpreadConstraint) { - *out = *in - if in.LabelSelector != nil { - in, out := &in.LabelSelector, &out.LabelSelector - *out = new(v1.LabelSelector) - (*in).DeepCopyInto(*out) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologySpreadConstraint. -func (in *TopologySpreadConstraint) DeepCopy() *TopologySpreadConstraint { - if in == nil { - return nil - } - out := new(TopologySpreadConstraint) - in.DeepCopyInto(out) - return out -} diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index d304627e65..17603cac07 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -21,7 +21,6 @@ limitations under the License. package api import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -93,29 +92,6 @@ func (in *DeschedulerStrategy) DeepCopy() *DeschedulerStrategy { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NamespacedTopologySpreadConstraint) DeepCopyInto(out *NamespacedTopologySpreadConstraint) { - *out = *in - if in.TopologySpreadConstraints != nil { - in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints - *out = make([]TopologySpreadConstraint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespacedTopologySpreadConstraint. -func (in *NamespacedTopologySpreadConstraint) DeepCopy() *NamespacedTopologySpreadConstraint { - if in == nil { - return nil - } - out := new(NamespacedTopologySpreadConstraint) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Namespaces) DeepCopyInto(out *Namespaces) { *out = *in @@ -291,13 +267,6 @@ func (in *StrategyParameters) DeepCopyInto(out *StrategyParameters) { *out = new(int32) **out = **in } - if in.NamespacedTopologySpreadConstraints != nil { - in, out := &in.NamespacedTopologySpreadConstraints, &out.NamespacedTopologySpreadConstraints - *out = make([]NamespacedTopologySpreadConstraint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } return } @@ -310,24 +279,3 @@ func (in *StrategyParameters) DeepCopy() *StrategyParameters { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TopologySpreadConstraint) DeepCopyInto(out *TopologySpreadConstraint) { - *out = *in - if in.LabelSelector != nil { - in, out := &in.LabelSelector, &out.LabelSelector - *out = new(v1.LabelSelector) - (*in).DeepCopyInto(*out) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologySpreadConstraint. -func (in *TopologySpreadConstraint) DeepCopy() *TopologySpreadConstraint { - if in == nil { - return nil - } - out := new(TopologySpreadConstraint) - in.DeepCopyInto(out) - return out -} diff --git a/pkg/descheduler/descheduler.go b/pkg/descheduler/descheduler.go index 776ef96198..f8b0f500f1 100644 --- a/pkg/descheduler/descheduler.go +++ b/pkg/descheduler/descheduler.go @@ -70,14 +70,14 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer sharedInformerFactory.WaitForCacheSync(stopChannel) strategyFuncs := map[string]strategyFunction{ - "RemoveDuplicates": strategies.RemoveDuplicatePods, - "LowNodeUtilization": strategies.LowNodeUtilization, - "RemovePodsViolatingInterPodAntiAffinity": strategies.RemovePodsViolatingInterPodAntiAffinity, - "RemovePodsViolatingNodeAffinity": strategies.RemovePodsViolatingNodeAffinity, - "RemovePodsViolatingNodeTaints": strategies.RemovePodsViolatingNodeTaints, - "RemovePodsHavingTooManyRestarts": strategies.RemovePodsHavingTooManyRestarts, - "PodLifeTime": strategies.PodLifeTime, - "TopologySpreadConstraint": strategies.TopologySpreadConstraint, + "RemoveDuplicates": strategies.RemoveDuplicatePods, + "LowNodeUtilization": strategies.LowNodeUtilization, + "RemovePodsViolatingInterPodAntiAffinity": strategies.RemovePodsViolatingInterPodAntiAffinity, + "RemovePodsViolatingNodeAffinity": strategies.RemovePodsViolatingNodeAffinity, + "RemovePodsViolatingNodeTaints": strategies.RemovePodsViolatingNodeTaints, + "RemovePodsHavingTooManyRestarts": strategies.RemovePodsHavingTooManyRestarts, + "PodLifeTime": strategies.PodLifeTime, + "RemovePodsViolatingTopologySpreadConstraint": strategies.RemovePodsViolatingTopologySpreadConstraint, } nodeSelector := rs.NodeSelector diff --git a/pkg/descheduler/strategies/topologyspreadyconstraint.go b/pkg/descheduler/strategies/topologyspreadconstraint.go similarity index 57% rename from pkg/descheduler/strategies/topologyspreadyconstraint.go rename to pkg/descheduler/strategies/topologyspreadconstraint.go index 75b06e73d6..7c97ff1b76 100644 --- a/pkg/descheduler/strategies/topologyspreadyconstraint.go +++ b/pkg/descheduler/strategies/topologyspreadconstraint.go @@ -20,26 +20,22 @@ import ( "context" "math" - "github.com/golang/glog" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" "sigs.k8s.io/descheduler/pkg/api" "sigs.k8s.io/descheduler/pkg/descheduler/evictions" podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" ) -func TopologySpreadConstraint(ctx context.Context, client clientset.Interface, strategy api.DeschedulerStrategy, nodes []*v1.Node, evictLocalStoragePods bool, podEvictor *evictions.PodEvictor) { - if !strategy.Enabled { - return - } - - glog.Infof("Found following parameters for TopologySpreadConstraint %v", strategy) - //evictPodsViolatingSpreadConstraints(ds.Client, policyGroupVersion, nodes, ds.DryRun, nodePodCount, strategy.Params.NamespacedTopologySpreadConstraints) - -} +// @seanmalloy notes: +// +// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#topologyspreadconstraint-v1-core +// https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +// https://github.com/kubernetes-sigs/descheduler/blob/master/pkg/descheduler/strategies/pod_antiaffinity.go // AntiAffinityTerm's topology key value used in predicate metadata type topologyPair struct { @@ -55,30 +51,84 @@ type topologyPairToPodSetMap map[topologyPair]podSet // for each topologyKey, what is the map of topologyKey pairs to pods type topologyKeyToTopologyPairSetMap map[string]topologyPairToPodSetMap -// Create a map of Node Name to v1.Node -// for each namespace for which there is Topology Constraint -// for each TopologySpreadyConstraint in that namespace -// find all evictable pods in that namespace -// for each evictable pod in that namespace -// If the pod matches this TopologySpreadConstraint LabelSelector -// If the pod nodeName is present in the nodeMap -// create a topoPair with key as this TopologySpreadConstraint.TopologyKey and value as this pod's Node Label Value for this TopologyKey -// add the pod with key as this topoPair -// find the min number of pods in any topoPair for this topologyKey -// iterate through all topoPairs for this topologyKey and diff currentPods -minPods <=maxSkew -// if diff > maxSkew, add this pod in the current bucket for eviction - -// We get N podLists , one for each TopologyKey in a given Namespace -// Find the pods which are common to each of these podLists -// Evict these Pods -// TODO: Break this down into UT'able functions -func evictPodsViolatingSpreadConstraints( +// TODO: remove this type? +type NamespacedTopologySpreadConstraint struct { + Namespace string + TopologySpreadConstraints []v1.TopologySpreadConstraint +} + +func RemovePodsViolatingTopologySpreadConstraint( + ctx context.Context, client clientset.Interface, - policyGroupVersion string, - nodes []*v1.Node, dryRun bool, - nodePodCount nodePodEvictedCount, - namespacedTopologySpreadConstraints []api.NamespacedTopologySpreadConstraint, + strategy api.DeschedulerStrategy, + nodes []*v1.Node, + podEvictor *evictions.PodEvictor, ) { + // START HERE + + // TODO: move code from this function here + // + //evictPodsViolatingSpreadConstraints(ds.Client, policyGroupVersion, nodes, ds.DryRun, nodePodCount, strategy.Params.NamespacedTopologySpreadConstraints) + + // contents of evictPodsViolatingSpreadConstraints + // + // Create a map of Node Name to v1.Node + // for each namespace for which there is Topology Constraint + // for each TopologySpreadyConstraint in that namespace + // find all evictable pods in that namespace + // for each evictable pod in that namespace + // If the pod matches this TopologySpreadConstraint LabelSelector + // If the pod nodeName is present in the nodeMap + // create a topoPair with key as this TopologySpreadConstraint.TopologyKey and value as this pod's Node Label Value for this TopologyKey + // add the pod with key as this topoPair + // find the min number of pods in any topoPair for this topologyKey + // iterate through all topoPairs for this topologyKey and diff currentPods -minPods <=maxSkew + // if diff > maxSkew, add this pod in the current bucket for eviction + + // We get N podLists , one for each TopologyKey in a given Namespace + // Find the pods which are common to each of these podLists + // Evict these Pods + + // @seanmalloy + // + // find all canidate pods and namespaces for eviction + namespacedTopologySpreadConstrainPods := make(map[string][]*v1.Pod) + /*for _, node := range nodes { + pods, err := podutil.ListPodsOnANode(ctx, client, node, podEvictor.IsEvictable) + if err != nil { + return + } + for _, pod := range pods { + if pod.Spec.TopologySpreadConstraints != nil { + namespacedTopologySpreadConstrainPods[pod.Namespace] = append(namespacedTopologySpreadConstrainPods[pod.Namespace], pod) + } + } + } */ + + // @seanmalloy need to iterate through each namespace and evict pods to rebalance + for ns, pods := range namespacedTopologySpreadConstrainPods { + klog.V(1).Infof("processing namespace: %v pods; %v", ns, pods) + podsToEvict := getPodsViolatingPodsTopologySpreadConstraint(pods) + podutil.SortPodsBasedOnPriorityLowToHigh(podsToEvict) + /* for _, pod := range podsToEvict { + success, err := podEvictor.EvictPod(ctx, pod, node, "TopologySpreadConstraint") // START figure out how to pass in node here to fix compile error + if success { + klog.V(1).Infof("Evicted pod: %#v because it violated PodTopologySpreadConstraint", pod.Name) + } + + if err != nil { + klog.Errorf("Error evicting pod: (%#v)", err) + break + } + } */ + } + // START HERE + return + // @seanmalloy need to calculate which pods should be evicted to "balance" based on topology domains + // + // need to implement the stubbed in function getPodsViolatingPodsTopologySpreadConstraint + + // @seanmalloy need to handle multiple TopologySpreadConstraints on single pod namespaceToTopologyKeySet := make(map[string]topologyKeyToTopologyPairSetMap) @@ -87,6 +137,12 @@ func evictPodsViolatingSpreadConstraints( for _, node := range nodes { nodeMap[node.Name] = node } + + // TODO: the below line just makes an empty slice, and the struct type NamespacedTopologySpreadConstraint will be removed + // + // namespacedTopologySpreadConstraints variable was previously passed in as a strategy + // parameter, but this is no longer a strategy parameter. + namespacedTopologySpreadConstraints := []NamespacedTopologySpreadConstraint{} for _, namespacedConstraint := range namespacedTopologySpreadConstraints { if namespaceToTopologyKeySet[namespacedConstraint.Namespace] == nil { namespaceToTopologyKeySet[namespacedConstraint.Namespace] = make(topologyKeyToTopologyPairSetMap) @@ -105,25 +161,30 @@ func evictPodsViolatingSpreadConstraints( namespaceToTopologyKeySet[namespacedConstraint.Namespace][topoConstraint.TopologyKey][pair] = make(podSet) } } - pods, err := podutil.ListEvictablePodsByNamespace(client, false, namespacedConstraint.Namespace) - if err != nil || len(pods) == 0 { - glog.V(1).Infof("No Evictable pods found for Namespace %v", namespacedConstraint.Namespace) + + // TODO: pods is hard coded to a slice of empty pods + // + //pods, err := podutil.ListEvictablePodsByNamespace(client, false, namespacedConstraint.Namespace) + pods := []*v1.Pod{} + //if err != nil || len(pods) == 0 { + if len(pods) == 0 { + klog.V(1).Infof("No Evictable pods found for Namespace %v", namespacedConstraint.Namespace) continue } for _, pod := range pods { - glog.V(2).Infof("Processing pod %v", pod.Name) + klog.V(2).Infof("Processing pod %v", pod.Name) // does this pod labels match the constraint label selector selector, err := metav1.LabelSelectorAsSelector(topoConstraint.LabelSelector) if err != nil { - glog.V(2).Infof("Pod Labels dont match for %v", pod.Name) + klog.V(2).Infof("Pod Labels dont match for %v", pod.Name) continue } if !selector.Matches(labels.Set(pod.Labels)) { - glog.V(2).Infof("Pod Labels dont match for %v", pod.Name) + klog.V(2).Infof("Pod Labels dont match for %v", pod.Name) continue } - glog.V(1).Infof("Pod %v matched labels", pod.Name) + klog.V(1).Infof("Pod %v matched labels", pod.Name) // TODO: Need to determine if the topokey already present in the node or not if pod.Spec.NodeName == "" { continue @@ -131,7 +192,7 @@ func evictPodsViolatingSpreadConstraints( // see of this pods NodeName exists in the candidates nodes, else ignore _, ok := nodeMap[pod.Spec.NodeName] if !ok { - glog.V(2).Infof("Found a node %v in pod %v, which is not present in our map, ignoring it...", pod.Spec.NodeName, pod.Name) + klog.V(2).Infof("Found a node %v in pod %v, which is not present in our map, ignoring it...", pod.Spec.NodeName, pod.Name) continue } pair := topologyPair{key: topoConstraint.TopologyKey, value: nodeMap[pod.Spec.NodeName].Labels[topoConstraint.TopologyKey]} @@ -140,7 +201,7 @@ func evictPodsViolatingSpreadConstraints( namespaceToTopologyKeySet[namespacedConstraint.Namespace][topoConstraint.TopologyKey][pair] = make(podSet) } namespaceToTopologyKeySet[namespacedConstraint.Namespace][topoConstraint.TopologyKey][pair][pod] = struct{}{} - glog.V(2).Infof("Topo Pair %v, Count %v", pair, len(namespaceToTopologyKeySet[namespacedConstraint.Namespace][topoConstraint.TopologyKey][pair])) + klog.V(2).Infof("Topo Pair %v, Count %v", pair, len(namespaceToTopologyKeySet[namespacedConstraint.Namespace][topoConstraint.TopologyKey][pair])) } } @@ -160,11 +221,11 @@ func evictPodsViolatingSpreadConstraints( topologyPairToPods := namespaceToTopologyKeySet[namespacedConstraint.Namespace][topoConstraint.TopologyKey] for pair, v := range topologyPairToPods { podsInTopo := len(v) - glog.V(1).Infof("Min Pods in Any Pair %v, pair %v, PodCount %v", minPodsForGivenTopo, pair, podsInTopo) + klog.V(1).Infof("Min Pods in Any Pair %v, pair %v, PodCount %v", minPodsForGivenTopo, pair, podsInTopo) if int32(podsInTopo-minPodsForGivenTopo) > topoConstraint.MaxSkew { countToEvict := int32(podsInTopo-minPodsForGivenTopo) - topoConstraint.MaxSkew - glog.V(1).Infof("pair %v, Count to evict %v", pair, countToEvict) + klog.V(1).Infof("pair %v, Count to evict %v", pair, countToEvict) podsListToEvict := getPodsToEvict(countToEvict, v) allPodsToEvictPerTopoKey[topoConstraint.TopologyKey] = append(allPodsToEvictPerTopoKey[topoConstraint.TopologyKey], podsListToEvict...) @@ -186,25 +247,36 @@ func evictPodsViolatingSpreadConstraints( // defer the decision as late as possible to cause less schedulings for topoKey, podList := range allPodsToEvictPerTopoKey { - glog.V(1).Infof("Total pods to evict in TopoKey %v is %v", topoKey, len(podList)) - evictPodsSimple(client, podList, policyGroupVersion, dryRun) + klog.V(1).Infof("Total pods to evict in TopoKey %v is %v", topoKey, len(podList)) + //evictPodsSimple(client, podList, policyGroupVersion, dryRun) + for _, pod := range podList { + // TODO: node variable not defined + // + //success, err := podEvictor.EvictPod(ctx, pod, node) + //if success { + //klog.V(1).Infof("Evicted pod: %#v because it violate pod topology constraint", pod.Name) + //} + + //if err != nil { + // klog.Errorf("Error evicting pod: (%#v)", err) + // break + //} + klog.V(1).Infof("Evicted pod: %#v because it violate pod topology constraint", pod.Name) + } } } } -func evictPodsSimple(client clientset.Interface, podsListToEvict []*v1.Pod, policyGroupVersion string, dryRun bool) { - for _, podToEvict := range podsListToEvict { - glog.V(1).Infof("Evicting pods %v", podToEvict.Name) - success, err := evictions.EvictPod(client, podToEvict, policyGroupVersion, dryRun) - if !success { - glog.Infof("Error when evicting pod: %#v (%#v)\n", podToEvict.Name, err) - } else { - - } - } +// @seanmalloy +// +// TODO: this should find pods that are not balanced and return them. Try +// to reuse logic from previous code to write this function. +func getPodsViolatingPodsTopologySpreadConstraint(pods []*v1.Pod) []*v1.Pod { + return pods } +// TODO: this function is not called func intersectAllPodsList(allPodsToEvictPerTopoKey map[string][]*v1.Pod) []*v1.Pod { // increment each pod's count by 1 // if the pod count reaches the number of topoKeys, it should be evicted @@ -212,7 +284,7 @@ func intersectAllPodsList(allPodsToEvictPerTopoKey map[string][]*v1.Pod) []*v1.P finalList := []*v1.Pod{} totalTopoKeys := len(allPodsToEvictPerTopoKey) - glog.V(1).Infof("Total topokeys found %v", totalTopoKeys) + klog.V(1).Infof("Total topokeys found %v", totalTopoKeys) for _, podList := range allPodsToEvictPerTopoKey { for _, pod := range podList { key := pod.Name + "-" + pod.Namespace @@ -239,6 +311,7 @@ func getPodsToEvict(countToEvict int32, podMap map[*v1.Pod]struct{}) []*v1.Pod { return podList } +// TODO: this function is not called func addTopologyPair(topoMap map[topologyPair]podSet, pair topologyPair, pod *v1.Pod) { if topoMap[pair] == nil { topoMap[pair] = make(map[*v1.Pod]struct{}) diff --git a/vendor/github.com/golang/glog/LICENSE b/vendor/github.com/golang/glog/LICENSE deleted file mode 100644 index 37ec93a14f..0000000000 --- a/vendor/github.com/golang/glog/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/golang/glog/README b/vendor/github.com/golang/glog/README deleted file mode 100644 index 387b4eb689..0000000000 --- a/vendor/github.com/golang/glog/README +++ /dev/null @@ -1,44 +0,0 @@ -glog -==== - -Leveled execution logs for Go. - -This is an efficient pure Go implementation of leveled logs in the -manner of the open source C++ package - https://github.com/google/glog - -By binding methods to booleans it is possible to use the log package -without paying the expense of evaluating the arguments to the log. -Through the -vmodule flag, the package also provides fine-grained -control over logging at the file level. - -The comment from glog.go introduces the ideas: - - Package glog implements logging analogous to the Google-internal - C++ INFO/ERROR/V setup. It provides functions Info, Warning, - Error, Fatal, plus formatting variants such as Infof. It - also provides V-style logging controlled by the -v and - -vmodule=file=2 flags. - - Basic examples: - - glog.Info("Prepare to repel boarders") - - glog.Fatalf("Initialization failed: %s", err) - - See the documentation for the V function for an explanation - of these examples: - - if glog.V(2) { - glog.Info("Starting transaction...") - } - - glog.V(2).Infoln("Processed", nItems, "elements") - - -The repository contains an open source version of the log package -used inside Google. The master copy of the source lives inside -Google, not here. The code in this repo is for export only and is not itself -under development. Feature requests will be ignored. - -Send bug reports to golang-nuts@googlegroups.com. diff --git a/vendor/github.com/golang/glog/glog.go b/vendor/github.com/golang/glog/glog.go deleted file mode 100644 index 54bd7afdca..0000000000 --- a/vendor/github.com/golang/glog/glog.go +++ /dev/null @@ -1,1180 +0,0 @@ -// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/ -// -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package glog implements logging analogous to the Google-internal C++ INFO/ERROR/V setup. -// It provides functions Info, Warning, Error, Fatal, plus formatting variants such as -// Infof. It also provides V-style logging controlled by the -v and -vmodule=file=2 flags. -// -// Basic examples: -// -// glog.Info("Prepare to repel boarders") -// -// glog.Fatalf("Initialization failed: %s", err) -// -// See the documentation for the V function for an explanation of these examples: -// -// if glog.V(2) { -// glog.Info("Starting transaction...") -// } -// -// glog.V(2).Infoln("Processed", nItems, "elements") -// -// Log output is buffered and written periodically using Flush. Programs -// should call Flush before exiting to guarantee all log output is written. -// -// By default, all log statements write to files in a temporary directory. -// This package provides several flags that modify this behavior. -// As a result, flag.Parse must be called before any logging is done. -// -// -logtostderr=false -// Logs are written to standard error instead of to files. -// -alsologtostderr=false -// Logs are written to standard error as well as to files. -// -stderrthreshold=ERROR -// Log events at or above this severity are logged to standard -// error as well as to files. -// -log_dir="" -// Log files will be written to this directory instead of the -// default temporary directory. -// -// Other flags provide aids to debugging. -// -// -log_backtrace_at="" -// When set to a file and line number holding a logging statement, -// such as -// -log_backtrace_at=gopherflakes.go:234 -// a stack trace will be written to the Info log whenever execution -// hits that statement. (Unlike with -vmodule, the ".go" must be -// present.) -// -v=0 -// Enable V-leveled logging at the specified level. -// -vmodule="" -// The syntax of the argument is a comma-separated list of pattern=N, -// where pattern is a literal file name (minus the ".go" suffix) or -// "glob" pattern and N is a V level. For instance, -// -vmodule=gopher*=3 -// sets the V level to 3 in all Go files whose names begin "gopher". -// -package glog - -import ( - "bufio" - "bytes" - "errors" - "flag" - "fmt" - "io" - stdLog "log" - "os" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" -) - -// severity identifies the sort of log: info, warning etc. It also implements -// the flag.Value interface. The -stderrthreshold flag is of type severity and -// should be modified only through the flag.Value interface. The values match -// the corresponding constants in C++. -type severity int32 // sync/atomic int32 - -// These constants identify the log levels in order of increasing severity. -// A message written to a high-severity log file is also written to each -// lower-severity log file. -const ( - infoLog severity = iota - warningLog - errorLog - fatalLog - numSeverity = 4 -) - -const severityChar = "IWEF" - -var severityName = []string{ - infoLog: "INFO", - warningLog: "WARNING", - errorLog: "ERROR", - fatalLog: "FATAL", -} - -// get returns the value of the severity. -func (s *severity) get() severity { - return severity(atomic.LoadInt32((*int32)(s))) -} - -// set sets the value of the severity. -func (s *severity) set(val severity) { - atomic.StoreInt32((*int32)(s), int32(val)) -} - -// String is part of the flag.Value interface. -func (s *severity) String() string { - return strconv.FormatInt(int64(*s), 10) -} - -// Get is part of the flag.Value interface. -func (s *severity) Get() interface{} { - return *s -} - -// Set is part of the flag.Value interface. -func (s *severity) Set(value string) error { - var threshold severity - // Is it a known name? - if v, ok := severityByName(value); ok { - threshold = v - } else { - v, err := strconv.Atoi(value) - if err != nil { - return err - } - threshold = severity(v) - } - logging.stderrThreshold.set(threshold) - return nil -} - -func severityByName(s string) (severity, bool) { - s = strings.ToUpper(s) - for i, name := range severityName { - if name == s { - return severity(i), true - } - } - return 0, false -} - -// OutputStats tracks the number of output lines and bytes written. -type OutputStats struct { - lines int64 - bytes int64 -} - -// Lines returns the number of lines written. -func (s *OutputStats) Lines() int64 { - return atomic.LoadInt64(&s.lines) -} - -// Bytes returns the number of bytes written. -func (s *OutputStats) Bytes() int64 { - return atomic.LoadInt64(&s.bytes) -} - -// Stats tracks the number of lines of output and number of bytes -// per severity level. Values must be read with atomic.LoadInt64. -var Stats struct { - Info, Warning, Error OutputStats -} - -var severityStats = [numSeverity]*OutputStats{ - infoLog: &Stats.Info, - warningLog: &Stats.Warning, - errorLog: &Stats.Error, -} - -// Level is exported because it appears in the arguments to V and is -// the type of the v flag, which can be set programmatically. -// It's a distinct type because we want to discriminate it from logType. -// Variables of type level are only changed under logging.mu. -// The -v flag is read only with atomic ops, so the state of the logging -// module is consistent. - -// Level is treated as a sync/atomic int32. - -// Level specifies a level of verbosity for V logs. *Level implements -// flag.Value; the -v flag is of type Level and should be modified -// only through the flag.Value interface. -type Level int32 - -// get returns the value of the Level. -func (l *Level) get() Level { - return Level(atomic.LoadInt32((*int32)(l))) -} - -// set sets the value of the Level. -func (l *Level) set(val Level) { - atomic.StoreInt32((*int32)(l), int32(val)) -} - -// String is part of the flag.Value interface. -func (l *Level) String() string { - return strconv.FormatInt(int64(*l), 10) -} - -// Get is part of the flag.Value interface. -func (l *Level) Get() interface{} { - return *l -} - -// Set is part of the flag.Value interface. -func (l *Level) Set(value string) error { - v, err := strconv.Atoi(value) - if err != nil { - return err - } - logging.mu.Lock() - defer logging.mu.Unlock() - logging.setVState(Level(v), logging.vmodule.filter, false) - return nil -} - -// moduleSpec represents the setting of the -vmodule flag. -type moduleSpec struct { - filter []modulePat -} - -// modulePat contains a filter for the -vmodule flag. -// It holds a verbosity level and a file pattern to match. -type modulePat struct { - pattern string - literal bool // The pattern is a literal string - level Level -} - -// match reports whether the file matches the pattern. It uses a string -// comparison if the pattern contains no metacharacters. -func (m *modulePat) match(file string) bool { - if m.literal { - return file == m.pattern - } - match, _ := filepath.Match(m.pattern, file) - return match -} - -func (m *moduleSpec) String() string { - // Lock because the type is not atomic. TODO: clean this up. - logging.mu.Lock() - defer logging.mu.Unlock() - var b bytes.Buffer - for i, f := range m.filter { - if i > 0 { - b.WriteRune(',') - } - fmt.Fprintf(&b, "%s=%d", f.pattern, f.level) - } - return b.String() -} - -// Get is part of the (Go 1.2) flag.Getter interface. It always returns nil for this flag type since the -// struct is not exported. -func (m *moduleSpec) Get() interface{} { - return nil -} - -var errVmoduleSyntax = errors.New("syntax error: expect comma-separated list of filename=N") - -// Syntax: -vmodule=recordio=2,file=1,gfs*=3 -func (m *moduleSpec) Set(value string) error { - var filter []modulePat - for _, pat := range strings.Split(value, ",") { - if len(pat) == 0 { - // Empty strings such as from a trailing comma can be ignored. - continue - } - patLev := strings.Split(pat, "=") - if len(patLev) != 2 || len(patLev[0]) == 0 || len(patLev[1]) == 0 { - return errVmoduleSyntax - } - pattern := patLev[0] - v, err := strconv.Atoi(patLev[1]) - if err != nil { - return errors.New("syntax error: expect comma-separated list of filename=N") - } - if v < 0 { - return errors.New("negative value for vmodule level") - } - if v == 0 { - continue // Ignore. It's harmless but no point in paying the overhead. - } - // TODO: check syntax of filter? - filter = append(filter, modulePat{pattern, isLiteral(pattern), Level(v)}) - } - logging.mu.Lock() - defer logging.mu.Unlock() - logging.setVState(logging.verbosity, filter, true) - return nil -} - -// isLiteral reports whether the pattern is a literal string, that is, has no metacharacters -// that require filepath.Match to be called to match the pattern. -func isLiteral(pattern string) bool { - return !strings.ContainsAny(pattern, `\*?[]`) -} - -// traceLocation represents the setting of the -log_backtrace_at flag. -type traceLocation struct { - file string - line int -} - -// isSet reports whether the trace location has been specified. -// logging.mu is held. -func (t *traceLocation) isSet() bool { - return t.line > 0 -} - -// match reports whether the specified file and line matches the trace location. -// The argument file name is the full path, not the basename specified in the flag. -// logging.mu is held. -func (t *traceLocation) match(file string, line int) bool { - if t.line != line { - return false - } - if i := strings.LastIndex(file, "/"); i >= 0 { - file = file[i+1:] - } - return t.file == file -} - -func (t *traceLocation) String() string { - // Lock because the type is not atomic. TODO: clean this up. - logging.mu.Lock() - defer logging.mu.Unlock() - return fmt.Sprintf("%s:%d", t.file, t.line) -} - -// Get is part of the (Go 1.2) flag.Getter interface. It always returns nil for this flag type since the -// struct is not exported -func (t *traceLocation) Get() interface{} { - return nil -} - -var errTraceSyntax = errors.New("syntax error: expect file.go:234") - -// Syntax: -log_backtrace_at=gopherflakes.go:234 -// Note that unlike vmodule the file extension is included here. -func (t *traceLocation) Set(value string) error { - if value == "" { - // Unset. - t.line = 0 - t.file = "" - } - fields := strings.Split(value, ":") - if len(fields) != 2 { - return errTraceSyntax - } - file, line := fields[0], fields[1] - if !strings.Contains(file, ".") { - return errTraceSyntax - } - v, err := strconv.Atoi(line) - if err != nil { - return errTraceSyntax - } - if v <= 0 { - return errors.New("negative or zero value for level") - } - logging.mu.Lock() - defer logging.mu.Unlock() - t.line = v - t.file = file - return nil -} - -// flushSyncWriter is the interface satisfied by logging destinations. -type flushSyncWriter interface { - Flush() error - Sync() error - io.Writer -} - -func init() { - flag.BoolVar(&logging.toStderr, "logtostderr", false, "log to standard error instead of files") - flag.BoolVar(&logging.alsoToStderr, "alsologtostderr", false, "log to standard error as well as files") - flag.Var(&logging.verbosity, "v", "log level for V logs") - flag.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr") - flag.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging") - flag.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace") - - // Default stderrThreshold is ERROR. - logging.stderrThreshold = errorLog - - logging.setVState(0, nil, false) - go logging.flushDaemon() -} - -// Flush flushes all pending log I/O. -func Flush() { - logging.lockAndFlushAll() -} - -// loggingT collects all the global state of the logging setup. -type loggingT struct { - // Boolean flags. Not handled atomically because the flag.Value interface - // does not let us avoid the =true, and that shorthand is necessary for - // compatibility. TODO: does this matter enough to fix? Seems unlikely. - toStderr bool // The -logtostderr flag. - alsoToStderr bool // The -alsologtostderr flag. - - // Level flag. Handled atomically. - stderrThreshold severity // The -stderrthreshold flag. - - // freeList is a list of byte buffers, maintained under freeListMu. - freeList *buffer - // freeListMu maintains the free list. It is separate from the main mutex - // so buffers can be grabbed and printed to without holding the main lock, - // for better parallelization. - freeListMu sync.Mutex - - // mu protects the remaining elements of this structure and is - // used to synchronize logging. - mu sync.Mutex - // file holds writer for each of the log types. - file [numSeverity]flushSyncWriter - // pcs is used in V to avoid an allocation when computing the caller's PC. - pcs [1]uintptr - // vmap is a cache of the V Level for each V() call site, identified by PC. - // It is wiped whenever the vmodule flag changes state. - vmap map[uintptr]Level - // filterLength stores the length of the vmodule filter chain. If greater - // than zero, it means vmodule is enabled. It may be read safely - // using sync.LoadInt32, but is only modified under mu. - filterLength int32 - // traceLocation is the state of the -log_backtrace_at flag. - traceLocation traceLocation - // These flags are modified only under lock, although verbosity may be fetched - // safely using atomic.LoadInt32. - vmodule moduleSpec // The state of the -vmodule flag. - verbosity Level // V logging level, the value of the -v flag/ -} - -// buffer holds a byte Buffer for reuse. The zero value is ready for use. -type buffer struct { - bytes.Buffer - tmp [64]byte // temporary byte array for creating headers. - next *buffer -} - -var logging loggingT - -// setVState sets a consistent state for V logging. -// l.mu is held. -func (l *loggingT) setVState(verbosity Level, filter []modulePat, setFilter bool) { - // Turn verbosity off so V will not fire while we are in transition. - logging.verbosity.set(0) - // Ditto for filter length. - atomic.StoreInt32(&logging.filterLength, 0) - - // Set the new filters and wipe the pc->Level map if the filter has changed. - if setFilter { - logging.vmodule.filter = filter - logging.vmap = make(map[uintptr]Level) - } - - // Things are consistent now, so enable filtering and verbosity. - // They are enabled in order opposite to that in V. - atomic.StoreInt32(&logging.filterLength, int32(len(filter))) - logging.verbosity.set(verbosity) -} - -// getBuffer returns a new, ready-to-use buffer. -func (l *loggingT) getBuffer() *buffer { - l.freeListMu.Lock() - b := l.freeList - if b != nil { - l.freeList = b.next - } - l.freeListMu.Unlock() - if b == nil { - b = new(buffer) - } else { - b.next = nil - b.Reset() - } - return b -} - -// putBuffer returns a buffer to the free list. -func (l *loggingT) putBuffer(b *buffer) { - if b.Len() >= 256 { - // Let big buffers die a natural death. - return - } - l.freeListMu.Lock() - b.next = l.freeList - l.freeList = b - l.freeListMu.Unlock() -} - -var timeNow = time.Now // Stubbed out for testing. - -/* -header formats a log header as defined by the C++ implementation. -It returns a buffer containing the formatted header and the user's file and line number. -The depth specifies how many stack frames above lives the source line to be identified in the log message. - -Log lines have this form: - Lmmdd hh:mm:ss.uuuuuu threadid file:line] msg... -where the fields are defined as follows: - L A single character, representing the log level (eg 'I' for INFO) - mm The month (zero padded; ie May is '05') - dd The day (zero padded) - hh:mm:ss.uuuuuu Time in hours, minutes and fractional seconds - threadid The space-padded thread ID as returned by GetTID() - file The file name - line The line number - msg The user-supplied message -*/ -func (l *loggingT) header(s severity, depth int) (*buffer, string, int) { - _, file, line, ok := runtime.Caller(3 + depth) - if !ok { - file = "???" - line = 1 - } else { - slash := strings.LastIndex(file, "/") - if slash >= 0 { - file = file[slash+1:] - } - } - return l.formatHeader(s, file, line), file, line -} - -// formatHeader formats a log header using the provided file name and line number. -func (l *loggingT) formatHeader(s severity, file string, line int) *buffer { - now := timeNow() - if line < 0 { - line = 0 // not a real line number, but acceptable to someDigits - } - if s > fatalLog { - s = infoLog // for safety. - } - buf := l.getBuffer() - - // Avoid Fprintf, for speed. The format is so simple that we can do it quickly by hand. - // It's worth about 3X. Fprintf is hard. - _, month, day := now.Date() - hour, minute, second := now.Clock() - // Lmmdd hh:mm:ss.uuuuuu threadid file:line] - buf.tmp[0] = severityChar[s] - buf.twoDigits(1, int(month)) - buf.twoDigits(3, day) - buf.tmp[5] = ' ' - buf.twoDigits(6, hour) - buf.tmp[8] = ':' - buf.twoDigits(9, minute) - buf.tmp[11] = ':' - buf.twoDigits(12, second) - buf.tmp[14] = '.' - buf.nDigits(6, 15, now.Nanosecond()/1000, '0') - buf.tmp[21] = ' ' - buf.nDigits(7, 22, pid, ' ') // TODO: should be TID - buf.tmp[29] = ' ' - buf.Write(buf.tmp[:30]) - buf.WriteString(file) - buf.tmp[0] = ':' - n := buf.someDigits(1, line) - buf.tmp[n+1] = ']' - buf.tmp[n+2] = ' ' - buf.Write(buf.tmp[:n+3]) - return buf -} - -// Some custom tiny helper functions to print the log header efficiently. - -const digits = "0123456789" - -// twoDigits formats a zero-prefixed two-digit integer at buf.tmp[i]. -func (buf *buffer) twoDigits(i, d int) { - buf.tmp[i+1] = digits[d%10] - d /= 10 - buf.tmp[i] = digits[d%10] -} - -// nDigits formats an n-digit integer at buf.tmp[i], -// padding with pad on the left. -// It assumes d >= 0. -func (buf *buffer) nDigits(n, i, d int, pad byte) { - j := n - 1 - for ; j >= 0 && d > 0; j-- { - buf.tmp[i+j] = digits[d%10] - d /= 10 - } - for ; j >= 0; j-- { - buf.tmp[i+j] = pad - } -} - -// someDigits formats a zero-prefixed variable-width integer at buf.tmp[i]. -func (buf *buffer) someDigits(i, d int) int { - // Print into the top, then copy down. We know there's space for at least - // a 10-digit number. - j := len(buf.tmp) - for { - j-- - buf.tmp[j] = digits[d%10] - d /= 10 - if d == 0 { - break - } - } - return copy(buf.tmp[i:], buf.tmp[j:]) -} - -func (l *loggingT) println(s severity, args ...interface{}) { - buf, file, line := l.header(s, 0) - fmt.Fprintln(buf, args...) - l.output(s, buf, file, line, false) -} - -func (l *loggingT) print(s severity, args ...interface{}) { - l.printDepth(s, 1, args...) -} - -func (l *loggingT) printDepth(s severity, depth int, args ...interface{}) { - buf, file, line := l.header(s, depth) - fmt.Fprint(buf, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, false) -} - -func (l *loggingT) printf(s severity, format string, args ...interface{}) { - buf, file, line := l.header(s, 0) - fmt.Fprintf(buf, format, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, false) -} - -// printWithFileLine behaves like print but uses the provided file and line number. If -// alsoLogToStderr is true, the log message always appears on standard error; it -// will also appear in the log file unless --logtostderr is set. -func (l *loggingT) printWithFileLine(s severity, file string, line int, alsoToStderr bool, args ...interface{}) { - buf := l.formatHeader(s, file, line) - fmt.Fprint(buf, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, alsoToStderr) -} - -// output writes the data to the log files and releases the buffer. -func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoToStderr bool) { - l.mu.Lock() - if l.traceLocation.isSet() { - if l.traceLocation.match(file, line) { - buf.Write(stacks(false)) - } - } - data := buf.Bytes() - if !flag.Parsed() { - os.Stderr.Write([]byte("ERROR: logging before flag.Parse: ")) - os.Stderr.Write(data) - } else if l.toStderr { - os.Stderr.Write(data) - } else { - if alsoToStderr || l.alsoToStderr || s >= l.stderrThreshold.get() { - os.Stderr.Write(data) - } - if l.file[s] == nil { - if err := l.createFiles(s); err != nil { - os.Stderr.Write(data) // Make sure the message appears somewhere. - l.exit(err) - } - } - switch s { - case fatalLog: - l.file[fatalLog].Write(data) - fallthrough - case errorLog: - l.file[errorLog].Write(data) - fallthrough - case warningLog: - l.file[warningLog].Write(data) - fallthrough - case infoLog: - l.file[infoLog].Write(data) - } - } - if s == fatalLog { - // If we got here via Exit rather than Fatal, print no stacks. - if atomic.LoadUint32(&fatalNoStacks) > 0 { - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(1) - } - // Dump all goroutine stacks before exiting. - // First, make sure we see the trace for the current goroutine on standard error. - // If -logtostderr has been specified, the loop below will do that anyway - // as the first stack in the full dump. - if !l.toStderr { - os.Stderr.Write(stacks(false)) - } - // Write the stack trace for all goroutines to the files. - trace := stacks(true) - logExitFunc = func(error) {} // If we get a write error, we'll still exit below. - for log := fatalLog; log >= infoLog; log-- { - if f := l.file[log]; f != nil { // Can be nil if -logtostderr is set. - f.Write(trace) - } - } - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(255) // C++ uses -1, which is silly because it's anded with 255 anyway. - } - l.putBuffer(buf) - l.mu.Unlock() - if stats := severityStats[s]; stats != nil { - atomic.AddInt64(&stats.lines, 1) - atomic.AddInt64(&stats.bytes, int64(len(data))) - } -} - -// timeoutFlush calls Flush and returns when it completes or after timeout -// elapses, whichever happens first. This is needed because the hooks invoked -// by Flush may deadlock when glog.Fatal is called from a hook that holds -// a lock. -func timeoutFlush(timeout time.Duration) { - done := make(chan bool, 1) - go func() { - Flush() // calls logging.lockAndFlushAll() - done <- true - }() - select { - case <-done: - case <-time.After(timeout): - fmt.Fprintln(os.Stderr, "glog: Flush took longer than", timeout) - } -} - -// stacks is a wrapper for runtime.Stack that attempts to recover the data for all goroutines. -func stacks(all bool) []byte { - // We don't know how big the traces are, so grow a few times if they don't fit. Start large, though. - n := 10000 - if all { - n = 100000 - } - var trace []byte - for i := 0; i < 5; i++ { - trace = make([]byte, n) - nbytes := runtime.Stack(trace, all) - if nbytes < len(trace) { - return trace[:nbytes] - } - n *= 2 - } - return trace -} - -// logExitFunc provides a simple mechanism to override the default behavior -// of exiting on error. Used in testing and to guarantee we reach a required exit -// for fatal logs. Instead, exit could be a function rather than a method but that -// would make its use clumsier. -var logExitFunc func(error) - -// exit is called if there is trouble creating or writing log files. -// It flushes the logs and exits the program; there's no point in hanging around. -// l.mu is held. -func (l *loggingT) exit(err error) { - fmt.Fprintf(os.Stderr, "log: exiting because of error: %s\n", err) - // If logExitFunc is set, we do that instead of exiting. - if logExitFunc != nil { - logExitFunc(err) - return - } - l.flushAll() - os.Exit(2) -} - -// syncBuffer joins a bufio.Writer to its underlying file, providing access to the -// file's Sync method and providing a wrapper for the Write method that provides log -// file rotation. There are conflicting methods, so the file cannot be embedded. -// l.mu is held for all its methods. -type syncBuffer struct { - logger *loggingT - *bufio.Writer - file *os.File - sev severity - nbytes uint64 // The number of bytes written to this file -} - -func (sb *syncBuffer) Sync() error { - return sb.file.Sync() -} - -func (sb *syncBuffer) Write(p []byte) (n int, err error) { - if sb.nbytes+uint64(len(p)) >= MaxSize { - if err := sb.rotateFile(time.Now()); err != nil { - sb.logger.exit(err) - } - } - n, err = sb.Writer.Write(p) - sb.nbytes += uint64(n) - if err != nil { - sb.logger.exit(err) - } - return -} - -// rotateFile closes the syncBuffer's file and starts a new one. -func (sb *syncBuffer) rotateFile(now time.Time) error { - if sb.file != nil { - sb.Flush() - sb.file.Close() - } - var err error - sb.file, _, err = create(severityName[sb.sev], now) - sb.nbytes = 0 - if err != nil { - return err - } - - sb.Writer = bufio.NewWriterSize(sb.file, bufferSize) - - // Write header. - var buf bytes.Buffer - fmt.Fprintf(&buf, "Log file created at: %s\n", now.Format("2006/01/02 15:04:05")) - fmt.Fprintf(&buf, "Running on machine: %s\n", host) - fmt.Fprintf(&buf, "Binary: Built with %s %s for %s/%s\n", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH) - fmt.Fprintf(&buf, "Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg\n") - n, err := sb.file.Write(buf.Bytes()) - sb.nbytes += uint64(n) - return err -} - -// bufferSize sizes the buffer associated with each log file. It's large -// so that log records can accumulate without the logging thread blocking -// on disk I/O. The flushDaemon will block instead. -const bufferSize = 256 * 1024 - -// createFiles creates all the log files for severity from sev down to infoLog. -// l.mu is held. -func (l *loggingT) createFiles(sev severity) error { - now := time.Now() - // Files are created in decreasing severity order, so as soon as we find one - // has already been created, we can stop. - for s := sev; s >= infoLog && l.file[s] == nil; s-- { - sb := &syncBuffer{ - logger: l, - sev: s, - } - if err := sb.rotateFile(now); err != nil { - return err - } - l.file[s] = sb - } - return nil -} - -const flushInterval = 30 * time.Second - -// flushDaemon periodically flushes the log file buffers. -func (l *loggingT) flushDaemon() { - for _ = range time.NewTicker(flushInterval).C { - l.lockAndFlushAll() - } -} - -// lockAndFlushAll is like flushAll but locks l.mu first. -func (l *loggingT) lockAndFlushAll() { - l.mu.Lock() - l.flushAll() - l.mu.Unlock() -} - -// flushAll flushes all the logs and attempts to "sync" their data to disk. -// l.mu is held. -func (l *loggingT) flushAll() { - // Flush from fatal down, in case there's trouble flushing. - for s := fatalLog; s >= infoLog; s-- { - file := l.file[s] - if file != nil { - file.Flush() // ignore error - file.Sync() // ignore error - } - } -} - -// CopyStandardLogTo arranges for messages written to the Go "log" package's -// default logs to also appear in the Google logs for the named and lower -// severities. Subsequent changes to the standard log's default output location -// or format may break this behavior. -// -// Valid names are "INFO", "WARNING", "ERROR", and "FATAL". If the name is not -// recognized, CopyStandardLogTo panics. -func CopyStandardLogTo(name string) { - sev, ok := severityByName(name) - if !ok { - panic(fmt.Sprintf("log.CopyStandardLogTo(%q): unrecognized severity name", name)) - } - // Set a log format that captures the user's file and line: - // d.go:23: message - stdLog.SetFlags(stdLog.Lshortfile) - stdLog.SetOutput(logBridge(sev)) -} - -// logBridge provides the Write method that enables CopyStandardLogTo to connect -// Go's standard logs to the logs provided by this package. -type logBridge severity - -// Write parses the standard logging line and passes its components to the -// logger for severity(lb). -func (lb logBridge) Write(b []byte) (n int, err error) { - var ( - file = "???" - line = 1 - text string - ) - // Split "d.go:23: message" into "d.go", "23", and "message". - if parts := bytes.SplitN(b, []byte{':'}, 3); len(parts) != 3 || len(parts[0]) < 1 || len(parts[2]) < 1 { - text = fmt.Sprintf("bad log format: %s", b) - } else { - file = string(parts[0]) - text = string(parts[2][1:]) // skip leading space - line, err = strconv.Atoi(string(parts[1])) - if err != nil { - text = fmt.Sprintf("bad line number: %s", b) - line = 1 - } - } - // printWithFileLine with alsoToStderr=true, so standard log messages - // always appear on standard error. - logging.printWithFileLine(severity(lb), file, line, true, text) - return len(b), nil -} - -// setV computes and remembers the V level for a given PC -// when vmodule is enabled. -// File pattern matching takes the basename of the file, stripped -// of its .go suffix, and uses filepath.Match, which is a little more -// general than the *? matching used in C++. -// l.mu is held. -func (l *loggingT) setV(pc uintptr) Level { - fn := runtime.FuncForPC(pc) - file, _ := fn.FileLine(pc) - // The file is something like /a/b/c/d.go. We want just the d. - if strings.HasSuffix(file, ".go") { - file = file[:len(file)-3] - } - if slash := strings.LastIndex(file, "/"); slash >= 0 { - file = file[slash+1:] - } - for _, filter := range l.vmodule.filter { - if filter.match(file) { - l.vmap[pc] = filter.level - return filter.level - } - } - l.vmap[pc] = 0 - return 0 -} - -// Verbose is a boolean type that implements Infof (like Printf) etc. -// See the documentation of V for more information. -type Verbose bool - -// V reports whether verbosity at the call site is at least the requested level. -// The returned value is a boolean of type Verbose, which implements Info, Infoln -// and Infof. These methods will write to the Info log if called. -// Thus, one may write either -// if glog.V(2) { glog.Info("log this") } -// or -// glog.V(2).Info("log this") -// The second form is shorter but the first is cheaper if logging is off because it does -// not evaluate its arguments. -// -// Whether an individual call to V generates a log record depends on the setting of -// the -v and --vmodule flags; both are off by default. If the level in the call to -// V is at least the value of -v, or of -vmodule for the source file containing the -// call, the V call will log. -func V(level Level) Verbose { - // This function tries hard to be cheap unless there's work to do. - // The fast path is two atomic loads and compares. - - // Here is a cheap but safe test to see if V logging is enabled globally. - if logging.verbosity.get() >= level { - return Verbose(true) - } - - // It's off globally but it vmodule may still be set. - // Here is another cheap but safe test to see if vmodule is enabled. - if atomic.LoadInt32(&logging.filterLength) > 0 { - // Now we need a proper lock to use the logging structure. The pcs field - // is shared so we must lock before accessing it. This is fairly expensive, - // but if V logging is enabled we're slow anyway. - logging.mu.Lock() - defer logging.mu.Unlock() - if runtime.Callers(2, logging.pcs[:]) == 0 { - return Verbose(false) - } - v, ok := logging.vmap[logging.pcs[0]] - if !ok { - v = logging.setV(logging.pcs[0]) - } - return Verbose(v >= level) - } - return Verbose(false) -} - -// Info is equivalent to the global Info function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Info(args ...interface{}) { - if v { - logging.print(infoLog, args...) - } -} - -// Infoln is equivalent to the global Infoln function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Infoln(args ...interface{}) { - if v { - logging.println(infoLog, args...) - } -} - -// Infof is equivalent to the global Infof function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Infof(format string, args ...interface{}) { - if v { - logging.printf(infoLog, format, args...) - } -} - -// Info logs to the INFO log. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Info(args ...interface{}) { - logging.print(infoLog, args...) -} - -// InfoDepth acts as Info but uses depth to determine which call frame to log. -// InfoDepth(0, "msg") is the same as Info("msg"). -func InfoDepth(depth int, args ...interface{}) { - logging.printDepth(infoLog, depth, args...) -} - -// Infoln logs to the INFO log. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Infoln(args ...interface{}) { - logging.println(infoLog, args...) -} - -// Infof logs to the INFO log. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Infof(format string, args ...interface{}) { - logging.printf(infoLog, format, args...) -} - -// Warning logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Warning(args ...interface{}) { - logging.print(warningLog, args...) -} - -// WarningDepth acts as Warning but uses depth to determine which call frame to log. -// WarningDepth(0, "msg") is the same as Warning("msg"). -func WarningDepth(depth int, args ...interface{}) { - logging.printDepth(warningLog, depth, args...) -} - -// Warningln logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Warningln(args ...interface{}) { - logging.println(warningLog, args...) -} - -// Warningf logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Warningf(format string, args ...interface{}) { - logging.printf(warningLog, format, args...) -} - -// Error logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Error(args ...interface{}) { - logging.print(errorLog, args...) -} - -// ErrorDepth acts as Error but uses depth to determine which call frame to log. -// ErrorDepth(0, "msg") is the same as Error("msg"). -func ErrorDepth(depth int, args ...interface{}) { - logging.printDepth(errorLog, depth, args...) -} - -// Errorln logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Errorln(args ...interface{}) { - logging.println(errorLog, args...) -} - -// Errorf logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Errorf(format string, args ...interface{}) { - logging.printf(errorLog, format, args...) -} - -// Fatal logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Fatal(args ...interface{}) { - logging.print(fatalLog, args...) -} - -// FatalDepth acts as Fatal but uses depth to determine which call frame to log. -// FatalDepth(0, "msg") is the same as Fatal("msg"). -func FatalDepth(depth int, args ...interface{}) { - logging.printDepth(fatalLog, depth, args...) -} - -// Fatalln logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Fatalln(args ...interface{}) { - logging.println(fatalLog, args...) -} - -// Fatalf logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Fatalf(format string, args ...interface{}) { - logging.printf(fatalLog, format, args...) -} - -// fatalNoStacks is non-zero if we are to exit without dumping goroutine stacks. -// It allows Exit and relatives to use the Fatal logs. -var fatalNoStacks uint32 - -// Exit logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Exit(args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.print(fatalLog, args...) -} - -// ExitDepth acts as Exit but uses depth to determine which call frame to log. -// ExitDepth(0, "msg") is the same as Exit("msg"). -func ExitDepth(depth int, args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.printDepth(fatalLog, depth, args...) -} - -// Exitln logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -func Exitln(args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.println(fatalLog, args...) -} - -// Exitf logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Exitf(format string, args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.printf(fatalLog, format, args...) -} diff --git a/vendor/github.com/golang/glog/glog_file.go b/vendor/github.com/golang/glog/glog_file.go deleted file mode 100644 index 65075d2811..0000000000 --- a/vendor/github.com/golang/glog/glog_file.go +++ /dev/null @@ -1,124 +0,0 @@ -// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/ -// -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// File I/O for logs. - -package glog - -import ( - "errors" - "flag" - "fmt" - "os" - "os/user" - "path/filepath" - "strings" - "sync" - "time" -) - -// MaxSize is the maximum size of a log file in bytes. -var MaxSize uint64 = 1024 * 1024 * 1800 - -// logDirs lists the candidate directories for new log files. -var logDirs []string - -// If non-empty, overrides the choice of directory in which to write logs. -// See createLogDirs for the full list of possible destinations. -var logDir = flag.String("log_dir", "", "If non-empty, write log files in this directory") - -func createLogDirs() { - if *logDir != "" { - logDirs = append(logDirs, *logDir) - } - logDirs = append(logDirs, os.TempDir()) -} - -var ( - pid = os.Getpid() - program = filepath.Base(os.Args[0]) - host = "unknownhost" - userName = "unknownuser" -) - -func init() { - h, err := os.Hostname() - if err == nil { - host = shortHostname(h) - } - - current, err := user.Current() - if err == nil { - userName = current.Username - } - - // Sanitize userName since it may contain filepath separators on Windows. - userName = strings.Replace(userName, `\`, "_", -1) -} - -// shortHostname returns its argument, truncating at the first period. -// For instance, given "www.google.com" it returns "www". -func shortHostname(hostname string) string { - if i := strings.Index(hostname, "."); i >= 0 { - return hostname[:i] - } - return hostname -} - -// logName returns a new log file name containing tag, with start time t, and -// the name for the symlink for tag. -func logName(tag string, t time.Time) (name, link string) { - name = fmt.Sprintf("%s.%s.%s.log.%s.%04d%02d%02d-%02d%02d%02d.%d", - program, - host, - userName, - tag, - t.Year(), - t.Month(), - t.Day(), - t.Hour(), - t.Minute(), - t.Second(), - pid) - return name, program + "." + tag -} - -var onceLogDirs sync.Once - -// create creates a new log file and returns the file and its filename, which -// contains tag ("INFO", "FATAL", etc.) and t. If the file is created -// successfully, create also attempts to update the symlink for that tag, ignoring -// errors. -func create(tag string, t time.Time) (f *os.File, filename string, err error) { - onceLogDirs.Do(createLogDirs) - if len(logDirs) == 0 { - return nil, "", errors.New("log: no log dirs") - } - name, link := logName(tag, t) - var lastErr error - for _, dir := range logDirs { - fname := filepath.Join(dir, name) - f, err := os.Create(fname) - if err == nil { - symlink := filepath.Join(dir, link) - os.Remove(symlink) // ignore err - os.Symlink(name, symlink) // ignore err - return f, fname, nil - } - lastErr = err - } - return nil, "", fmt.Errorf("log: cannot create log: %v", lastErr) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 1b31418847..05bba3dd88 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -37,9 +37,6 @@ github.com/go-openapi/swag # github.com/gogo/protobuf v1.3.1 github.com/gogo/protobuf/proto github.com/gogo/protobuf/sortkeys -# github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b -## explicit -github.com/golang/glog # github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 github.com/golang/groupcache/lru # github.com/golang/protobuf v1.4.2