diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 7c1c34d7761..248c4a6eb5d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -102,6 +102,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Count HTTP 429 responses in the elasticsearch output {pull}8056[8056] - Report configured queue type. {pull}8091[8091] - Added the `add_process_metadata` processor to enrich events with process information. {pull}6789[6789] +- Report number of open file handles on Windows. {pull}8329[8329] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index b3e05fe08b9..dc829bd3a51 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -451,7 +451,7 @@ Apache License 2.0 -------------------------------------------------------------------- Dependency: github.com/elastic/go-sysinfo -Revision: b30cfd89b3c329a5b84f237573c8422730d2e170 +Revision: 7b021494a9562d0c3f0422d49b9980709c5650e9 License type (autodetected): Apache-2.0 ./vendor/github.com/elastic/go-sysinfo/LICENSE.txt: -------------------------------------------------------------------- @@ -486,7 +486,7 @@ Apache License 2.0 -------------------------------------------------------------------- Dependency: github.com/elastic/go-windows -Revision: a4ab469ab3d6f167cba8cb515c3273c6b56f9fc3 +Revision: bb1581babc04d5cb29a2bfa7a9ac6781c730c8dd License type (autodetected): Apache-2.0 ./vendor/github.com/elastic/go-windows/LICENSE.txt: -------------------------------------------------------------------- diff --git a/libbeat/cmd/instance/metrics.go b/libbeat/cmd/instance/metrics.go index d9de7eb0f03..35df8092d87 100644 --- a/libbeat/cmd/instance/metrics.go +++ b/libbeat/cmd/instance/metrics.go @@ -40,13 +40,8 @@ func init() { } func setupMetrics(name string) error { - monitoring.NewFunc(beatMetrics, "memstats", reportMemStats, monitoring.Report) - monitoring.NewFunc(beatMetrics, "cpu", reportBeatCPU, monitoring.Report) - monitoring.NewFunc(systemMetrics, "cpu", reportSystemCPUUsage, monitoring.Report) - setupPlatformSpecificMetrics() - beatProcessStats = &process.Stats{ Procs: []string{name}, EnvWhitelist: nil, @@ -54,19 +49,28 @@ func setupMetrics(name string) error { CacheCmdLine: true, IncludeTop: process.IncludeTopConfig{}, } + err := beatProcessStats.Init() + if err != nil { + return err + } + + monitoring.NewFunc(beatMetrics, "memstats", reportMemStats, monitoring.Report) + monitoring.NewFunc(beatMetrics, "cpu", reportBeatCPU, monitoring.Report) + + setupPlatformSpecificMetrics() - return err + return nil } func setupPlatformSpecificMetrics() { if runtime.GOOS != "windows" { monitoring.NewFunc(systemMetrics, "load", reportSystemLoadAverage, monitoring.Report) + } else { + setupWindowsHandlesMetrics() } - if runtime.GOOS == "linux" { - monitoring.NewFunc(beatMetrics, "fd", reportFDUsage, monitoring.Report) - } + setupLinuxBSDFDMetrics() } func reportMemStats(m monitoring.Mode, V monitoring.Visitor) { @@ -214,62 +218,6 @@ func getCPUUsage() (float64, *process.Ticks, error) { return totalCPUUsage, &p, nil } -func reportFDUsage(_ monitoring.Mode, V monitoring.Visitor) { - V.OnRegistryStart() - defer V.OnRegistryFinished() - - open, hardLimit, softLimit, err := getFDUsage() - if err != nil { - logp.Err("Error while retrieving FD information: %v", err) - return - } - - monitoring.ReportInt(V, "open", int64(open)) - monitoring.ReportNamespace(V, "limit", func() { - monitoring.ReportInt(V, "hard", int64(hardLimit)) - monitoring.ReportInt(V, "soft", int64(softLimit)) - }) -} - -func getFDUsage() (open, hardLimit, softLimit uint64, err error) { - state, err := getBeatProcessState() - if err != nil { - return 0, 0, 0, err - } - - iOpen, err := state.GetValue("fd.open") - if err != nil { - return 0, 0, 0, fmt.Errorf("error getting number of open FD: %v", err) - } - - open, ok := iOpen.(uint64) - if !ok { - return 0, 0, 0, fmt.Errorf("error converting value of open FDs to uint64: %v", iOpen) - } - - iHardLimit, err := state.GetValue("fd.limit.hard") - if err != nil { - return 0, 0, 0, fmt.Errorf("error getting FD hard limit: %v", err) - } - - hardLimit, ok = iHardLimit.(uint64) - if !ok { - return 0, 0, 0, fmt.Errorf("error converting values of FD hard limit: %v", iHardLimit) - } - - iSoftLimit, err := state.GetValue("fd.limit.soft") - if err != nil { - return 0, 0, 0, fmt.Errorf("error getting FD hard limit: %v", err) - } - - softLimit, ok = iSoftLimit.(uint64) - if !ok { - return 0, 0, 0, fmt.Errorf("error converting values of FD hard limit: %v", iSoftLimit) - } - - return open, hardLimit, softLimit, nil -} - func reportSystemLoadAverage(_ monitoring.Mode, V monitoring.Visitor) { V.OnRegistryStart() defer V.OnRegistryFinished() diff --git a/libbeat/cmd/instance/metrics_file_descriptors.go b/libbeat/cmd/instance/metrics_file_descriptors.go new file mode 100644 index 00000000000..d120efabcc7 --- /dev/null +++ b/libbeat/cmd/instance/metrics_file_descriptors.go @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux freebsd,cgo + +package instance + +import ( + "fmt" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/monitoring" +) + +func setupLinuxBSDFDMetrics() { + monitoring.NewFunc(beatMetrics, "handles", reportFDUsage, monitoring.Report) +} + +func reportFDUsage(_ monitoring.Mode, V monitoring.Visitor) { + V.OnRegistryStart() + defer V.OnRegistryFinished() + + open, hardLimit, softLimit, err := getFDUsage() + if err != nil { + logp.Err("Error while retrieving FD information: %v", err) + return + } + + monitoring.ReportInt(V, "open", int64(open)) + monitoring.ReportNamespace(V, "limit", func() { + monitoring.ReportInt(V, "hard", int64(hardLimit)) + monitoring.ReportInt(V, "soft", int64(softLimit)) + }) +} + +func getFDUsage() (open, hardLimit, softLimit uint64, err error) { + state, err := getBeatProcessState() + if err != nil { + return 0, 0, 0, err + } + + iOpen, err := state.GetValue("fd.open") + if err != nil { + return 0, 0, 0, fmt.Errorf("error getting number of open FD: %v", err) + } + + open, ok := iOpen.(uint64) + if !ok { + return 0, 0, 0, fmt.Errorf("error converting value of open FDs to uint64: %v", iOpen) + } + + iHardLimit, err := state.GetValue("fd.limit.hard") + if err != nil { + return 0, 0, 0, fmt.Errorf("error getting FD hard limit: %v", err) + } + + hardLimit, ok = iHardLimit.(uint64) + if !ok { + return 0, 0, 0, fmt.Errorf("error converting values of FD hard limit: %v", iHardLimit) + } + + iSoftLimit, err := state.GetValue("fd.limit.soft") + if err != nil { + return 0, 0, 0, fmt.Errorf("error getting FD hard limit: %v", err) + } + + softLimit, ok = iSoftLimit.(uint64) + if !ok { + return 0, 0, 0, fmt.Errorf("error converting values of FD hard limit: %v", iSoftLimit) + } + + return open, hardLimit, softLimit, nil +} diff --git a/libbeat/cmd/instance/metrics_file_descriptors_stub.go b/libbeat/cmd/instance/metrics_file_descriptors_stub.go new file mode 100644 index 00000000000..f6665ce2944 --- /dev/null +++ b/libbeat/cmd/instance/metrics_file_descriptors_stub.go @@ -0,0 +1,24 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !linux +// +build !freebsd !cgo + +package instance + +// FDUsage is only supported on Linux and FreeBSD. +func setupLinuxBSDFDMetrics() {} diff --git a/libbeat/cmd/instance/metrics_handles.go b/libbeat/cmd/instance/metrics_handles.go new file mode 100644 index 00000000000..6848a20f105 --- /dev/null +++ b/libbeat/cmd/instance/metrics_handles.go @@ -0,0 +1,67 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package instance + +import ( + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/monitoring" + sysinfo "github.com/elastic/go-sysinfo" + "github.com/elastic/go-sysinfo/types" +) + +const ( + fileHandlesNotReported = "Following metrics will not be reported: beat.handles.open" +) + +var ( + handleCounter types.OpenHandleCounter +) + +func setupWindowsHandlesMetrics() { + beatProcessSysInfo, err := sysinfo.Self() + if err != nil { + logp.Err("Error while getting own process info: %v", err) + logp.Err(fileHandlesNotReported) + return + } + + var ok bool + handleCounter, ok = beatProcessSysInfo.(types.OpenHandleCounter) + if !ok { + logp.Err("Process does not implement types.OpenHandleCounter: %v", beatProcessSysInfo) + logp.Err(fileHandlesNotReported) + return + } + + monitoring.NewFunc(beatMetrics, "handles", reportOpenHandles, monitoring.Report) +} + +func reportOpenHandles(_ monitoring.Mode, V monitoring.Visitor) { + V.OnRegistryStart() + defer V.OnRegistryFinished() + + n, err := handleCounter.OpenHandleCount() + if err != nil { + logp.Err("Error while retrieving the number of open file handles: %v", err) + return + } + + monitoring.ReportInt(V, "open", int64(n)) +} diff --git a/libbeat/cmd/instance/metrics_handles_stub.go b/libbeat/cmd/instance/metrics_handles_stub.go new file mode 100644 index 00000000000..677cc5aaf28 --- /dev/null +++ b/libbeat/cmd/instance/metrics_handles_stub.go @@ -0,0 +1,23 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !windows + +package instance + +// Counting number of open handles is only supported on Windows. +func setupWindowsHandlesMetrics() {} diff --git a/libbeat/monitoring/report/log/log.go b/libbeat/monitoring/report/log/log.go index 1254f4a16bc..ecd680bc627 100644 --- a/libbeat/monitoring/report/log/log.go +++ b/libbeat/monitoring/report/log/log.go @@ -52,9 +52,9 @@ var gauges = map[string]bool{ "beat.cpu.total.value": true, "beat.cpu.total.ticks": true, "beat.cpu.total.time": true, - "beat.fd.open": true, - "beat.fd.limit.hard": true, - "beat.fd.limit.soft": true, + "beat.handles.open": true, + "beat.handles.limit.hard": true, + "beat.handles.limit.soft": true, "system.load.1": true, "system.load.5": true, "system.load.15": true, diff --git a/vendor/github.com/elastic/go-sysinfo/providers/linux/process_linux.go b/vendor/github.com/elastic/go-sysinfo/providers/linux/process_linux.go index b096aa5c9d6..4c6c713b583 100644 --- a/vendor/github.com/elastic/go-sysinfo/providers/linux/process_linux.go +++ b/vendor/github.com/elastic/go-sysinfo/providers/linux/process_linux.go @@ -150,11 +150,13 @@ func (p *process) CPUTime() (types.CPUTimes, error) { }, nil } -func (p *process) FileDescriptors() ([]string, error) { +// OpenHandles returns the list of open file descriptors of the process. +func (p *process) OpenHandles() ([]string, error) { return p.Proc.FileDescriptorTargets() } -func (p *process) FileDescriptorCount() (int, error) { +// OpenHandles returns the number of open file descriptors of the process. +func (p *process) OpenHandleCount() (int, error) { return p.Proc.FileDescriptorsLen() } diff --git a/vendor/github.com/elastic/go-sysinfo/providers/windows/process_windows.go b/vendor/github.com/elastic/go-sysinfo/providers/windows/process_windows.go index fc115d11a91..7229da5a7c1 100644 --- a/vendor/github.com/elastic/go-sysinfo/providers/windows/process_windows.go +++ b/vendor/github.com/elastic/go-sysinfo/providers/windows/process_windows.go @@ -276,3 +276,15 @@ func (p *process) CPUTime() (types.CPUTimes, error) { System: windows.FiletimeToDuration(&kernelTime), }, nil } + +// OpenHandles returns the number of open handles of the process. +func (p *process) OpenHandleCount() (int, error) { + handle, err := p.open() + if err != nil { + return 0, err + } + defer syscall.CloseHandle(handle) + + count, err := windows.GetProcessHandleCount(handle) + return int(count), err +} diff --git a/vendor/github.com/elastic/go-sysinfo/types/process.go b/vendor/github.com/elastic/go-sysinfo/types/process.go index 27ca7296e1f..fce67badc3b 100644 --- a/vendor/github.com/elastic/go-sysinfo/types/process.go +++ b/vendor/github.com/elastic/go-sysinfo/types/process.go @@ -39,9 +39,14 @@ type Environment interface { Environment() (map[string]string, error) } -type FileDescriptor interface { - FileDescriptors() ([]string, error) - FileDescriptorCount() (int, error) +// OpenHandleEnumerator lists the open file handles. +type OpenHandleEnumerator interface { + OpenHandles() ([]string, error) +} + +// OpenHandleCount returns the number the open file handles. +type OpenHandleCounter interface { + OpenHandleCount() (int, error) } type CPUTimer interface { diff --git a/vendor/github.com/elastic/go-windows/CHANGELOG.md b/vendor/github.com/elastic/go-windows/CHANGELOG.md index 973f716379f..af006df463e 100644 --- a/vendor/github.com/elastic/go-windows/CHANGELOG.md +++ b/vendor/github.com/elastic/go-windows/CHANGELOG.md @@ -7,3 +7,4 @@ - GetProcessImageFileName - EnumProcesses [#6](https://github.com/elastic/go-windows/pull/6) +* Add GetProcessHandleCount to kernel32: [#7](https://github.com/elastic/go-windows/pull/7) diff --git a/vendor/github.com/elastic/go-windows/kernel32.go b/vendor/github.com/elastic/go-windows/kernel32.go index 35e1b3ecb2e..53d01db9625 100644 --- a/vendor/github.com/elastic/go-windows/kernel32.go +++ b/vendor/github.com/elastic/go-windows/kernel32.go @@ -34,6 +34,7 @@ import ( //sys _GetSystemTimes(idleTime *syscall.Filetime, kernelTime *syscall.Filetime, userTime *syscall.Filetime) (err error) = kernel32.GetSystemTimes //sys _GlobalMemoryStatusEx(buffer *MemoryStatusEx) (err error) = kernel32.GlobalMemoryStatusEx //sys _ReadProcessMemory(handle syscall.Handle, baseAddress uintptr, buffer uintptr, size uintptr, numRead *uintptr) (err error) = kernel32.ReadProcessMemory +//sys _GetProcessHandleCount(handle syscall.Handle, pdwHandleCount *uint32) (err error) = kernel32.GetProcessHandleCount var ( sizeofMemoryStatusEx = uint32(unsafe.Sizeof(MemoryStatusEx{})) @@ -236,3 +237,13 @@ func ReadProcessMemory(handle syscall.Handle, baseAddress uintptr, dest []byte) } return numRead, nil } + +// GetProcessHandleCount retrieves the number of open handles of a process. +// https://docs.microsoft.com/en-us/windows/desktop/api/processthreadsapi/nf-processthreadsapi-getprocesshandlecount +func GetProcessHandleCount(process syscall.Handle) (uint32, error) { + var count uint32 + if err := _GetProcessHandleCount(process, &count); err != nil { + return 0, errors.Wrap(err, "GetProcessHandleCount failed") + } + return count, nil +} diff --git a/vendor/github.com/elastic/go-windows/zsyscall_windows.go b/vendor/github.com/elastic/go-windows/zsyscall_windows.go index c861d3a3fd5..27e86ec4f72 100644 --- a/vendor/github.com/elastic/go-windows/zsyscall_windows.go +++ b/vendor/github.com/elastic/go-windows/zsyscall_windows.go @@ -45,6 +45,7 @@ var ( procGetSystemTimes = modkernel32.NewProc("GetSystemTimes") procGlobalMemoryStatusEx = modkernel32.NewProc("GlobalMemoryStatusEx") procReadProcessMemory = modkernel32.NewProc("ReadProcessMemory") + procGetProcessHandleCount = modkernel32.NewProc("GetProcessHandleCount") procGetFileVersionInfoW = modversion.NewProc("GetFileVersionInfoW") procGetFileVersionInfoSizeW = modversion.NewProc("GetFileVersionInfoSizeW") procVerQueryValueW = modversion.NewProc("VerQueryValueW") @@ -108,6 +109,18 @@ func _ReadProcessMemory(handle syscall.Handle, baseAddress uintptr, buffer uintp return } +func _GetProcessHandleCount(handle syscall.Handle, pdwHandleCount *uint32) (err error) { + r1, _, e1 := syscall.Syscall(procGetProcessHandleCount.Addr(), 2, uintptr(handle), uintptr(unsafe.Pointer(pdwHandleCount)), 0) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + func _GetFileVersionInfo(filename string, reserved uint32, dataLen uint32, data *byte) (success bool, err error) { var _p0 *uint16 _p0, err = syscall.UTF16PtrFromString(filename) diff --git a/vendor/vendor.json b/vendor/vendor.json index 1c31eb71def..2a25bd03fdd 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -542,44 +542,44 @@ { "checksumSHA1": "QhFIpuHPaV6hKejKcc2wm6y4MSQ=", "path": "github.com/elastic/go-sysinfo", - "revision": "b30cfd89b3c329a5b84f237573c8422730d2e170", - "revisionTime": "2018-08-06T09:31:13Z" + "revision": "7b021494a9562d0c3f0422d49b9980709c5650e9", + "revisionTime": "2018-09-11T17:37:16Z" }, { "checksumSHA1": "GiZCjX17K265TtamGZZw4R2Jwbk=", "path": "github.com/elastic/go-sysinfo/internal/registry", - "revision": "b30cfd89b3c329a5b84f237573c8422730d2e170", - "revisionTime": "2018-08-06T09:31:13Z" + "revision": "7b021494a9562d0c3f0422d49b9980709c5650e9", + "revisionTime": "2018-09-11T17:37:16Z" }, { "checksumSHA1": "432ecsMRmLpy5OvXMhQE/k9KWLQ=", "path": "github.com/elastic/go-sysinfo/providers/darwin", - "revision": "b30cfd89b3c329a5b84f237573c8422730d2e170", - "revisionTime": "2018-08-06T09:31:13Z" + "revision": "7b021494a9562d0c3f0422d49b9980709c5650e9", + "revisionTime": "2018-09-11T17:37:16Z" }, { - "checksumSHA1": "UnVdYPym09BEdsTqyu/kfg+zDwY=", + "checksumSHA1": "1eCL0MsvmiyjNvh0tcnnR4rmcWk=", "path": "github.com/elastic/go-sysinfo/providers/linux", - "revision": "b30cfd89b3c329a5b84f237573c8422730d2e170", - "revisionTime": "2018-08-06T09:31:13Z" + "revision": "7b021494a9562d0c3f0422d49b9980709c5650e9", + "revisionTime": "2018-09-11T17:37:16Z" }, { "checksumSHA1": "RWLvcP1w9ynKbuCqiW6prwd+EDU=", "path": "github.com/elastic/go-sysinfo/providers/shared", - "revision": "b30cfd89b3c329a5b84f237573c8422730d2e170", - "revisionTime": "2018-08-06T09:31:13Z" + "revision": "7b021494a9562d0c3f0422d49b9980709c5650e9", + "revisionTime": "2018-09-11T17:37:16Z" }, { - "checksumSHA1": "O6meYKnifW547nGtX7VqUWtFAOk=", + "checksumSHA1": "lWVD4w1xiAkFZgQPZAYz+fTZsrU=", "path": "github.com/elastic/go-sysinfo/providers/windows", - "revision": "b30cfd89b3c329a5b84f237573c8422730d2e170", - "revisionTime": "2018-08-06T09:31:13Z" + "revision": "7b021494a9562d0c3f0422d49b9980709c5650e9", + "revisionTime": "2018-09-11T17:37:16Z" }, { - "checksumSHA1": "ofmVTozc/VYPiwzxksVLQjJ/scI=", + "checksumSHA1": "tIqFxnZi9XvC70dMoZDSoUtEVQY=", "path": "github.com/elastic/go-sysinfo/types", - "revision": "b30cfd89b3c329a5b84f237573c8422730d2e170", - "revisionTime": "2018-08-06T09:31:13Z" + "revision": "7b021494a9562d0c3f0422d49b9980709c5650e9", + "revisionTime": "2018-09-11T17:37:16Z" }, { "checksumSHA1": "tNszmkpuJYZMX8l8rlnvBDtoc1M=", @@ -726,10 +726,10 @@ "versionExact": "v0.6.1" }, { - "checksumSHA1": "6QV5LopksTjBfMpZWKHhe2m0hAY=", + "checksumSHA1": "rnd3qf1FE22X3MxXWbetqq6EoBk=", "path": "github.com/elastic/go-windows", - "revision": "a4ab469ab3d6f167cba8cb515c3273c6b56f9fc3", - "revisionTime": "2018-08-13T15:18:21Z" + "revision": "bb1581babc04d5cb29a2bfa7a9ac6781c730c8dd", + "revisionTime": "2018-08-31T13:10:45Z" }, { "checksumSHA1": "RPOLNUpw00QUUaA/U4YbPVf6WlA=",