diff --git a/bill-of-materials.json b/bill-of-materials.json index 6863e0a695ba..3b7c4743cac8 100644 --- a/bill-of-materials.json +++ b/bill-of-materials.json @@ -8,6 +8,15 @@ } ] }, + { + "project": "github.com/ahrtr/gocontainer", + "licenses": [ + { + "type": "MIT License", + "confidence": 1 + } + ] + }, { "project": "github.com/anishathalye/porcupine", "licenses": [ diff --git a/go.mod b/go.mod index c2c1a0447937..7a20b92f1b91 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ replace ( ) require ( + github.com/ahrtr/gocontainer v0.3.0 github.com/bgentry/speakeasy v0.1.0 github.com/cheggaaa/pb/v3 v3.1.0 github.com/coreos/go-semver v0.3.0 diff --git a/go.sum b/go.sum index 77697d048d9c..6f54740e63fc 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/ahrtr/gocontainer v0.3.0 h1:/4wM0VhaLEYZMoF6WT8ZHUmf2n9BVpCD3uMaKrA0iHY= +github.com/ahrtr/gocontainer v0.3.0/go.mod h1:cQoR5/JTMoDNEkk5vGaohPZ+nnTQVB2nk2Y012WJWsM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= diff --git a/tools/rw-benchmark/linkedmap.go b/tools/rw-benchmark/linkedmap.go deleted file mode 100644 index 83e1d828fbdb..000000000000 --- a/tools/rw-benchmark/linkedmap.go +++ /dev/null @@ -1,310 +0,0 @@ -// Copyright 2022 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package linkedmap implements a linked hashmap, based on a map and a doubly linked list. The iteration ordering is normally -// the order in which keys were inserted into the map, or the order in which the keys were accessed if the accessOrder flag is set. -// If a linkedMap is configured as access-order, then the first element in the list is the eldest element, which means it's the least recently inserted -// or accessed element; while the last element is the newest element, which means it's the most recently inserted or accessed element. -// -// To iterate over an linkedMap (where lm is an instance of linkedmap.Interface): -// -// it, hasNext := lm.Iterator() -// var k, v interface{} -// for hasNext { -// k, v, hasNext = it() -// // do something with k & v -// } -// -// To iterate over an linkedMap in reverse order (where lm is an instance of linkedmap.Interface): -// -// it, hasPrev := lm.ReverseIterator() -// var k, v interface{} -// for hasPrev { -// k, v, hasPrev = it() -// // do something with k & v -// } -// -// The implementation of LinkedMap is coming from https://github.com/ahrtr/gocontainer/tree/master/map/linkedmap, -// with some minor changes. -package main - -// LinkedMap is a type of linked map, and linkedMap implements this interface. -type LinkedMap interface { - // Size returns the number of elements in the collection. - Size() int - // IsEmpty returns true if this container contains no elements. - IsEmpty() bool - // Clear removes all the elements from this container. - Clear() - - // Put associates the specified value with the specified key in this map. If the map previously contained a mapping for the key, - // the old value is replaced by the specified value. - // It returns the previous value associated with the specified key, or nil if there was no mapping for the key. - // A nil return can also indicate that the map previously associated nil with the specified key. - Put(k, v interface{}) interface{} - - // WithAccessOrder configures the iteration ordering for this linked map, - // true for access-order, and false for insertion-order. - WithAccessOrder(accessOrder bool) LinkedMap - - // Get returns the value to which the specified key is mapped, or nil if this map contains no mapping for the key. - Get(k interface{}) interface{} - // GetOrDefault returns the value to which the specified key is mapped, or the defaultValue if this map contains no mapping for the key. - GetOrDefault(k, defaultValue interface{}) interface{} - - // ContainsKey returns true if this map contains a mapping for the specified key. - ContainsKey(k interface{}) bool - // ContainsValue returns true if this map maps one or more keys to the specified value. - ContainsValue(v interface{}) bool - - // Remove removes the mapping for a key from this map if it is present. - // It returns the value to which this map previously associated the key, and true, - // or nil and false if the map contained no mapping for the key. - Remove(k interface{}) (interface{}, bool) - // RemoveFirstElement removes the first element from this map, which is the head of the list. - // It returns the (key, value, true) if the map isn't empty, or (nil, nil, false) if the map is empty. - RemoveFirstElement() (interface{}, interface{}, bool) - // RemoveLastElement removes the last element from this map, which is the tail of the list. - // It returns the (key, value, true) if the map isn't empty, or (nil, nil, false) if the map is empty. - RemoveLastElement() (interface{}, interface{}, bool) - - // Iterator returns an iterator over the elements in this map in proper sequence. - Iterator() (func() (interface{}, interface{}, bool), bool) - // ReverseIterator returns an iterator over the elements in this map in reverse sequence as Iterator. - ReverseIterator() (func() (interface{}, interface{}, bool), bool) -} - -type element struct { - key interface{} - value interface{} - prev *element - next *element -} - -// linkedMap implements the Interface. -type linkedMap struct { - data map[interface{}]*element - accessOrder bool - head *element - tail *element - length int -} - -// NewLinkedMap creates a linkedMap. -func NewLinkedMap() LinkedMap { - return &linkedMap{ - data: map[interface{}]*element{}, - accessOrder: false, - head: nil, - tail: nil, - length: 0, - } -} - -func (lm *linkedMap) WithAccessOrder(accessOrder bool) LinkedMap { - lm.accessOrder = accessOrder - return lm -} - -func (lm *linkedMap) Size() int { - return lm.length -} - -func (lm *linkedMap) IsEmpty() bool { - return lm.Size() == 0 -} - -func (lm *linkedMap) Put(k, v interface{}) interface{} { - var retVal interface{} - if oldElement, ok := lm.data[k]; ok { - - retVal = oldElement.value - oldElement.value = v - // move the element to the end of the list - if lm.accessOrder { - lm.unlink(oldElement) - lm.linkLast(oldElement) - } - } else { - e := &element{ - key: k, - value: v, - } - lm.data[k] = e - lm.linkLast(e) - } - - return retVal -} - -func (lm *linkedMap) Get(k interface{}) interface{} { - if oldElement, ok := lm.data[k]; ok { - // move the element to the end of the list - if lm.accessOrder { - lm.unlink(oldElement) - lm.linkLast(oldElement) - } - return oldElement.value - } - - return nil -} - -func (lm *linkedMap) GetOrDefault(k, defaultValue interface{}) interface{} { - if oldElement, ok := lm.data[k]; ok { - // move the element to the end of the list - if lm.accessOrder { - lm.unlink(oldElement) - lm.linkLast(oldElement) - } - return oldElement.value - } - - return defaultValue -} - -func (lm *linkedMap) ContainsKey(k interface{}) bool { - _, ok := lm.data[k] - return ok -} - -func (lm *linkedMap) ContainsValue(v interface{}) bool { - e := lm.head - for e != nil { - if e.value == v { - return true - } - e = e.next - } - - return false -} - -func (lm *linkedMap) Remove(k interface{}) (interface{}, bool) { - if oldElement, ok := lm.data[k]; ok { - retVal := oldElement.value - delete(lm.data, k) - lm.unlink(oldElement) - oldElement.key, oldElement.value = nil, nil - return retVal, true - } - - return nil, false -} - -func (lm *linkedMap) RemoveFirstElement() (interface{}, interface{}, bool) { - if lm.head != nil { - e := lm.head - k, v := e.key, e.value - - lm.unlink(e) - e.key, e.value = nil, nil - - return k, v, true - } - - return nil, nil, false -} - -func (lm *linkedMap) RemoveLastElement() (interface{}, interface{}, bool) { - if lm.tail != nil { - e := lm.tail - k, v := e.key, e.value - - lm.unlink(e) - e.key, e.value = nil, nil - - return k, v, true - } - - return nil, nil, false -} - -func (lm *linkedMap) Clear() { - lm.data = map[interface{}]*element{} - - for e := lm.head; e != nil; { - next := e.next - e.prev, e.next, e.key, e.value = nil, nil, nil, nil - e = next - } - - lm.head, lm.tail, lm.length = nil, nil, 0 -} - -func (lm *linkedMap) Iterator() (func() (interface{}, interface{}, bool), bool) { - e := lm.head - - return func() (interface{}, interface{}, bool) { - var k, v interface{} - if e != nil { - k = e.key - v = e.value - e = e.next - } else { - k, v = nil, nil - } - return k, v, e != nil - }, e != nil -} - -func (lm *linkedMap) ReverseIterator() (func() (interface{}, interface{}, bool), bool) { - e := lm.tail - - return func() (interface{}, interface{}, bool) { - var k, v interface{} - if e != nil { - k = e.key - v = e.value - e = e.prev - } else { - k, v = nil, nil - } - return k, v, e != nil - }, e != nil -} - -// linkLast links val as last element. -func (lm *linkedMap) linkLast(e *element) { - e.prev, e.next = lm.tail, nil - - if nil == lm.tail { - lm.head, lm.tail = e, e - } else { - lm.tail.next = e - lm.tail = e - } - - lm.length++ -} - -// unlink removes the specified element e in this list. -func (lm *linkedMap) unlink(e *element) { - ePrev, eNext := e.prev, e.next - e.prev, e.next = nil, nil - - if nil == ePrev { - lm.head = eNext - } else { - ePrev.next = eNext - } - - if nil == eNext { - lm.tail = ePrev - } else { - eNext.prev = ePrev - } - - lm.length-- -} diff --git a/tools/rw-benchmark/linkedmap_test.go b/tools/rw-benchmark/linkedmap_test.go deleted file mode 100644 index ae8c4271fa2f..000000000000 --- a/tools/rw-benchmark/linkedmap_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2022 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "testing" -) - -func TestLinkedMapSize(t *testing.T) { - lm := NewLinkedMap() - lm.Put(24, "benjamin") - lm.Put(43, "alice") - lm.Put(18, "john") - - if lm.Size() != 3 { - t.Errorf("The length isn't expected, expect: 3, actual: %d\n", lm.Size()) - } - - lm.Remove(43) - if lm.Size() != 2 { - t.Errorf("The length isn't expected, expect: 2, actual: %d\n", lm.Size()) - } - - lm.Clear() - if lm.Size() != 0 { - t.Errorf("The length isn't expected, expect: 0, actual: %d\n", lm.Size()) - } - if !lm.IsEmpty() { - t.Error("The container should be empty") - } -} - -func TestLinkedMapValue(t *testing.T) { - lm := NewLinkedMap() - keys := []int{24, 43, 18, 23, 35} - values := []string{"benjamin", "alice", "john", "tom", "bill"} - for i := 0; i < len(keys); i++ { - lm.Put(keys[i], values[i]) - } - - // test ContainsKey & ContainsValue - for _, k := range keys { - if !lm.ContainsKey(k) { - t.Errorf("The linkedMap should contain key: %d\n", k) - } - } - - for _, v := range values { - if !lm.ContainsValue(v) { - t.Errorf("The linkedMap should contain value: %s\n", v) - } - } - - // test Get & GetOrDefault - for i, k := range keys { - v := lm.Get(k) - if v != values[i] { - t.Errorf("The value associated with key: %v isn't expected, expect: %v, actual %v\n", k, values[i], v) - } - } - - v := lm.GetOrDefault(50, "defaultName") - if v != "defaultName" { - t.Errorf("The returned value should be the default value, but actual: %v\n", v) - } - - // test Remove, RemoveFirstElement and RemoveLastElement - v, ok := lm.Remove(43) - if v != "alice" || !ok { - t.Errorf("Failed to remove element with key 43, returned value: %v, success: %t\n", v, ok) - } - - k, v, ok := lm.RemoveFirstElement() - if k != 24 || v != "benjamin" || !ok { - t.Errorf("Failed to remove the first element, key: %v, value: %v, success: %t\n", k, v, ok) - } - - k, v, ok = lm.RemoveLastElement() - if k != 35 || v != "bill" || !ok { - t.Errorf("Failed to remove the last element, key: %v, value: %v, success: %t\n", k, v, ok) - } - - if lm.Size() != 2 { - t.Errorf("The length isn't expected, expect: 2, actual: %d\n", lm.Size()) - } -} - -func TestLinkedMapIterate(t *testing.T) { - lm := NewLinkedMap() - keys := []interface{}{24, 43, 18, 23, 35} - values := []interface{}{"benjamin", "alice", "john", "tom", "bill"} - for i := 0; i < len(keys); i++ { - lm.Put(keys[i], values[i]) - } - - checkIterateResult(t, lm, keys, values) - checkReverseIterateResult(t, lm, []interface{}{35, 23, 18, 43, 24}, []interface{}{"bill", "tom", "john", "alice", "benjamin"}) -} - -func TestLinkedMapAccessOrder(t *testing.T) { - lm := NewLinkedMap().WithAccessOrder(true) - keys := []int{24, 43, 18, 23, 35} - values := []string{"benjamin", "alice", "john", "tom", "bill"} - for i := 0; i < len(keys); i++ { - lm.Put(keys[i], values[i]) - } - - lm.Get(23) - lm.Get(24) - lm.Get(18) - lm.Get(35) - lm.Get(43) - - checkIterateResult(t, lm, []interface{}{23, 24, 18, 35, 43}, []interface{}{"tom", "benjamin", "john", "bill", "alice"}) - checkReverseIterateResult(t, lm, []interface{}{43, 35, 18, 24, 23}, []interface{}{"alice", "bill", "john", "benjamin", "tom"}) -} - -func checkIterateResult(t *testing.T, lm LinkedMap, expectedKey, expectedValue []interface{}) { - it, hasNext := lm.Iterator() - var k, v interface{} - - for i := 0; i < len(expectedKey); i++ { - if !hasNext { - t.Error("Unexpectedly reaching the end") - break - } - - k, v, hasNext = it() - if k != expectedKey[i] || v != expectedValue[i] { - t.Errorf("Unexpected key or value, Iterate: %d, expect: (%v, %v), actual: (%v, %v)\n", i, expectedKey[i], expectedValue[i], k, v) - } - } - - if hasNext { - t.Error("The iterator should have already reached the end") - } -} - -func checkReverseIterateResult(t *testing.T, lm LinkedMap, expectedKey, expectedValue []interface{}) { - it, hasPrev := lm.ReverseIterator() - var k, v interface{} - - for i := 0; i < len(expectedKey); i++ { - if !hasPrev { - t.Error("Unexpectedly reaching the end") - break - } - - k, v, hasPrev = it() - if k != expectedKey[i] || v != expectedValue[i] { - t.Errorf("Unexpected key or value, Iterate: %d, expect: (%v, %v), actual: (%v, %v)\n", i, expectedKey[i], expectedValue[i], k, v) - } - } - - if hasPrev { - t.Error("The iterator should have already reached the end") - } -} diff --git a/tools/rw-benchmark/plot_data.go b/tools/rw-benchmark/plot_data.go index 563e1f90751c..a1b29acab37e 100644 --- a/tools/rw-benchmark/plot_data.go +++ b/tools/rw-benchmark/plot_data.go @@ -24,6 +24,7 @@ import ( "strconv" "strings" + "github.com/ahrtr/gocontainer/map/linkedmap" "github.com/go-echarts/go-echarts/v2/charts" "github.com/go-echarts/go-echarts/v2/components" "github.com/go-echarts/go-echarts/v2/opts" @@ -98,7 +99,7 @@ func parseParams(args ...string) (legends []string, csvFiles []string, outFile s // // In the value(array) of the 3rd map, the first value is the // read-qps, the second value is the write-qps. -func loadCSV(filename string) (LinkedMap, error) { +func loadCSV(filename string) (linkedmap.Interface, error) { // Open the CSV file f, err := os.Open(filename) if err != nil { @@ -113,7 +114,7 @@ func loadCSV(filename string) (LinkedMap, error) { return nil, fmt.Errorf("failed to read csv file %q, error: %w", filename, err) } - lmRatio := NewLinkedMap() + lmRatio := linkedmap.New() // Parse the data for i, rec := range records { // When `REPEAT_COUNT` is 1, then there are 6 fields in each record. @@ -168,24 +169,24 @@ func loadCSV(filename string) (LinkedMap, error) { // Save the data into LinkedMap. // The first level map: lmRatio var ( - lmValueSize LinkedMap - lmConn LinkedMap + lmValueSize linkedmap.Interface + lmConn linkedmap.Interface ) lm := lmRatio.Get(ratio) if lm == nil { - lmValueSize = NewLinkedMap() + lmValueSize = linkedmap.New() lmRatio.Put(ratio, lmValueSize) } else { - lmValueSize = lm.(LinkedMap) + lmValueSize = lm.(linkedmap.Interface) } // The second level map: lmValueSize lm = lmValueSize.Get(valSize) if lm == nil { - lmConn = NewLinkedMap() + lmConn = linkedmap.New() lmValueSize.Put(valSize, lmConn) } else { - lmConn = lm.(LinkedMap) + lmConn = lm.(linkedmap.Interface) } // The third level map: lmConns @@ -195,8 +196,8 @@ func loadCSV(filename string) (LinkedMap, error) { return lmRatio, nil } -func loadData(files ...string) ([]LinkedMap, error) { - var dataMaps []LinkedMap +func loadData(files ...string) ([]linkedmap.Interface, error) { + var dataMaps []linkedmap.Interface for _, f := range files { lm, err := loadCSV(f) if err != nil { @@ -210,7 +211,7 @@ func loadData(files ...string) ([]LinkedMap, error) { // convertBenchmarkData converts the benchmark data to format // which is suitable for the line chart. -func convertBenchmarkData(lmConn LinkedMap) ([]uint64, []float64, []float64) { +func convertBenchmarkData(lmConn linkedmap.Interface) ([]uint64, []float64, []float64) { var ( conns []uint64 rQPS []float64 @@ -321,7 +322,7 @@ func renderChart(page *components.Page, ratio float64, valueSize uint64, legends // // In the value(array) of the 3rd map, the first value is the // read-qps, the second value is the write-qps. -func renderPage(legends []string, dataMap []LinkedMap, outFile string) error { +func renderPage(legends []string, dataMap []linkedmap.Interface, outFile string) error { page := components.NewPage() it1, hasNext1 := dataMap[0].Iterator() @@ -331,7 +332,7 @@ func renderPage(legends []string, dataMap []LinkedMap, outFile string) error { k1, v1, hasNext1 = it1() ratio := k1.(float64) - lmValueSize := v1.(LinkedMap) + lmValueSize := v1.(linkedmap.Interface) // Loop the second level map (lmValueSize) it2, hasNext2 := lmValueSize.Iterator() @@ -339,7 +340,7 @@ func renderPage(legends []string, dataMap []LinkedMap, outFile string) error { for hasNext2 { k2, v2, hasNext2 = it2() valueSize := k2.(uint64) - lmConn := v2.(LinkedMap) + lmConn := v2.(linkedmap.Interface) var ( conns []uint64 @@ -354,10 +355,10 @@ func renderPage(legends []string, dataMap []LinkedMap, outFile string) error { // Convert the related benchmark data in the second CSV file if present. if len(dataMap) > 1 { if lm1 := dataMap[1].Get(ratio); lm1 != nil { - lmValueSize2 := lm1.(LinkedMap) + lmValueSize2 := lm1.(linkedmap.Interface) if lm2 := lmValueSize2.Get(valueSize); lm2 != nil { - lmConn2 := lm2.(LinkedMap) + lmConn2 := lm2.(linkedmap.Interface) conn2, rQPS2, wQPS2 := convertBenchmarkData(lmConn2) if reflect.DeepEqual(conns, conn2) { rQPSs = append(rQPSs, rQPS2)