From 172b94fb2f8ecd80c25961116f660d2b7cc70c38 Mon Sep 17 00:00:00 2001 From: Balachandar Mani Date: Thu, 24 Aug 2023 14:17:13 -0700 Subject: [PATCH] transformer infra get traverse path and database get optimization (#98) --- translib/transformer/xlate.go | 72 +++++++++++++++++---------- translib/transformer/xlate_from_db.go | 22 ++++++-- 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/translib/transformer/xlate.go b/translib/transformer/xlate.go index ce684cf53a7b..e6aaf51a9e2e 100644 --- a/translib/transformer/xlate.go +++ b/translib/transformer/xlate.go @@ -85,7 +85,7 @@ func TraverseDb(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[str dataMap[i] = make(map[string]map[string]db.Value) } - err := traverseDbHelper(dbs, spec, &dataMap, parentKey, dbTblKeyGetCache) + err := traverseDbHelper(dbs, &spec, &dataMap, parentKey, dbTblKeyGetCache) if err != nil { xfmrLogDebug("Didn't get all data from traverseDbHelper") return err @@ -109,7 +109,7 @@ func TraverseDb(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[str return nil } -func traverseDbHelper(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[string]map[string]db.Value, parentKey *db.Key, dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool) error { +func traverseDbHelper(dbs [db.MaxDB]*db.DB, spec *KeySpec, result *map[db.DBNum]map[string]map[string]db.Value, parentKey *db.Key, dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool) error { var err error var dbOpts db.Options = getDBOptions(spec.DbNum) @@ -119,28 +119,19 @@ func traverseDbHelper(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]m // get an entry with a specific key if spec.Ts.Name != XFMR_NONE_STRING { // Do not traverse for NONE table data, err := dbs[spec.DbNum].GetEntry(&spec.Ts, spec.Key) - queriedDbInfo := make(map[string]map[string]bool) - queriedDbTblInfo := make(map[string]bool) - queriedDbTblInfo[strings.Join(spec.Key.Comp, separator)] = true - queriedDbInfo[spec.Ts.Name] = queriedDbTblInfo - if dbTblKeyGetCache == nil { - dbTblKeyGetCache = make(map[db.DBNum]map[string]map[string]bool) - } - dbTblKeyGetCache[spec.DbNum] = queriedDbInfo + dbKeyStr := strings.Join(spec.Key.Comp, separator) if err != nil { - log.Warningf("Didn't get data for tbl(%v), key(%v) in traverseDbHelper", spec.Ts.Name, spec.Key) + updateDbDataMapAndKeyCache(dbKeyStr, &data, spec, result, dbTblKeyGetCache, false) + if log.V(5) { + log.Warningf("Didn't get data for tbl(%v), key(%v) in traverseDbHelper", spec.Ts.Name, spec.Key) + } return err } - - if (*result)[spec.DbNum][spec.Ts.Name] == nil { - (*result)[spec.DbNum][spec.Ts.Name] = map[string]db.Value{strings.Join(spec.Key.Comp, separator): data} - } else { - (*result)[spec.DbNum][spec.Ts.Name][strings.Join(spec.Key.Comp, separator)] = data - } + updateDbDataMapAndKeyCache(dbKeyStr, &data, spec, result, dbTblKeyGetCache, true) } if len(spec.Child) > 0 { for _, ch := range spec.Child { - err = traverseDbHelper(dbs, ch, result, &spec.Key, dbTblKeyGetCache) + err = traverseDbHelper(dbs, &ch, result, &spec.Key, dbTblKeyGetCache) } } } else { @@ -152,28 +143,59 @@ func traverseDbHelper(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]m return err } xfmrLogDebug("keys for table %v in DB %v are %v", spec.Ts.Name, spec.DbNum, keys) - for i := range keys { - if parentKey != nil && !spec.IgnoreParentKey { + parentDbKeyStr := "" + if parentKey != nil && !spec.IgnoreParentKey { + parentDbKeyStr = strings.Join((*parentKey).Comp, separator) + } + for _, dbKey := range keys { + dbKeyStr := strings.Join(dbKey.Comp, separator) + if len(parentDbKeyStr) > 0 { // TODO - multi-depth with a custom delimiter - if !strings.Contains(strings.Join(keys[i].Comp, separator), strings.Join((*parentKey).Comp, separator)) { + if !strings.Contains(dbKeyStr, parentDbKeyStr) { continue } } - spec.Key = keys[i] - err = traverseDbHelper(dbs, spec, result, parentKey, dbTblKeyGetCache) + data, err := dbs[spec.DbNum].GetEntry(&spec.Ts, dbKey) if err != nil { - xfmrLogDebug("Traversal didn't fetch for : %v", err) + log.Warningf("Table.GetEntry returned error %v for tbl(%v), and the key %v in traverseDbHelper", err, spec.Ts.Name, dbKey) + updateDbDataMapAndKeyCache(dbKeyStr, &data, spec, result, dbTblKeyGetCache, false) + } else if data.IsPopulated() { + updateDbDataMapAndKeyCache(dbKeyStr, &data, spec, result, dbTblKeyGetCache, true) + } + if len(spec.Child) > 0 { + for _, ch := range spec.Child { + err = traverseDbHelper(dbs, &ch, result, &dbKey, dbTblKeyGetCache) + } } } } else if len(spec.Child) > 0 { for _, ch := range spec.Child { - err = traverseDbHelper(dbs, ch, result, &spec.Key, dbTblKeyGetCache) + err = traverseDbHelper(dbs, &ch, result, &spec.Key, dbTblKeyGetCache) } } } return err } +func updateDbDataMapAndKeyCache(dbKeyStr string, data *db.Value, spec *KeySpec, + result *map[db.DBNum]map[string]map[string]db.Value, dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool, readOk bool) { + if (*result)[spec.DbNum][spec.Ts.Name] == nil { + (*result)[spec.DbNum][spec.Ts.Name] = map[string]db.Value{dbKeyStr: *data} + } else { + (*result)[spec.DbNum][spec.Ts.Name][dbKeyStr] = *data + } + if dbTblKeyGetCache == nil { + dbTblKeyGetCache = make(map[db.DBNum]map[string]map[string]bool) + } + if dbTblKeyGetCache[spec.DbNum] == nil { + dbTblKeyGetCache[spec.DbNum] = make(map[string]map[string]bool) + } + if dbTblKeyGetCache[spec.DbNum][spec.Ts.Name] == nil { + dbTblKeyGetCache[spec.DbNum][spec.Ts.Name] = make(map[string]bool) + } + dbTblKeyGetCache[spec.DbNum][spec.Ts.Name][dbKeyStr] = readOk +} + func XlateUriToKeySpec(uri string, requestUri string, ygRoot *ygot.GoStruct, t *interface{}, txCache interface{}) (*[]KeySpec, error) { var err error diff --git a/translib/transformer/xlate_from_db.go b/translib/transformer/xlate_from_db.go index c9d05f39d699..18342fc6b6e2 100644 --- a/translib/transformer/xlate_from_db.go +++ b/translib/transformer/xlate_from_db.go @@ -303,11 +303,12 @@ func sonicDbToYangListFill(inParamsForGet xlateFromDbParams) []typeMapOfInterfac xpath := inParamsForGet.xpath dbTblData := (*dbDataMap)[dbIdx][table] - for keyStr := range dbTblData { - curMap := make(map[string]interface{}) + delKeyCnt := 0 + for keyStr, dbVal := range dbTblData { dbSpecData, ok := xDbSpecMap[table] if ok && dbSpecData.keyName == nil && xDbSpecMap[xpath].dbEntry != nil { yangKeys := yangKeyFromEntryGet(xDbSpecMap[xpath].dbEntry) + curMap := make(map[string]interface{}) sonicKeyDataAdd(dbIdx, yangKeys, table, xDbSpecMap[xpath].dbEntry.Name, keyStr, curMap) if len(curMap) > 0 { linParamsForGet := formXlateFromDbParams(inParamsForGet.dbs[dbIdx], inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, inParamsForGet.uri, inParamsForGet.requestUri, xpath, inParamsForGet.oper, table, keyStr, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) @@ -315,12 +316,23 @@ func sonicDbToYangListFill(inParamsForGet xlateFromDbParams) []typeMapOfInterfac curMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap inParamsForGet.dbDataMap = dbDataMap + if len(curMap) > 0 { + mapSlice = append(mapSlice, curMap) + } + delKeyCnt++ + dbTblData[keyStr] = db.Value{} + delete(inParamsForGet.dbTblKeyGetCache[dbIdx][table], keyStr) + } else if len(dbVal.Field) == 0 { + delKeyCnt++ } } - if len(curMap) > 0 { - mapSlice = append(mapSlice, curMap) - } } + + if len(dbTblData) == delKeyCnt { + delete((*dbDataMap)[dbIdx], table) + delete(inParamsForGet.dbTblKeyGetCache[dbIdx], table) + } + return mapSlice }