Skip to content

Commit

Permalink
Transformer Infra memory optimization and enhancements (sonic-net#80)
Browse files Browse the repository at this point in the history 
* Transformer infra initialization, application(common_app) enhancements for following:
*Transformer infra memory and startup time optimizations
*Support for gNMI style PUT/REPLACE on leaf-list.
*Sort DB keys for tables having mutli-element key with optional elements(sonic-yang container representing the table has multiple lists) before DB operation.
*Modifications to improve debuggability/logging.

* Transformer infra SET(CREATE/REPLACE/UPDATE) and GET code flow modifications to incorporate following:
*Support gNMI style PUT/REPLACE on leaf-list in SET flow
*Conditionally invoke subtree only once to improve SET request performance
*Support for key-transformer on sonic-yang GET request.
*Improve debuggability/logging.

* Transformer infra utility enhacements for CRUD and SET
  *Add utility functions in transformer infra for CRUD AND GET cases to retrieve yang node info
  *Key-transformer support on Sonic-Yang list for GET flow
  *Call back error message packaging
  *Changes to Improve debuggability and logging

* Transformer infra enhacements for DELETE flow
  *Json based table sorting for non CVL table ordering
  *Changes to Improve debuggability and logging.

* REPLACE case bug-fix to use copy of original instance fields in common_app to handle leaf-list target PUT/REPLACE to swap contents of leaf-list.

* Add copyright info

* Use the new ocbinds.GetSchema() API from master

* Addressed review comments

---------

Co-authored-by: ranjinidn <[email protected]>
Co-authored-by: ranjinidn <[email protected]>
Co-authored-by: ranjinidn <[email protected]>
  • Loading branch information
@amrutasali @ranjinidn
4 people authored May 24, 2023
1 parent df76c05 commit 1340e29
Show file tree
Hide file tree
Showing 17 changed files with 4,255 additions and 3,314 deletions.
397 changes: 222 additions & 175 deletions translib/common_app.go
View file

Large diffs are not rendered by default.

25 changes: 25 additions & 0 deletions translib/db/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,10 @@ func (d DB) String() string {
d.client, d.Opts, d.txState, d.txCmds)
}

func (dbNo DBNum) Name() string {
return (getDBInstName(dbNo))
}

func getDBInstName (dbNo DBNum) string {
switch dbNo {
case ApplDB:
Expand All @@ -313,6 +317,27 @@ func getDBInstName (dbNo DBNum) string {
return ""
}

func GetdbNameToIndex(dbName string) DBNum {
dbIndex := ConfigDB
switch dbName {
case "APPL_DB":
dbIndex = ApplDB
case "ASIC_DB":
dbIndex = AsicDB
case "COUNTERS_DB":
dbIndex = CountersDB
case "CONFIG_DB":
dbIndex = ConfigDB
case "FLEX_COUNTER_DB":
dbIndex = FlexCounterDB
case "STATE_DB":
dbIndex = StateDB
case "ERROR_DB":
dbIndex = ErrorDB
}
return dbIndex
}

// NewDB is the factory method to create new DB's.
func NewDB(opt Options) (*DB, error) {

Expand Down
2 changes: 1 addition & 1 deletion translib/transformer/subscribe_req_xlate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1951,7 +1951,7 @@ func (keyRslvr *DbYangKeyResolver) handleValueXfmr(xfmrName string, operation Op
if log.V(dbLgLvl) {
log.Info(keyRslvr.reqLogId, "resolveDbKey: keyLeafRefNode xfmrValue; ", xfmrName)
}
inParams := formXfmrDbInputRequest(int(operation), keyRslvr.dbIdx, keyRslvr.tableName, strings.Join(keyRslvr.key.Comp, keyRslvr.delim), keyName, keyVal)
inParams := formXfmrDbInputRequest(operation, keyRslvr.dbIdx, keyRslvr.tableName, strings.Join(keyRslvr.key.Comp, keyRslvr.delim), keyName, keyVal)
keyLeafVal, err = valueXfmrHandler(inParams, xfmrName)
if err != nil {
return
Expand Down
196 changes: 107 additions & 89 deletions translib/transformer/transformer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,74 +19,71 @@
package transformer

import (
"fmt"
"github.com/openconfig/goyang/pkg/yang"
"bufio"
"io/ioutil"
"os"
"path/filepath"
"runtime/debug"
"strings"
"bufio"
"path/filepath"
"io/ioutil"

log "github.com/golang/glog"

"github.com/Azure/sonic-mgmt-common/translib/ocbinds"
"github.com/openconfig/goyang/pkg/yang"
)

var YangPath = "/usr/models/yang/" // OpenConfig-*.yang and sonic yang models path
var ModelsListFile = "models_list"
var ModelsListFile = "models_list"
var TblInfoJsonFile = "sonic_table_info.json"

func reportIfError(errs []error) {
if len(errs) > 0 {
for _, err := range errs {
fmt.Fprintln(os.Stderr, err)
func getOcModelsList() []string {
var fileList []string
file, err := os.Open(YangPath + ModelsListFile)
if err != nil {
return fileList
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
fileEntry := scanner.Text()
if !strings.HasPrefix(fileEntry, "#") {
_, err := os.Stat(YangPath + fileEntry)
if err != nil {
continue
}
fileList = append(fileList, fileEntry)
}
}
return fileList
}

func getOcModelsList () ([]string) {
var fileList []string
file, err := os.Open(YangPath + ModelsListFile)
if err != nil {
return fileList
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
fileEntry := scanner.Text()
if !strings.HasPrefix(fileEntry, "#") {
_, err := os.Stat(YangPath + fileEntry)
if err != nil {
continue
}
fileList = append(fileList, fileEntry)
}
}
return fileList
}
func getDefaultModelsList() []string {
var files []string
fileInfo, err := ioutil.ReadDir(YangPath)
if err != nil {
return files
}

func getDefaultModelsList () ([]string) {
var files []string
fileInfo, err := ioutil.ReadDir(YangPath)
if err != nil {
return files
}

for _, file := range fileInfo {
if strings.HasPrefix(file.Name(), "sonic-") && !strings.HasSuffix(file.Name(), "-dev.yang") && filepath.Ext(file.Name()) == ".yang" {
files = append(files, file.Name())
}
}
return files
for _, file := range fileInfo {
if strings.HasPrefix(file.Name(), "sonic-") && !strings.HasSuffix(file.Name(), "-dev.yang") && filepath.Ext(file.Name()) == ".yang" {
files = append(files, file.Name())
}
}
return files
}

func init() {
initYangModelsPath()
initRegex()
ocList := getOcModelsList()
ocList := getOcModelsList()
yangFiles := getDefaultModelsList()
yangFiles = append(yangFiles, ocList...)
fmt.Println("Yang model List:", yangFiles)
yangFiles = append(yangFiles, ocList...)
xfmrLogInfo("Yang model List: %v", yangFiles)
err := loadYangModules(yangFiles...)
if err != nil {
fmt.Fprintln(os.Stderr, err)
}
if err != nil {
log.Error(err)
}
debug.FreeOSMemory()
}

func initYangModelsPath() {
Expand All @@ -97,100 +94,121 @@ func initYangModelsPath() {
YangPath = path
}

fmt.Println("Yang modles path:", YangPath)
xfmrLogDebug("Yang modles path: %v", YangPath)
}

func loadYangModules(files ...string) error {
type ModProfile map[string]interface{}

func loadYangModules(files ...string) error {
var err error

paths := []string{YangPath}

for _, path := range paths {
expanded, err := yang.PathsWithModules(path)
if err != nil {
fmt.Fprintln(os.Stderr, err)
xfmrLogInfo("Couldn't load yang module %v due to %v ", path, err)
continue
}
yang.AddPath(expanded...)
}

ms := yang.NewModules()
ygSchema, err := ocbinds.Schema()
if err != nil {
return err
}

for _, name := range files {
if err := ms.Read(name); err != nil {
fmt.Fprintln(os.Stderr, err)
continue
modProfiles := make(map[string]ModProfile)
rootSchema := ygSchema.RootSchema()
for _, schema := range rootSchema.Dir {
if _, found := schema.Annotation["modulename"]; found {
m := schema.Annotation["modulename"].(string)
for _, fn := range files {
f := strings.Split(fn, ".yang")
if m == f[0] {
modProfiles[m] = schema.Annotation
break
}
}
}
}

// Process the Yang files
reportIfError(ms.Process())

// Keep track of the top level modules we read in.
// Those are the only modules we want to print below.
mods := map[string]*yang.Module{}
var names []string

for _, m := range ms.Modules {
if mods[m.Name] == nil {
mods[m.Name] = m
names = append(names, m.Name)
annotMs := yang.NewModules()
for _, name := range files {
if strings.Contains(name, "-annot") {
if err := annotMs.Read(name); err != nil {
xfmrLogInfo("Couldn't read yang annotation %v due to %v", name, err)
continue
}
}
}

sonic_entries := make([]*yang.Entry, 0)
oc_entries := make(map[string]*yang.Entry)
oc_annot_entries := make([]*yang.Entry, 0)
oc_annot_entries := make([]*yang.Entry, 0)
sonic_annot_entries := make([]*yang.Entry, 0)

for _, n := range names {
if strings.Contains(n, "annot") && strings.Contains(n, "sonic") {
sonic_annot_entries = append(sonic_annot_entries, yang.ToEntry(mods[n]))
} else if strings.Contains(n, "annot") {
yangMdlNmDt := strings.Split(n, "-annot")
for _, m := range annotMs.Modules {
if strings.Contains(m.Name, "sonic") {
sonic_annot_entries = append(sonic_annot_entries, yang.ToEntry(m))
} else {
yangMdlNmDt := strings.Split(m.Name, "-annot")
if len(yangMdlNmDt) > 0 {
addMdlCpbltEntry(yangMdlNmDt[0])
}
oc_annot_entries = append(oc_annot_entries, yang.ToEntry(mods[n]))
} else if strings.Contains(n, "sonic") {
sonic_entries = append(sonic_entries, yang.ToEntry(mods[n]))
} else if oc_entries[n] == nil {
oc_entries[n] = yang.ToEntry(mods[n])
oc_annot_entries = append(oc_annot_entries, yang.ToEntry(m))
}
}

sonic_entries := make([]*yang.Entry, 0)
oc_entries := make(map[string]*yang.Entry)

// Iterate over SchemaTree
for k, v := range ygSchema.SchemaTree["Device"].Dir {
mod := strings.Split(v.Annotation["schemapath"].(string), "/")
if _, found := modProfiles[mod[1]]; found {
if strings.Contains(k, "sonic-") {
sonic_entries = append(sonic_entries, v)
} else if oc_entries[k] == nil {
oc_entries[k] = v
}
}
}

// populate model capabilities data
for yngMdlNm := range(xMdlCpbltMap) {
for yngMdlNm := range xMdlCpbltMap {
org := ""
ver := ""
ocVerSet := false
yngEntry := oc_entries[yngMdlNm]
if (yngEntry != nil) {
if yngEntry != nil {
// OC yang has version in standard extension oc-ext:openconfig-version
if strings.HasPrefix(yngMdlNm, "openconfig-") {
for _, ext := range yngEntry.Exts {
dataTagArr := strings.Split(ext.Keyword, ":")
tagType := dataTagArr[len(dataTagArr)-1]
if tagType == "openconfig-version" {
ver = ext.NName()
fmt.Printf("Found version %v for yang module %v", ver, yngMdlNm)
xfmrLogDebug("Found version %v for yang module %v", ver, yngMdlNm)
if len(strings.TrimSpace(ver)) > 0 {
ocVerSet = true
}
break
break
}

}
}
}
if ((strings.HasPrefix(yngMdlNm, "ietf-")) || (!ocVerSet)) {
if (strings.HasPrefix(yngMdlNm, "ietf-")) || (!ocVerSet) {
// as per RFC7895 revision date to be used as version
ver = mods[yngMdlNm].Current() //gives the most recent revision date for yang module
if value, found := modProfiles[yngMdlNm]["revison"]; found {
ver = value.(string)
}
}
if value, found := modProfiles[yngMdlNm]["organization"]; found {
org = value.(string)
}
org = mods[yngMdlNm].Organization.Name
addMdlCpbltData(yngMdlNm, ver, org)
}

dbMapBuild(sonic_entries)
annotDbSpecMap(sonic_annot_entries)
annotToDbMapBuild(oc_annot_entries)
Expand Down
Loading

0 comments on commit 1340e29

Please sign in to comment.