Skip to content

Commit

Permalink
CVL Changes sonic-net#10: Custom Validation infra, Unit Test reorgani…
Browse files Browse the repository at this point in the history
…zation and new test cases addition (sonic-net#39)

Adding Custom Validation Infrastructure
Unit Test files reorganization
Adding Test YANG Schemas
Adding Unit Test for new CVL APIs
Adding Unit Test for leafref, when, must expression evaluation
  • Loading branch information
dutta-partha authored Jan 12, 2021
1 parent 4aa0899 commit d90d239
Show file tree
Hide file tree
Showing 33 changed files with 4,238 additions and 838 deletions.
2 changes: 1 addition & 1 deletion cvl/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ schema:

test-schema: | schema
$(MAKE) -C testdata/schema
cp $(CVL_SCHEMA_DIR)/*.yin $(CVL_TEST_SCHEMA_DIR)/
cp -n $(CVL_SCHEMA_DIR)/*.yin $(CVL_TEST_SCHEMA_DIR)/

tests:
$(MAKE) -C tests
Expand Down
18 changes: 13 additions & 5 deletions cvl/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ Below steps need to be done to enable CVL logging.

2. Change the logging flags from "false" to "true" as below:

```
{
"TRACE_CACHE": "true",
"TRACE_LIBYANG": "true",
Expand All @@ -29,15 +30,22 @@ Below steps need to be done to enable CVL logging.
"TRACE_DELETE": "true",
"TRACE_SEMANTIC": "true",
"TRACE_SYNTAX": "true",
"TRACE_ONERROR": "true",
"__comment1__": "Set LOGTOSTDER to 'true' to log on standard error",
"LOGTOSTDERR": "true",
"__comment2__": "Display log upto INFO level",
"STDERRTHRESHOLD": "INFO",
"__comment3__": "Display log upto INFO level 8",
"VERBOSITY": "8",
"LOGTOSTDERR": "false",
"__comment2__": "Log messages to standard error at or above this severity level",
"STDERRTHRESHOLD": "ERROR",
"__comment3__": "Log to /tmp/cvl.log file",
"LOG_TO_FILE": "true",
"__comment4__": "Limit log file size in bytes, 0 means no limit, default 10MB",
"LOG_FILE_SIZE": "10485760",
"__comment5__": "Set verbosity level(1 to 8) for verbose logs",
"VERBOSITY": "0",
"SKIP_VALIDATION": "false",
"SKIP_SEMANTIC_VALIDATION": "false"
}
```

3. Below environment variables need to be set at the end in /usr/bin/rest-server.sh in mgmt-framework docker.

export CVL_DEBUG=1
Expand Down
13 changes: 8 additions & 5 deletions cvl/conf/cvl_cfg.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,16 @@
"TRACE_DELETE": "false",
"TRACE_SEMANTIC": "false",
"TRACE_SYNTAX": "false",
"__comment1__": "Log trace data when error occurs",
"TRACE_ONERROR": "true",
"__comment2__": "Set LOGTOSTDER to 'true' to log on standard error",
"TRACE_ONERROR": "false",
"__comment1__": "Set LOGTOSTDER to 'true' to log on standard error",
"LOGTOSTDERR": "false",
"__comment3__": "Display log upto INFO level",
"__comment2__": "Display log upto INFO level",
"STDERRTHRESHOLD": "ERROR",
"__comment4__": "Display log upto INFO level 8",
"__comment3__": "Log to /tmp/cvl.log file",
"LOG_TO_FILE": "false",
"__comment4__": "Limit log file size in bytes, 0 means no limit, default 10MB",
"LOG_FILE_SIZE": "10485760",
"__comment5__": "Display log upto INFO level 8",
"VERBOSITY": "0",
"SKIP_VALIDATION": "false",
"SKIP_SEMANTIC_VALIDATION": "false"
Expand Down
132 changes: 132 additions & 0 deletions cvl/custom_validation/common.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
////////////////////////////////////////////////////////////////////////////////
// //
// Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
// its subsidiaries. //
// //
// Licensed under the Apache License, Version 2.0 (the "License"); //
// you may not use this file except in compliance with the License. //
// You may obtain a copy of the License at //
// //
// http://www.apache.org/licenses/LICENSE-2.0 //
// //
// Unless required by applicable law or agreed to in writing, software //
// distributed under the License is distributed on an "AS IS" BASIS, //
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
// See the License for the specific language governing permissions and //
// limitations under the License. //
// //
////////////////////////////////////////////////////////////////////////////////

package custom_validation

import (
"reflect"
"github.com/antchfx/xmlquery"
"github.com/go-redis/redis/v7"
"github.com/Azure/sonic-mgmt-common/cvl/internal/util"
"github.com/Azure/sonic-mgmt-common/cvl/internal/yparser"
)

type CustomValidation struct {}

type CVLValidateType uint
const (
VALIDATE_NONE CVLValidateType = iota //Data is used as dependent data
VALIDATE_SYNTAX //Syntax is checked and data is used as dependent data
VALIDATE_SEMANTICS //Semantics is checked
VALIDATE_ALL //Syntax and Semantics are checked
)

type CVLOperation uint
const (
OP_NONE CVLOperation = 0 //Used to just validate the config without any operation
OP_CREATE = 1 << 0//For Create operation
OP_UPDATE = 1 << 1//For Update operation
OP_DELETE = 1 << 2//For Delete operation
)

//CVLRetCode CVL Error codes
type CVLRetCode int
const (
CVL_SUCCESS CVLRetCode = iota
CVL_ERROR
CVL_NOT_IMPLEMENTED
CVL_INTERNAL_UNKNOWN
CVL_FAILURE
CVL_SYNTAX_ERROR = CVLRetCode(yparser.YP_SYNTAX_ERROR)
CVL_SEMANTIC_ERROR = CVLRetCode(yparser.YP_SEMANTIC_ERROR)
CVL_SYNTAX_MISSING_FIELD = CVLRetCode(yparser.YP_SYNTAX_MISSING_FIELD)
CVL_SYNTAX_INVALID_FIELD = CVLRetCode(yparser.YP_SYNTAX_INVALID_FIELD) /* Invalid Field */
CVL_SYNTAX_INVALID_INPUT_DATA = CVLRetCode(yparser.YP_SYNTAX_INVALID_INPUT_DATA) /*Invalid Input Data */
CVL_SYNTAX_MULTIPLE_INSTANCE = CVLRetCode(yparser.YP_SYNTAX_MULTIPLE_INSTANCE) /* Multiple Field Instances */
CVL_SYNTAX_DUPLICATE = CVLRetCode(yparser.YP_SYNTAX_DUPLICATE) /* Duplicate Fields */
CVL_SYNTAX_ENUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_ENUM_INVALID) /* Invalid enum value */
CVL_SYNTAX_ENUM_INVALID_NAME = CVLRetCode(yparser.YP_SYNTAX_ENUM_INVALID_NAME) /* Invalid enum name */
CVL_SYNTAX_ENUM_WHITESPACE = CVLRetCode(yparser.YP_SYNTAX_ENUM_WHITESPACE) /* Enum name with leading/trailing whitespaces */
CVL_SYNTAX_OUT_OF_RANGE = CVLRetCode(yparser.YP_SYNTAX_OUT_OF_RANGE) /* Value out of range/length/pattern (data) */
CVL_SYNTAX_MINIMUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_MINIMUM_INVALID) /* min-elements constraint not honored */
CVL_SYNTAX_MAXIMUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_MAXIMUM_INVALID) /* max-elements constraint not honored */
CVL_SEMANTIC_DEPENDENT_DATA_MISSING = CVLRetCode(yparser.YP_SEMANTIC_DEPENDENT_DATA_MISSING) /* Dependent Data is missing */
CVL_SEMANTIC_MANDATORY_DATA_MISSING = CVLRetCode(yparser.YP_SEMANTIC_MANDATORY_DATA_MISSING) /* Mandatory Data is missing */
CVL_SEMANTIC_KEY_ALREADY_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_ALREADY_EXIST) /* Key already existing. */
CVL_SEMANTIC_KEY_NOT_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_NOT_EXIST) /* Key is missing. */
CVL_SEMANTIC_KEY_DUPLICATE = CVLRetCode(yparser.YP_SEMANTIC_KEY_DUPLICATE) /* Duplicate key. */
CVL_SEMANTIC_KEY_INVALID = CVLRetCode(yparser.YP_SEMANTIC_KEY_INVALID)
)

//CVLEditConfigData Strcture for key and data in API
type CVLEditConfigData struct {
VType CVLValidateType //Validation type
VOp CVLOperation //Operation type
Key string //Key format : "PORT|Ethernet4"
Data map[string]string //Value : {"alias": "40GE0/28", "mtu" : 9100, "admin_status": down}
}

//CVLErrorInfo CVL Error Structure
type CVLErrorInfo struct {
TableName string /* Table having error */
ErrCode CVLRetCode /* CVL Error return Code. */
CVLErrDetails string /* CVL Error Message details. */
Keys []string /* Keys of the Table having error. */
Value string /* Field Value throwing error */
Field string /* Field Name throwing error . */
Msg string /* Detailed error message. */
ConstraintErrMsg string /* Constraint error message. */
ErrAppTag string
}

type CustValidationCache struct {
Data interface{}
}

//CustValidationCtxt Custom validation context passed to custom validation function
type CustValidationCtxt struct {
ReqData []CVLEditConfigData //All request data
CurCfg *CVLEditConfigData //Current request data for which validation should be done
YNodeName string //YANG node name
YNodeVal string //YANG node value, leaf-list will have "," separated value
YCur *xmlquery.Node //YANG data tree
SessCache *CustValidationCache //Session cache, can be used for storing data, persistent in session
RClient *redis.Client //Redis client
}

//InvokeCustomValidation Common function to invoke custom validation
//TBD should we do this using GO plugin feature ?
func InvokeCustomValidation(cv *CustomValidation, name string, args... interface{}) CVLErrorInfo {
inputs := make([]reflect.Value, len(args))
for i := range args {
inputs[i] = reflect.ValueOf(args[i])
}

f := reflect.ValueOf(cv).MethodByName(name)
if !f.IsNil() {
v := f.Call(inputs)
util.TRACE_LEVEL_LOG(util.TRACE_SEMANTIC,
"InvokeCustomValidation: %s(), CVLErrorInfo: %v", name, v[0])

return (v[0].Interface()).(CVLErrorInfo)
}

return CVLErrorInfo{ErrCode: CVL_SUCCESS}
}

63 changes: 62 additions & 1 deletion cvl/cvl.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ import (
"sync"
"io/ioutil"
"path/filepath"
custv "github.com/Azure/sonic-mgmt-common/cvl/custom_validation"
"unsafe"
)

//DB number
Expand Down Expand Up @@ -106,6 +108,7 @@ type modelTableInfo struct {
whenExpr map[string][]*whenInfo
tablesForMustExp map[string]CVLOperation
refFromTables []tblFieldPair //list of table or table/field referring to this table
custValidation map[string]string // Map for custom validation node and function name
dfltLeafVal map[string]string //map of leaf names and default value
mandatoryNodes map[string]bool //map of leaf names and mandatory flag
}
Expand Down Expand Up @@ -140,6 +143,7 @@ type CVL struct {
maxTableElem map[string]int //max element count per table
batchLeaf []*yparser.YParserLeafValue //field name and value
yv *YValidator //Custom YANG validator for validating external dependencies
custvCache custv.CustValidationCache //Custom validation cache per session
}

// Struct for model namepsace and prefix
Expand Down Expand Up @@ -449,6 +453,7 @@ func storeModelInfo(modelFile string, module *yparser.YParserModule) {
tInfo.redisTableSize = lInfo.RedisTableSize
tInfo.keys = lInfo.Keys
tInfo.mapLeaf = lInfo.MapLeaf
tInfo.custValidation = lInfo.CustValidation
tInfo.mandatoryNodes = lInfo.MandatoryNodes

//store default values used in must and when exp
Expand Down Expand Up @@ -955,6 +960,62 @@ func (c *CVL) addCfgDataItem(configData *map[string]interface{},
return tblName, key
}

//Perform user defined custom validation
func (c *CVL) doCustomValidation(node *xmlquery.Node,
custvCfg []custv.CVLEditConfigData,
curCustvCfg *custv.CVLEditConfigData, yangListName,
tbl, key string) CVLErrorInfo {

cvlErrObj := CVLErrorInfo{ErrCode : CVL_SUCCESS}

// yangListName provides the correct table name defined in sonic-yang
// For ex. VLAN_INTERFACE_LIST and VLAN_INTERFACE_IPADDR_LIST are in same container
for nodeName, custFunc := range modelInfo.tableInfo[yangListName].custValidation {
//find the node value
//node value is empty for custom validation function at list level
nodeVal := ""
if !strings.HasSuffix(nodeName, "_LIST") {
for nodeLeaf := node.FirstChild; nodeLeaf != nil;
nodeLeaf = nodeLeaf.NextSibling {
if (nodeName != nodeLeaf.Data) {
continue
}

if (len(nodeLeaf.Attr) > 0) &&
(nodeLeaf.Attr[0].Name.Local == "leaf-list") {
nodeVal = curCustvCfg.Data[nodeName]
} else {
nodeVal = nodeLeaf.FirstChild.Data
}
}

}

//Call custom validation functions
CVL_LOG(INFO_TRACE, "Calling custom validation function %s", custFunc)
pCustv := &custv.CustValidationCtxt{
ReqData: custvCfg,
CurCfg: curCustvCfg,
YNodeName: nodeName,
YNodeVal: nodeVal,
YCur: node,
SessCache: &(c.custvCache),
RClient: redisClient}

errObj := custv.InvokeCustomValidation(&custv.CustomValidation{},
custFunc, pCustv)

cvlErrObj = *(*CVLErrorInfo)(unsafe.Pointer(&errObj))

if (cvlErrObj.ErrCode != CVL_SUCCESS) {
CVL_LOG(WARNING, "Custom validation failed, Error = %v", cvlErrObj)
return cvlErrObj
}
}

return cvlErrObj
}

// getLeafRefInfo This function returns leafrefInfo structure based on table name,
// target table name and leaf node name where leafRef is present
func getLeafRefInfo(tblName, fldName, targetTblName string) *leafRefInfo {
Expand All @@ -978,4 +1039,4 @@ func isMandatoryTrueNode(tblName, field string) bool {
}

return false
}
}
16 changes: 14 additions & 2 deletions cvl/cvl_api.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package cvl

import (
"fmt"
"reflect"
"encoding/json"
"github.com/go-redis/redis/v7"
toposort "github.com/philopon/go-toposort"
Expand All @@ -29,7 +30,9 @@ import (
. "github.com/Azure/sonic-mgmt-common/cvl/internal/util"
"strings"
"github.com/antchfx/xmlquery"
"unsafe"
"runtime"
custv "github.com/Azure/sonic-mgmt-common/cvl/custom_validation"
"time"
"sync"
)
Expand Down Expand Up @@ -321,14 +324,16 @@ func (c *CVL) ValidateEditConfig(cfgData []CVLEditConfigData) (cvlErr CVLErrorIn
caller = f.Name()
}

CVL_LOG(INFO_DEBUG, "ValidateEditConfig() called from %s() : %v", caller, cfgData)
CVL_LOG(INFO_DEBUG, "ValidateEditConfig() called from %s() : %v", caller, cfgData)

if SkipValidation() {
CVL_LOG(INFO_TRACE, "Skipping CVL validation.")
return cvlErrObj, CVL_SUCCESS
}

//Type cast to custom validation cfg data
sliceHeader := *(*reflect.SliceHeader)(unsafe.Pointer(&cfgData))
custvCfg := *(*[]custv.CVLEditConfigData)(unsafe.Pointer(&sliceHeader))

c.clearTmpDbCache()
//c.yv.root.FirstChild = nil
Expand Down Expand Up @@ -544,6 +549,13 @@ func (c *CVL) ValidateEditConfig(cfgData []CVLEditConfigData) (cvlErr CVLErrorIn
continue
}

//Step 3.2 : Run all custom validations
cvlErrObj= c.doCustomValidation(node, custvCfg, &custvCfg[i], yangListName,
tbl, key)
if cvlErrObj.ErrCode != CVL_SUCCESS {
return cvlErrObj,cvlErrObj.ErrCode
}

//Step 3.3 : Perform semantic validation
if cvlErrObj = c.validateSemantics(node, yangListName, key, &cfgData[i]);
cvlErrObj.ErrCode != CVL_SUCCESS {
Expand Down Expand Up @@ -1087,4 +1099,4 @@ func (c *CVL) GetAllReferringTables(tableName string) (map[string][]string) {
}

return refTbls
}
}
Loading

0 comments on commit d90d239

Please sign in to comment.