Initial import from sonic-mgmt-framework repo (sonic-net#10)

* Initial import from sonic-mgmt-framework repo

1) Moved yangs, translib, transformer, cvl and go library patch files
from sonic-mgmt-framework repo. Mgmt-framework base is f789b295.
These are used by both mgmt-framework and telemetry services.

2) Implemented gomodule based go dependency management. Uses vendoing
to create a private copy of the libraries under "vendor" directory and
all patches are applied on this copy. This happens automatically
during build; vendor copy is not committed to repo. New script
patches/apply.sh has been introduced to apply the patches. This can be
invoked from mgmt-framework and telemetry makefiles to patch their
respective vendor directories.

3) During build, two deb files are generated. No docker image is
generated for this repo.
    - sonic-mgmt-common_1.0.0_{arch}.deb
    - sonic-mgmt-common-codegen_1.0.0_{arch}.deb

4) sonic-mgmt-common.deb includes yangs, cvl schema and other
configuration files required by translib and cvl during runtime. Both
mgmt-framework and telemetry dockers will install it.

5) sonic-mgmt-common-codegen.deb includes source files generated by
ygot and yang code generators. When package cache is enabled sonic
build will compile the repo only when there are some changes. So code
generators may not be run always, but mgmt-framework and telemetry
repo compilation requires these generated source files. Such source
files are created by installing sonic-mgmt-common-codegen.deb on the
build slave docker itself. This package is not installed on switch.

* Fix compilation and packaging issue with yangs

* Cherry pick 32fa9d8 and 5bd7a97 from sonic-mgmt-framework

* CVL test schema fixes

CVL gotest cases require both main cvl schema files and additional
test shema files. Modified the makefile to copy the main schema files
to build/cvl/testdata/schema directory, which will be used as the
schema root by gotest.

.gitignore

@@ -0,0 +1,11 @@
+.vscode
+*.code-workspace
+build
+vendor
+__pycache__
+*.pyc
+*.rdb
+*.swp
+*.yin
+*.tree
+translib/ocbinds/ocbinds.go

Makefile

@@ -0,0 +1,89 @@
+################################################################################
+#                                                                              #
+#  Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or   #
+#  its subsidiaries.                                                           #
+#                                                                              #
+#  Licensed under the Apache License, Version 2.0 (the "License");             #
+#  you may not use this file except in compliance with the License.            #
+#  You may obtain a copy of the License at                                     #
+#                                                                              #
+#     http://www.apache.org/licenses/LICENSE-2.0                               #
+#                                                                              #
+#  Unless required by applicable law or agreed to in writing, software         #
+#  distributed under the License is distributed on an "AS IS" BASIS,           #
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.    #
+#  See the License for the specific language governing permissions and         #
+#  limitations under the License.                                              #
+#                                                                              #
+################################################################################
+
+TOPDIR := $(abspath .)
+BUILD_DIR := build
+
+GOPATH ?= /tmp/go
+GO     ?= /usr/local/go/bin/go
+
+GO_MOD     = go.mod
+GO_DEPS    = vendor/.done
+GO_PATCHES = $(shell find patches -type f)
+GOYANG_BIN = $(abspath $(BUILD_DIR)/bin/goyang)
+
+export TOPDIR GO GOPATH
+
+all: models cvl translib
+
+$(GO_MOD):
+	$(GO) mod init github.com/Azure/sonic-mgmt-common
+
+$(GO_DEPS): $(GO_MOD) $(GO_PATCHES)
+	$(GO) mod vendor
+	patches/apply.sh vendor
+	touch  $@
+
+go-deps: $(GO_DEPS)
+
+go-deps-clean:
+	$(RM) -r vendor
+
+.PHONY: cvl
+cvl: $(GO_DEPS)
+	$(MAKE) -C ./cvl
+
+cvl-all: $(GO_DEPS)
+	$(MAKE) -C ./cvl all
+
+cvl-clean:
+	$(MAKE) -C ./cvl clean
+
+cvl-test:
+	$(MAKE) -C ./cvl gotest
+
+.PHONY: translib
+translib: $(GO_DEPS)
+	$(MAKE) -C ./translib
+
+translib-all: $(GO_DEPS)
+	$(MAKE) -C ./translib all
+
+translib-clean:
+	$(MAKE) -C ./translib clean
+
+.PHONY: models
+models:
+	$(MAKE) -C models/yang
+
+models-clean:
+	$(MAKE) -C models/yang clean
+
+annotgen: $(GOYANG_BIN)
+
+$(GOYANG_BIN): $(GO_DEPS)
+	cd vendor/github.com/openconfig/goyang && \
+		$(GO) build -o $@ *.go
+
+clean: models-clean translib-clean cvl-clean
+	git check-ignore debian/* | xargs -r $(RM) -r
+	$(RM) -r $(BUILD_DIR)
+
+cleanall: clean go-deps-clean
+	git clean -fdX tools

config/transformer/models_list

@@ -0,0 +1,3 @@
+#List yang models transformer need to load
+openconfig-acl.yang
+openconfig-acl-annot.yang

cvl/Makefile

@@ -0,0 +1,72 @@
+################################################################################
+#                                                                              #
+#  Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or   #
+#  its subsidiaries.                                                           #
+#                                                                              #
+#  Licensed under the Apache License, Version 2.0 (the "License");             #
+#  you may not use this file except in compliance with the License.            #
+#  You may obtain a copy of the License at                                     #
+#                                                                              #
+#     http://www.apache.org/licenses/LICENSE-2.0                               #
+#                                                                              #
+#  Unless required by applicable law or agreed to in writing, software         #
+#  distributed under the License is distributed on an "AS IS" BASIS,           #
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.    #
+#  See the License for the specific language governing permissions and         #
+#  limitations under the License.                                              #
+#                                                                              #
+################################################################################
+
+GO?=go
+SRC_FILES=$(shell find . -name '*.go' | grep -v '_test.go' | grep -v '/tests/')
+TEST_FILES=$(wildcard *_test.go)
+TOP_DIR := ..
+BUILD_DIR:=$(TOP_DIR)/build/cvl
+CVL_PKG=$(TOP_DIR)/build/pkg/linux_amd64/cvl.a
+
+CVL_SCHEMA_DIR = $(BUILD_DIR)/schema
+CVL_TEST_DIR = $(TOP_DIR)/build/tests/cvl
+CVL_TEST_BIN = $(CVL_TEST_DIR)/cvl.test
+CVL_TEST_SCHEMA_DIR = $(CVL_TEST_DIR)/testdata/schema
+
+DEFAULT_TARGETS = deps schema
+ifeq ($(NO_TEST_BINS),)
+DEFAULT_TARGETS += $(CVL_TEST_BIN) test-schema
+endif
+
+default: $(DEFAULT_TARGETS)
+
+all: $(DEFAULT_TARGETS) $(CVL_PKG)
+
+deps: 
+	mkdir -p $(BUILD_DIR)
+
+$(CVL_PKG): $(SRC_FILES)
+	@echo "Building $@"
+	$(GO) build -mod=vendor -o $(CVL_PKG) ../cvl
+
+$(CVL_TEST_BIN): $(TEST_FILES) $(SRC_FILES) | test-schema
+	cp -r testdata/*.json $(@D)/testdata
+	$(GO) test -mod=vendor -cover -coverpkg=../cvl,../cvl/internal/util,../cvl/internal/yparser -c ../cvl -o $@
+
+.PHONY: schema
+schema:
+	$(MAKE) -C schema
+
+test-schema: | schema
+	$(MAKE) -C testdata/schema
+	cp $(CVL_SCHEMA_DIR)/*.yin $(CVL_TEST_SCHEMA_DIR)/
+
+tests:
+	$(MAKE) -C tests
+
+gotest:schema test-schema
+	CVL_CFG_FILE=$(abspath .)/conf/cvl_cfg.json CVL_SCHEMA_PATH=$(CVL_TEST_SCHEMA_DIR) tests/run_test.sh
+
+clean:
+	$(MAKE) -C tests clean
+	$(MAKE) -C schema clean
+	$(MAKE) -C testdata/schema clean
+	$(RM) -r $(CVL_PKG)
+	$(RM) -r $(CVL_TEST_DIR)
+

cvl/README.md

@@ -0,0 +1,62 @@
+1. Install latest version of pyang tool.
+
+2. Install libyang from https://github.com/CESNET/libyang along with its dependency.
+
+3. Run 'make' from top level 'cvl' directory.
+
+4. Refer to top level makefile rules for compiling individual targets. 
+
+5. 'schema' directory should contain all .yin files
+
+6. On the target the 'schema' directory needs to be present in the same directory where application executable file is present.
+
+
+Debugging Info:
+===============
+
+Below steps need to be done to enable CVL logging.
+
+1. Find the CVL json config file in mgmt-framework docker in switch at "/usr/sbin/cvl_cfg.json" .
+
+2. Change the logging flags from "false" to "true" as below:
+
+	{
+		"TRACE_CACHE": "true",
+		"TRACE_LIBYANG": "true",
+		"TRACE_YPARSER": "true",
+		"TRACE_CREATE": "true",
+		"TRACE_UPDATE": "true",
+		"TRACE_DELETE": "true",
+		"TRACE_SEMANTIC": "true",
+		"TRACE_SYNTAX": "true",
+		"__comment1__": "Set LOGTOSTDER to 'true' to log on standard error",
+		"LOGTOSTDERR": "true",
+		"__comment2__": "Display log upto INFO level",
+		"STDERRTHRESHOLD": "INFO",
+		"__comment3__": "Display log upto INFO level 8",
+		"VERBOSITY": "8",
+		"SKIP_VALIDATION": "false",
+		"SKIP_SEMANTIC_VALIDATION": "false"
+	}
+3. Below environment variables need to be set at the end in /usr/bin/rest-server.sh in mgmt-framework docker. 
+
+   export CVL_DEBUG=1
+   export CVL_CFG_FILE=/usr/sbin/cvl_cfg.json
+
+  Note : CVL_CFG_FILE enviroment variable can point to other location also.
+
+4. CVL Traces can be enabled both with restart and without mgmt-framework docker restart .
+
+	With Restart:
+	============
+ 	Restart mgmt-framework docker after which updated cvl_cfg.json file will be read. 
+
+	Without Restart:
+	===============
+	Issue SIGUSR2 to rest process(kill -SIGUSR2 <pid of rest process inside docker> , to read changed cvl_cfg.json with logging enabled. 
+
+5. After following above steps, CVL traces can be seen in syslog file in host container at /var/log/syslog. 
+
+6. To disable CVL traces , disable the fields in cvl_cfg.json file and then perform same steps as in Step 4.
+
+

cvl/conf/cvl_cfg.json

@@ -0,0 +1,20 @@
+{
+	"TRACE_CACHE": "false",
+	"TRACE_LIBYANG": "false",
+	"TRACE_YPARSER": "false", 
+	"TRACE_CREATE": "false",
+	"TRACE_UPDATE": "false",
+	"TRACE_DELETE": "false",
+	"TRACE_SEMANTIC": "false",
+	"TRACE_SYNTAX": "false",
+	"__comment1__": "Log trace data when error occurs",
+	"TRACE_ONERROR": "true",
+	"__comment2__": "Set LOGTOSTDER to 'true' to log on standard error",
+	"LOGTOSTDERR": "false",
+	"__comment3__": "Display log upto INFO level",
+	"STDERRTHRESHOLD": "ERROR",
+	"__comment4__": "Display log upto INFO level 8",
+	"VERBOSITY": "0",
+	"SKIP_VALIDATION": "false",
+	"SKIP_SEMANTIC_VALIDATION": "false"
+}