Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OCPVE-603: Add lvms tag to volume groups #447

Merged
merged 1 commit into from
Oct 10, 2023
Merged

OCPVE-603: Add lvms tag to volume groups #447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 78 additions & 7 deletions cmd/vgmanager/vgmanager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@ limitations under the License.
package vgmanager

import (
"context"
"fmt"
"os"

"github.com/go-logr/logr"
lvmv1 "github.com/openshift/lvm-operator/api/v1alpha1"
"github.com/openshift/lvm-operator/internal/controllers/vgmanager"
"github.com/openshift/lvm-operator/internal/controllers/vgmanager/dmsetup"
"github.com/openshift/lvm-operator/internal/controllers/vgmanager/filter"
Expand All @@ -29,18 +31,23 @@ import (
"github.com/openshift/lvm-operator/internal/controllers/vgmanager/lvmd"
"github.com/openshift/lvm-operator/internal/controllers/vgmanager/wipefs"
"github.com/spf13/cobra"
"sigs.k8s.io/controller-runtime/pkg/cache"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/webhook"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
corev1helper "k8s.io/component-helpers/scheduling/corev1"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"

ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)

const (
Expand Down Expand Up @@ -79,7 +86,20 @@ func NewCmd(opts *Options) *cobra.Command {
return cmd
}

func run(_ *cobra.Command, _ []string, opts *Options) error {
func run(cmd *cobra.Command, _ []string, opts *Options) error {
lvm := lvm.NewDefaultHostLVM()
nodeName := os.Getenv("NODE_NAME")
namespace := os.Getenv("POD_NAMESPACE")

setupClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: opts.Scheme})
if err != nil {
return fmt.Errorf("unable to initialize setup client for pre-manager startup checks: %w", err)
}

if err := addTagToVGs(cmd.Context(), setupClient, lvm, nodeName, namespace); err != nil {
opts.SetupLog.Error(err, "failed to tag existing volume groups")
}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: opts.Scheme,
Metrics: metricsserver.Options{
Expand Down Expand Up @@ -110,9 +130,9 @@ func run(_ *cobra.Command, _ []string, opts *Options) error {
LSBLK: lsblk.NewDefaultHostLSBLK(),
Wipefs: wipefs.NewDefaultHostWipefs(),
Dmsetup: dmsetup.NewDefaultHostDmsetup(),
LVM: lvm.NewDefaultHostLVM(),
NodeName: os.Getenv("NODE_NAME"),
Namespace: os.Getenv("POD_NAMESPACE"),
LVM: lvm,
NodeName: nodeName,
Namespace: namespace,
Filters: filter.DefaultFilters,
}).SetupWithManager(mgr); err != nil {
return fmt.Errorf("unable to create controller VGManager: %w", err)
Expand All @@ -132,3 +152,54 @@ func run(_ *cobra.Command, _ []string, opts *Options) error {

return nil
}

// addTagToVGs adds a lvms tag to the existing volume groups. This is a temporary logic that should be removed in v4.16.
func addTagToVGs(ctx context.Context, c client.Client, lvm lvm.LVM, nodeName string, namespace string) error {
logger := log.FromContext(ctx)

vgs, err := lvm.ListVGs()
if err != nil {
return fmt.Errorf("failed to list volume groups: %w", err)
}

lvmVolumeGroupList := &lvmv1.LVMVolumeGroupList{}
err = c.List(ctx, lvmVolumeGroupList, &client.ListOptions{Namespace: namespace})
if err != nil {
return fmt.Errorf("failed to list LVMVolumeGroups: %w", err)
}

// If there is a matching LVMVolumeGroup CR, tag the existing volume group
for _, vg := range vgs {
tagged := false
for _, lvmVolumeGroup := range lvmVolumeGroupList.Items {
if vg.Name != lvmVolumeGroup.Name {
continue
}
if lvmVolumeGroup.Spec.NodeSelector != nil {
node := &corev1.Node{}
err = c.Get(ctx, types.NamespacedName{Name: nodeName}, node)
if err != nil {
return fmt.Errorf("failed to get node %s: %w", nodeName, err)
}

matches, err := corev1helper.MatchNodeSelectorTerms(node, lvmVolumeGroup.Spec.NodeSelector)
if err != nil {
return fmt.Errorf("failed to match nodeSelector to node labels: %w", err)
}
if !matches {
continue
}
}

if err := lvm.AddTagToVG(vg.Name); err != nil {
return err
}
tagged = true
}
if !tagged {
logger.Info("skipping tagging volume group %s as there is no corresponding LVMVolumeGroup CR", vg.Name)
}
}

return nil
}
183 changes: 183 additions & 0 deletions cmd/vgmanager/vgmanager_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
package vgmanager

import (
"context"
"testing"

"github.com/openshift/lvm-operator/api/v1alpha1"
"github.com/openshift/lvm-operator/internal/controllers/vgmanager/lvm"
lvmmocks "github.com/openshift/lvm-operator/internal/controllers/vgmanager/lvm/mocks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)

func TestAddTagToVGs(t *testing.T) {
namespace := "test-namespace"
nodeName := "test-node"
hostnameLabelKey := "kubernetes.io/hostname"
vgName := "vgtest1"

testCases := []struct {
name string
clientObjects []client.Object
volumeGroups []lvm.VolumeGroup
addTagCount int
}{
{
name: "there is a matching CR in the same namespace, add a tag",
clientObjects: []client.Object{
&v1alpha1.LVMVolumeGroup{ObjectMeta: metav1.ObjectMeta{
Name: vgName,
Namespace: namespace,
}},
},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
},
addTagCount: 1,
},
{
name: "there are two matching CRs in the same namespace, add tags",
clientObjects: []client.Object{
&v1alpha1.LVMVolumeGroup{ObjectMeta: metav1.ObjectMeta{
Name: vgName,
Namespace: namespace,
}},
&v1alpha1.LVMVolumeGroup{ObjectMeta: metav1.ObjectMeta{
Name: "vgtest2",
Namespace: namespace,
}},
},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
{
Name: "vgtest2",
},
},
addTagCount: 2,
},
{
name: "there is no matching CR, do not add a tag",
clientObjects: []client.Object{},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
},
addTagCount: 0,
},
{
name: "there is a matching CR in a different namespace, do not add a tag",
clientObjects: []client.Object{
&v1alpha1.LVMVolumeGroup{ObjectMeta: metav1.ObjectMeta{
Name: vgName,
Namespace: "test-namespace-2",
}},
},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
},
addTagCount: 0,
},
{
name: "there is a matching CR with a matching node selector, add a tag",
clientObjects: []client.Object{
&v1alpha1.LVMVolumeGroup{
ObjectMeta: metav1.ObjectMeta{
Name: vgName,
Namespace: namespace,
},
Spec: v1alpha1.LVMVolumeGroupSpec{
NodeSelector: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: hostnameLabelKey,
Operator: corev1.NodeSelectorOpIn,
Values: []string{nodeName},
},
},
},
},
},
},
},
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName, Labels: map[string]string{
hostnameLabelKey: nodeName,
}}},
},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
},
addTagCount: 1,
},
{
name: "there is a matching CR with a non-matching node selector, do not add a tag",
clientObjects: []client.Object{
&v1alpha1.LVMVolumeGroup{
ObjectMeta: metav1.ObjectMeta{
Name: vgName,
Namespace: namespace,
},
Spec: v1alpha1.LVMVolumeGroupSpec{
NodeSelector: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: hostnameLabelKey,
Operator: corev1.NodeSelectorOpIn,
Values: []string{"other-node-name"},
},
},
},
},
},
},
},
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName, Labels: map[string]string{
hostnameLabelKey: nodeName,
}}},
},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
},
addTagCount: 0,
},
}

mockLVM := lvmmocks.NewMockLVM(t)

scheme, err := v1alpha1.SchemeBuilder.Build()
assert.NoError(t, err, "creating scheme")
err = corev1.AddToScheme(scheme)
assert.NoError(t, err, "adding corev1 to scheme")

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tc.clientObjects...).Build()
if tc.addTagCount > 0 {
mockLVM.EXPECT().AddTagToVG(mock.Anything).Return(nil).Times(tc.addTagCount)
}
mockLVM.EXPECT().ListVGs().Return(tc.volumeGroups, nil).Once()
err := addTagToVGs(context.Background(), c, mockLVM, nodeName, namespace)
assert.NoError(t, err)
})
}
}
27 changes: 23 additions & 4 deletions internal/controllers/vgmanager/lvm/lvm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ const (
lvRemoveCmd = "/usr/sbin/lvremove"
lvChangeCmd = "/usr/sbin/lvchange"
lvmDevicesCmd = "/usr/sbin/lvmdevices"

lvmsTag = "@lvms"
)

// VGReport represents the output of the `vgs --reportformat json` command
Expand Down Expand Up @@ -88,6 +90,7 @@ type LogicalVolume struct {
type LVM interface {
CreateVG(vg VolumeGroup) error
ExtendVG(vg VolumeGroup, pvs []string) (VolumeGroup, error)
AddTagToVG(vgName string) error
DeleteVG(vg VolumeGroup) error
GetVG(name string) (VolumeGroup, error)

Expand Down Expand Up @@ -153,7 +156,7 @@ type PhysicalVolume struct {
DevSize string `json:"dev_size"`
}

// Create creates a new volume group
// CreateVG creates a new volume group
func (hlvm *HostLVM) CreateVG(vg VolumeGroup) error {
if vg.Name == "" {
return fmt.Errorf("failed to create volume group. Volume group name is empty")
Expand All @@ -163,7 +166,7 @@ func (hlvm *HostLVM) CreateVG(vg VolumeGroup) error {
return fmt.Errorf("failed to create volume group. Physical volume list is empty")
}

args := []string{vg.Name}
args := []string{vg.Name, "--addtag", lvmsTag}

for _, pv := range vg.PVs {
args = append(args, pv.PvName)
Expand Down Expand Up @@ -202,6 +205,22 @@ func (hlvm *HostLVM) ExtendVG(vg VolumeGroup, pvs []string) (VolumeGroup, error)
return vg, nil
}

// AddTagToVG adds a lvms tag to the volume group
func (hlvm *HostLVM) AddTagToVG(vgName string) error {
if vgName == "" {
return fmt.Errorf("failed to add tag to the volume group. Volume group name is empty")
}

args := []string{vgName, "--addtag", lvmsTag}

_, err := hlvm.ExecuteCommandWithOutputAsHost(vgChangeCmd, args...)
if err != nil {
return fmt.Errorf("failed to add tag to the volume group %q. %v", vgName, err)
}

return nil
}

// Delete deletes a volume group and the physical volumes associated with it
func (hlvm *HostLVM) DeleteVG(vg VolumeGroup) error {
// Deactivate Volume Group
Expand Down Expand Up @@ -251,7 +270,7 @@ func (hlvm *HostLVM) GetVG(name string) (VolumeGroup, error) {
res := new(VGReport)

args := []string{
"vgs", "--units", "g", "--reportformat", "json",
"vgs", lvmsTag, "--units", "g", "--reportformat", "json",
}
if err := hlvm.execute(res, args...); err != nil {
return VolumeGroup{}, fmt.Errorf("failed to list volume groups. %v", err)
Expand Down Expand Up @@ -319,7 +338,7 @@ func (hlvm *HostLVM) ListPVs(vgName string) ([]PhysicalVolume, error) {
func (hlvm *HostLVM) ListVGs() ([]VolumeGroup, error) {
res := new(VGReport)

if err := hlvm.execute(res, "vgs", "--reportformat", "json"); err != nil {
if err := hlvm.execute(res, "vgs", lvmsTag, "--reportformat", "json"); err != nil {
return nil, fmt.Errorf("failed to list volume groups. %v", err)
}

Expand Down
Loading