Skip to content

Commit

Permalink
feat: add lvms tag to volume groups
Browse files Browse the repository at this point in the history
Signed-off-by: Suleyman Akbas <[email protected]>
  • Loading branch information
suleymanakbas91 committed Oct 10, 2023
1 parent 441ca07 commit 4aad88c
Show file tree
Hide file tree
Showing 4 changed files with 324 additions and 11 deletions.
85 changes: 78 additions & 7 deletions cmd/vgmanager/vgmanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@ limitations under the License.
package vgmanager

import (
"context"
"fmt"
"os"

"github.com/go-logr/logr"
lvmv1 "github.com/openshift/lvm-operator/api/v1alpha1"
"github.com/openshift/lvm-operator/internal/controllers/vgmanager"
"github.com/openshift/lvm-operator/internal/controllers/vgmanager/dmsetup"
"github.com/openshift/lvm-operator/internal/controllers/vgmanager/filter"
Expand All @@ -29,18 +31,23 @@ import (
"github.com/openshift/lvm-operator/internal/controllers/vgmanager/lvmd"
"github.com/openshift/lvm-operator/internal/controllers/vgmanager/wipefs"
"github.com/spf13/cobra"
"sigs.k8s.io/controller-runtime/pkg/cache"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/webhook"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
corev1helper "k8s.io/component-helpers/scheduling/corev1"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"

ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)

const (
Expand Down Expand Up @@ -79,7 +86,20 @@ func NewCmd(opts *Options) *cobra.Command {
return cmd
}

func run(_ *cobra.Command, _ []string, opts *Options) error {
func run(cmd *cobra.Command, _ []string, opts *Options) error {
lvm := lvm.NewDefaultHostLVM()
nodeName := os.Getenv("NODE_NAME")
namespace := os.Getenv("POD_NAMESPACE")

setupClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: opts.Scheme})
if err != nil {
return fmt.Errorf("unable to initialize setup client for pre-manager startup checks: %w", err)
}

if err := addTagToVGs(cmd.Context(), setupClient, lvm, nodeName, namespace); err != nil {
opts.SetupLog.Error(err, "failed to tag existing volume groups")
}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: opts.Scheme,
Metrics: metricsserver.Options{
Expand Down Expand Up @@ -110,9 +130,9 @@ func run(_ *cobra.Command, _ []string, opts *Options) error {
LSBLK: lsblk.NewDefaultHostLSBLK(),
Wipefs: wipefs.NewDefaultHostWipefs(),
Dmsetup: dmsetup.NewDefaultHostDmsetup(),
LVM: lvm.NewDefaultHostLVM(),
NodeName: os.Getenv("NODE_NAME"),
Namespace: os.Getenv("POD_NAMESPACE"),
LVM: lvm,
NodeName: nodeName,
Namespace: namespace,
Filters: filter.DefaultFilters,
}).SetupWithManager(mgr); err != nil {
return fmt.Errorf("unable to create controller VGManager: %w", err)
Expand All @@ -132,3 +152,54 @@ func run(_ *cobra.Command, _ []string, opts *Options) error {

return nil
}

// addTagToVGs adds a lvms tag to the existing volume groups. This is a temporary logic that should be removed in v4.16.
func addTagToVGs(ctx context.Context, c client.Client, lvm lvm.LVM, nodeName string, namespace string) error {
logger := log.FromContext(ctx)

vgs, err := lvm.ListVGs()
if err != nil {
return fmt.Errorf("failed to list volume groups: %w", err)
}

lvmVolumeGroupList := &lvmv1.LVMVolumeGroupList{}
err = c.List(ctx, lvmVolumeGroupList, &client.ListOptions{Namespace: namespace})
if err != nil {
return fmt.Errorf("failed to list LVMVolumeGroups: %w", err)
}

// If there is a matching LVMVolumeGroup CR, tag the existing volume group
for _, vg := range vgs {
tagged := false
for _, lvmVolumeGroup := range lvmVolumeGroupList.Items {
if vg.Name != lvmVolumeGroup.Name {
continue
}
if lvmVolumeGroup.Spec.NodeSelector != nil {
node := &corev1.Node{}
err = c.Get(ctx, types.NamespacedName{Name: nodeName}, node)
if err != nil {
return fmt.Errorf("failed to get node %s: %w", nodeName, err)
}

matches, err := corev1helper.MatchNodeSelectorTerms(node, lvmVolumeGroup.Spec.NodeSelector)
if err != nil {
return fmt.Errorf("failed to match nodeSelector to node labels: %w", err)
}
if !matches {
continue
}
}

if err := lvm.AddTagToVG(vg.Name); err != nil {
return err
}
tagged = true
}
if !tagged {
logger.Info("skipping tagging volume group %s as there is no corresponding LVMVolumeGroup CR", vg.Name)
}
}

return nil
}
181 changes: 181 additions & 0 deletions cmd/vgmanager/vgmanager_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
package vgmanager

import (
"context"
"github.com/openshift/lvm-operator/api/v1alpha1"
"github.com/openshift/lvm-operator/internal/controllers/vgmanager/lvm"
lvmmocks "github.com/openshift/lvm-operator/internal/controllers/vgmanager/lvm/mocks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"testing"
)

func TestAddTagToVGs(t *testing.T) {
namespace := "test-namespace"
nodeName := "test-node"
hostnameLabelKey := "kubernetes.io/hostname"
vgName := "vgtest1"

testCases := []struct {
name string
clientObjects []client.Object
volumeGroups []lvm.VolumeGroup
addTagCount int
}{
{
name: "there is a matching CR in the same namespace, add a tag",
clientObjects: []client.Object{
&v1alpha1.LVMVolumeGroup{ObjectMeta: metav1.ObjectMeta{
Name: vgName,
Namespace: namespace,
}},
},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
},
addTagCount: 1,
},
{
name: "there are two matching CRs in the same namespace, add tags",
clientObjects: []client.Object{
&v1alpha1.LVMVolumeGroup{ObjectMeta: metav1.ObjectMeta{
Name: vgName,
Namespace: namespace,
}},
&v1alpha1.LVMVolumeGroup{ObjectMeta: metav1.ObjectMeta{
Name: "vgtest2",
Namespace: namespace,
}},
},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
{
Name: "vgtest2",
},
},
addTagCount: 2,
},
{
name: "there is no matching CR, do not add a tag",
clientObjects: []client.Object{},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
},
addTagCount: 0,
},
{
name: "there is a matching CR in a different namespace, do not add a tag",
clientObjects: []client.Object{
&v1alpha1.LVMVolumeGroup{ObjectMeta: metav1.ObjectMeta{
Name: vgName,
Namespace: "test-namespace-2",
}},
},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
},
addTagCount: 0,
},
{
name: "there is a matching CR with a matching node selector, add a tag",
clientObjects: []client.Object{
&v1alpha1.LVMVolumeGroup{
ObjectMeta: metav1.ObjectMeta{
Name: vgName,
Namespace: namespace,
},
Spec: v1alpha1.LVMVolumeGroupSpec{
NodeSelector: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: hostnameLabelKey,
Operator: corev1.NodeSelectorOpIn,
Values: []string{nodeName},
},
},
},
},
},
},
},
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName, Labels: map[string]string{
hostnameLabelKey: nodeName,
}}},
},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
},
addTagCount: 1,
},
{
name: "there is a matching CR with a non-matching node selector, do not add a tag",
clientObjects: []client.Object{
&v1alpha1.LVMVolumeGroup{
ObjectMeta: metav1.ObjectMeta{
Name: vgName,
Namespace: namespace,
},
Spec: v1alpha1.LVMVolumeGroupSpec{
NodeSelector: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: hostnameLabelKey,
Operator: corev1.NodeSelectorOpIn,
Values: []string{"other-node-name"},
},
},
},
},
},
},
},
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName, Labels: map[string]string{
hostnameLabelKey: nodeName,
}}},
},
volumeGroups: []lvm.VolumeGroup{
{
Name: vgName,
},
},
addTagCount: 0,
},
}

mockLVM := lvmmocks.NewMockLVM(t)

scheme, err := v1alpha1.SchemeBuilder.Build()
assert.NoError(t, err, "creating scheme")
err = corev1.AddToScheme(scheme)
assert.NoError(t, err, "adding corev1 to scheme")

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tc.clientObjects...).Build()
if tc.addTagCount > 0 {
mockLVM.EXPECT().AddTagToVG(mock.Anything).Return(nil).Times(tc.addTagCount)
}
mockLVM.EXPECT().ListVGs().Return(tc.volumeGroups, nil).Once()
err := addTagToVGs(context.Background(), c, mockLVM, nodeName, namespace)
assert.NoError(t, err)
})
}
}
27 changes: 23 additions & 4 deletions internal/controllers/vgmanager/lvm/lvm.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ const (
lvRemoveCmd = "/usr/sbin/lvremove"
lvChangeCmd = "/usr/sbin/lvchange"
lvmDevicesCmd = "/usr/sbin/lvmdevices"

lvmsTag = "@lvms"
)

// VGReport represents the output of the `vgs --reportformat json` command
Expand Down Expand Up @@ -88,6 +90,7 @@ type LogicalVolume struct {
type LVM interface {
CreateVG(vg VolumeGroup) error
ExtendVG(vg VolumeGroup, pvs []string) (VolumeGroup, error)
AddTagToVG(vgName string) error
DeleteVG(vg VolumeGroup) error
GetVG(name string) (VolumeGroup, error)

Expand Down Expand Up @@ -153,7 +156,7 @@ type PhysicalVolume struct {
DevSize string `json:"dev_size"`
}

// Create creates a new volume group
// CreateVG creates a new volume group
func (hlvm *HostLVM) CreateVG(vg VolumeGroup) error {
if vg.Name == "" {
return fmt.Errorf("failed to create volume group. Volume group name is empty")
Expand All @@ -163,7 +166,7 @@ func (hlvm *HostLVM) CreateVG(vg VolumeGroup) error {
return fmt.Errorf("failed to create volume group. Physical volume list is empty")
}

args := []string{vg.Name}
args := []string{vg.Name, "--addtag", lvmsTag}

for _, pv := range vg.PVs {
args = append(args, pv.PvName)
Expand Down Expand Up @@ -202,6 +205,22 @@ func (hlvm *HostLVM) ExtendVG(vg VolumeGroup, pvs []string) (VolumeGroup, error)
return vg, nil
}

// AddTagToVG adds a lvms tag to the volume group
func (hlvm *HostLVM) AddTagToVG(vgName string) error {
if vgName == "" {
return fmt.Errorf("failed to add tag to the volume group. Volume group name is empty")
}

args := []string{vgName, "--addtag", lvmsTag}

_, err := hlvm.ExecuteCommandWithOutputAsHost(vgChangeCmd, args...)
if err != nil {
return fmt.Errorf("failed to add tag to the volume group %q. %v", vgName, err)
}

return nil
}

// Delete deletes a volume group and the physical volumes associated with it
func (hlvm *HostLVM) DeleteVG(vg VolumeGroup) error {
// Deactivate Volume Group
Expand Down Expand Up @@ -251,7 +270,7 @@ func (hlvm *HostLVM) GetVG(name string) (VolumeGroup, error) {
res := new(VGReport)

args := []string{
"vgs", "--units", "g", "--reportformat", "json",
"vgs", lvmsTag, "--units", "g", "--reportformat", "json",
}
if err := hlvm.execute(res, args...); err != nil {
return VolumeGroup{}, fmt.Errorf("failed to list volume groups. %v", err)
Expand Down Expand Up @@ -319,7 +338,7 @@ func (hlvm *HostLVM) ListPVs(vgName string) ([]PhysicalVolume, error) {
func (hlvm *HostLVM) ListVGs() ([]VolumeGroup, error) {
res := new(VGReport)

if err := hlvm.execute(res, "vgs", "--reportformat", "json"); err != nil {
if err := hlvm.execute(res, "vgs", lvmsTag, "--reportformat", "json"); err != nil {
return nil, fmt.Errorf("failed to list volume groups. %v", err)
}

Expand Down
Loading

0 comments on commit 4aad88c

Please sign in to comment.