From a0e11736787bab55f2b19e6683e404c14df4cde1 Mon Sep 17 00:00:00 2001 From: Donald Sharp Date: Fri, 7 Oct 2022 08:02:44 -0400 Subject: [PATCH] zebra: Read from the dplane_fpm_nl a route update Read from the fpm dplane a route update that will include status about whether or not the asic was successfull in offloading the route. Have this data passed up to zebra for processing and disseminate this data as appropriate. Signed-off-by: Donald Sharp --- doc/developer/fpm.rst | 16 ++++++ zebra/dplane_fpm_nl.c | 120 +++++++++++++++++++++++++++++++++++++++++- zebra/zebra_dplane.c | 10 ++-- zebra/zebra_dplane.h | 2 +- zebra/zebra_rib.c | 19 +++++-- 5 files changed, 158 insertions(+), 9 deletions(-) diff --git a/doc/developer/fpm.rst b/doc/developer/fpm.rst index 98498691330f..56d33671d2dd 100644 --- a/doc/developer/fpm.rst +++ b/doc/developer/fpm.rst @@ -101,3 +101,19 @@ Data ^^^^ The netlink or protobuf message payload. + + +Route Status Notification from ASIC +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The dplane_fpm_nl has the ability to read route netlink messages +from the underlying fpm implementation that can tell zebra +whether or not the route has been Offloaded/Failed or Trapped. +The end developer must send the data up the same socket that has +been created to listen for FPM messages from Zebra. The data sent +must have a Frame Header with Version set to 1, Message Type set to 1 +and an appropriate message Length. The message data must contain +a RTM_NEWROUTE netlink message that sends the prefix and nexthops +associated with the route. Finally rtm_flags must contain +RTM_F_OFFLOAD, RTM_F_TRAP and or RTM_F_OFFLOAD_FAILED to signify +what has happened to the route in the ASIC. diff --git a/zebra/dplane_fpm_nl.c b/zebra/dplane_fpm_nl.c index 8bbafdd5c288..337113988ec8 100644 --- a/zebra/dplane_fpm_nl.c +++ b/zebra/dplane_fpm_nl.c @@ -51,6 +51,7 @@ #include "zebra/kernel_netlink.h" #include "zebra/rt_netlink.h" #include "zebra/debug.h" +#include "fpm/fpm.h" #define SOUTHBOUND_DEFAULT_ADDR INADDR_LOOPBACK #define SOUTHBOUND_DEFAULT_PORT 2620 @@ -462,7 +463,13 @@ static void fpm_reconnect(struct fpm_nl_ctx *fnc) static void fpm_read(struct thread *t) { struct fpm_nl_ctx *fnc = THREAD_ARG(t); + fpm_msg_hdr_t fpm; ssize_t rv; + char buf[65535]; + struct nlmsghdr *hdr; + struct zebra_dplane_ctx *ctx; + size_t available_bytes; + size_t hdr_available_bytes; /* Let's ignore the input at the moment. */ rv = stream_read_try(fnc->ibuf, fnc->socket, @@ -494,11 +501,122 @@ static void fpm_read(struct thread *t) if (rv == -2) return; - stream_reset(fnc->ibuf); /* Account all bytes read. */ atomic_fetch_add_explicit(&fnc->counters.bytes_read, rv, memory_order_relaxed); + + available_bytes = STREAM_READABLE(fnc->ibuf); + while (available_bytes) { + if (available_bytes < (ssize_t)FPM_MSG_HDR_LEN) { + stream_pulldown(fnc->ibuf); + return; + } + + fpm.version = stream_getc(fnc->ibuf); + fpm.msg_type = stream_getc(fnc->ibuf); + fpm.msg_len = stream_getw(fnc->ibuf); + + if (fpm.version != FPM_PROTO_VERSION && + fpm.msg_type != FPM_MSG_TYPE_NETLINK) { + stream_reset(fnc->ibuf); + zlog_warn( + "%s: Received version/msg_type %u/%u, expected 1/1", + __func__, fpm.version, fpm.msg_type); + + FPM_RECONNECT(fnc); + return; + } + + /* + * If the passed in length doesn't even fill in the header + * something is wrong and reset. + */ + if (fpm.msg_len < FPM_MSG_HDR_LEN) { + zlog_warn( + "%s: Received message length: %u that does not even fill the FPM header", + __func__, fpm.msg_len); + FPM_RECONNECT(fnc); + return; + } + + /* + * If we have not received the whole payload, reset the stream + * back to the beginning of the header and move it to the + * top. + */ + if (fpm.msg_len > available_bytes) { + stream_rewind_getp(fnc->ibuf, FPM_MSG_HDR_LEN); + stream_pulldown(fnc->ibuf); + return; + } + + available_bytes -= FPM_MSG_HDR_LEN; + + /* + * Place the data from the stream into a buffer + */ + hdr = (struct nlmsghdr *)buf; + stream_get(buf, fnc->ibuf, fpm.msg_len - FPM_MSG_HDR_LEN); + hdr_available_bytes = fpm.msg_len - FPM_MSG_HDR_LEN; + available_bytes -= hdr_available_bytes; + + /* Sanity check: must be at least header size. */ + if (hdr->nlmsg_len < sizeof(*hdr)) { + zlog_warn( + "%s: [seq=%u] invalid message length %u (< %zu)", + __func__, hdr->nlmsg_seq, hdr->nlmsg_len, + sizeof(*hdr)); + continue; + } + if (hdr->nlmsg_len > fpm.msg_len) { + zlog_warn( + "%s: Received a inner header length of %u that is greater than the fpm total length of %u", + __func__, hdr->nlmsg_len, fpm.msg_len); + FPM_RECONNECT(fnc); + } + /* Not enough bytes available. */ + if (hdr->nlmsg_len > hdr_available_bytes) { + zlog_warn( + "%s: [seq=%u] invalid message length %u (> %zu)", + __func__, hdr->nlmsg_seq, hdr->nlmsg_len, + available_bytes); + continue; + } + + if (!(hdr->nlmsg_flags & NLM_F_REQUEST)) { + if (IS_ZEBRA_DEBUG_FPM) + zlog_debug( + "%s: [seq=%u] not a request, skipping", + __func__, hdr->nlmsg_seq); + + /* + * This request is a bust, go to the next one + */ + continue; + } + + switch (hdr->nlmsg_type) { + case RTM_NEWROUTE: + ctx = dplane_ctx_alloc(); + dplane_ctx_set_op(ctx, DPLANE_OP_ROUTE_NOTIFY); + if (netlink_route_change_read_unicast_internal( + hdr, 0, false, ctx) != 1) { + dplane_ctx_fini(&ctx); + stream_pulldown(fnc->ibuf); + return; + } + break; + default: + if (IS_ZEBRA_DEBUG_FPM) + zlog_debug( + "%s: Received message type %u which is not currently handled", + __func__, hdr->nlmsg_type); + break; + } + } + + stream_reset(fnc->ibuf); } static void fpm_write(struct thread *t) diff --git a/zebra/zebra_dplane.c b/zebra/zebra_dplane.c index 66adbe84babb..84dae7f2d604 100644 --- a/zebra/zebra_dplane.c +++ b/zebra/zebra_dplane.c @@ -2785,7 +2785,7 @@ static int dplane_ctx_ns_init(struct zebra_dplane_ctx *ctx, int dplane_ctx_route_init_basic(struct zebra_dplane_ctx *ctx, enum dplane_op_e op, struct route_entry *re, const struct prefix *p, - const struct prefix *src_p, afi_t afi, + const struct prefix_ipv6 *src_p, afi_t afi, safi_t safi) { int ret = EINVAL; @@ -2836,7 +2836,8 @@ int dplane_ctx_route_init(struct zebra_dplane_ctx *ctx, enum dplane_op_e op, int ret = EINVAL; const struct route_table *table = NULL; const struct rib_table_info *info; - const struct prefix *p, *src_p; + const struct prefix *p; + const struct prefix_ipv6 *src_p; struct zebra_ns *zns; struct zebra_vrf *zvrf; struct nexthop *nexthop; @@ -2853,7 +2854,7 @@ int dplane_ctx_route_init(struct zebra_dplane_ctx *ctx, enum dplane_op_e op, */ /* Prefixes: dest, and optional source */ - srcdest_rnode_prefixes(rn, &p, &src_p); + srcdest_rnode_prefixes(rn, &p, (const struct prefix **)&src_p); table = srcdest_rnode_table(rn); info = table->info; @@ -6309,6 +6310,9 @@ void dplane_rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p, if (!ctx) rib_add_multipath(afi, safi, p, src_p, re, ng, startup); else { + dplane_ctx_route_init_basic(ctx, dplane_ctx_get_op(ctx), re, p, + src_p, afi, safi); + dplane_provider_enqueue_to_zebra(ctx); } } diff --git a/zebra/zebra_dplane.h b/zebra/zebra_dplane.h index 3efbc33fe0a1..51f6f3d8979c 100644 --- a/zebra/zebra_dplane.h +++ b/zebra/zebra_dplane.h @@ -913,7 +913,7 @@ int dplane_ctx_route_init(struct zebra_dplane_ctx *ctx, enum dplane_op_e op, int dplane_ctx_route_init_basic(struct zebra_dplane_ctx *ctx, enum dplane_op_e op, struct route_entry *re, const struct prefix *p, - const struct prefix *src_p, afi_t afi, + const struct prefix_ipv6 *src_p, afi_t afi, safi_t safi); /* Encode next hop information into data plane context. */ diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c index a8caf0eee11c..b86780276b7a 100644 --- a/zebra/zebra_rib.c +++ b/zebra/zebra_rib.c @@ -2260,10 +2260,8 @@ static void rib_process_dplane_notify(struct zebra_dplane_ctx *ctx) } /* Ensure we clear the QUEUED flag */ - if (!zrouter.asic_offloaded) { - UNSET_FLAG(re->status, ROUTE_ENTRY_QUEUED); - UNSET_FLAG(re->status, ROUTE_ENTRY_ROUTE_REPLACING); - } + UNSET_FLAG(re->status, ROUTE_ENTRY_QUEUED); + UNSET_FLAG(re->status, ROUTE_ENTRY_ROUTE_REPLACING); /* Is this a notification that ... matters? We mostly care about * the route that is currently selected for installation; we may also @@ -2306,6 +2304,19 @@ static void rib_process_dplane_notify(struct zebra_dplane_ctx *ctx) dplane_ctx_get_type(ctx))); } goto done; + } else { + uint32_t flags = dplane_ctx_get_flags(ctx); + + if (CHECK_FLAG(flags, ZEBRA_FLAG_OFFLOADED)) { + UNSET_FLAG(re->flags, ZEBRA_FLAG_OFFLOAD_FAILED); + SET_FLAG(re->flags, ZEBRA_FLAG_OFFLOADED); + } + if (CHECK_FLAG(flags, ZEBRA_FLAG_OFFLOAD_FAILED)) { + UNSET_FLAG(re->flags, ZEBRA_FLAG_OFFLOADED); + SET_FLAG(re->flags, ZEBRA_FLAG_OFFLOAD_FAILED); + } + if (CHECK_FLAG(flags, ZEBRA_FLAG_TRAPPED)) + SET_FLAG(re->flags, ZEBRA_FLAG_TRAPPED); } /* We'll want to determine whether the installation status of the