zebra: Read from the dplane_fpm_nl a route update

Read from the fpm dplane a route update that will
include status about whether or not the asic was
successfull in offloading the route.

Have this data passed up to zebra for processing and disseminate
this data as appropriate.

Signed-off-by: Donald Sharp <[email protected]>

doc/developer/fpm.rst

@@ -101,3 +101,19 @@ Data
 ^^^^
 
 The netlink or protobuf message payload.
+
+
+Route Status Notification from ASIC
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The dplane_fpm_nl has the ability to read route netlink messages
+from the underlying fpm implementation that can tell zebra
+whether or not the route has been Offloaded/Failed or Trapped.
+The end developer must send the data up the same socket that has
+been created to listen for FPM messages from Zebra.  The data sent
+must have a Frame Header with Version set to 1, Message Type set to 1
+and an appropriate message Length.  The message data must contain
+a RTM_NEWROUTE netlink message that sends the prefix and nexthops
+associated with the route.  Finally rtm_flags must contain
+RTM_F_OFFLOAD, RTM_F_TRAP and or RTM_F_OFFLOAD_FAILED to signify
+what has happened to the route in the ASIC.

zebra/dplane_fpm_nl.c

@@ -51,6 +51,7 @@
 #include "zebra/kernel_netlink.h"
 #include "zebra/rt_netlink.h"
 #include "zebra/debug.h"
+#include "fpm/fpm.h"
 
 #define SOUTHBOUND_DEFAULT_ADDR INADDR_LOOPBACK
 #define SOUTHBOUND_DEFAULT_PORT 2620
@@ -462,7 +463,13 @@ static void fpm_reconnect(struct fpm_nl_ctx *fnc)
 static void fpm_read(struct thread *t)
 {
 	struct fpm_nl_ctx *fnc = THREAD_ARG(t);
+	fpm_msg_hdr_t fpm;
 	ssize_t rv;
+	char buf[65535];
+	struct nlmsghdr *hdr;
+	struct zebra_dplane_ctx *ctx;
+	size_t available_bytes;
+	size_t hdr_available_bytes;
 
 	/* Let's ignore the input at the moment. */
 	rv = stream_read_try(fnc->ibuf, fnc->socket,
@@ -494,11 +501,122 @@ static void fpm_read(struct thread *t)
 	if (rv == -2)
 		return;
 
-	stream_reset(fnc->ibuf);
 
 	/* Account all bytes read. */
 	atomic_fetch_add_explicit(&fnc->counters.bytes_read, rv,
 				  memory_order_relaxed);
+
+	available_bytes = STREAM_READABLE(fnc->ibuf);
+	while (available_bytes) {
+		if (available_bytes < (ssize_t)FPM_MSG_HDR_LEN) {
+			stream_pulldown(fnc->ibuf);
+			return;
+		}
+
+		fpm.version = stream_getc(fnc->ibuf);
+		fpm.msg_type = stream_getc(fnc->ibuf);
+		fpm.msg_len = stream_getw(fnc->ibuf);
+
+		if (fpm.version != FPM_PROTO_VERSION &&
+		    fpm.msg_type != FPM_MSG_TYPE_NETLINK) {
+			stream_reset(fnc->ibuf);
+			zlog_warn(
+				"%s: Received version/msg_type %u/%u, expected 1/1",
+				__func__, fpm.version, fpm.msg_type);
+
+			FPM_RECONNECT(fnc);
+			return;
+		}
+
+		/*
+		 * If the passed in length doesn't even fill in the header
+		 * something is wrong and reset.
+		 */
+		if (fpm.msg_len < FPM_MSG_HDR_LEN) {
+			zlog_warn(
+				"%s: Received message length: %u that does not even fill the FPM header",
+				__func__, fpm.msg_len);
+			FPM_RECONNECT(fnc);
+			return;
+		}
+
+		/*
+		 * If we have not received the whole payload, reset the stream
+		 * back to the beginning of the header and move it to the
+		 * top.
+		 */
+		if (fpm.msg_len > available_bytes) {
+			stream_rewind_getp(fnc->ibuf, FPM_MSG_HDR_LEN);
+			stream_pulldown(fnc->ibuf);
+			return;
+		}
+
+		available_bytes -= FPM_MSG_HDR_LEN;
+
+		/*
+		 * Place the data from the stream into a buffer
+		 */
+		hdr = (struct nlmsghdr *)buf;
+		stream_get(buf, fnc->ibuf, fpm.msg_len - FPM_MSG_HDR_LEN);
+		hdr_available_bytes = fpm.msg_len - FPM_MSG_HDR_LEN;
+		available_bytes -= hdr_available_bytes;
+
+		/* Sanity check: must be at least header size. */
+		if (hdr->nlmsg_len < sizeof(*hdr)) {
+			zlog_warn(
+				"%s: [seq=%u] invalid message length %u (< %zu)",
+				__func__, hdr->nlmsg_seq, hdr->nlmsg_len,
+				sizeof(*hdr));
+			continue;
+		}
+		if (hdr->nlmsg_len > fpm.msg_len) {
+			zlog_warn(
+				"%s: Received a inner header length of %u that is greater than the fpm total length of %u",
+				__func__, hdr->nlmsg_len, fpm.msg_len);
+			FPM_RECONNECT(fnc);
+		}
+		/* Not enough bytes available. */
+		if (hdr->nlmsg_len > hdr_available_bytes) {
+			zlog_warn(
+				"%s: [seq=%u] invalid message length %u (> %zu)",
+				__func__, hdr->nlmsg_seq, hdr->nlmsg_len,
+				available_bytes);
+			continue;
+		}
+
+		if (!(hdr->nlmsg_flags & NLM_F_REQUEST)) {
+			if (IS_ZEBRA_DEBUG_FPM)
+				zlog_debug(
+					"%s: [seq=%u] not a request, skipping",
+					__func__, hdr->nlmsg_seq);
+
+			/*
+			 * This request is a bust, go to the next one
+			 */
+			continue;
+		}
+
+		switch (hdr->nlmsg_type) {
+		case RTM_NEWROUTE:
+			ctx = dplane_ctx_alloc();
+			dplane_ctx_set_op(ctx, DPLANE_OP_ROUTE_NOTIFY);
+			if (netlink_route_change_read_unicast_internal(
+				    hdr, 0, false, ctx) != 1) {
+				dplane_ctx_fini(&ctx);
+				stream_pulldown(fnc->ibuf);
+				return;
+			}
+			break;
+		default:
+			if (IS_ZEBRA_DEBUG_FPM)
+				zlog_debug(
+					"%s: Received message type %u which is not currently handled",
+					__func__, hdr->nlmsg_type);
+			break;
+		}
+	}
+
+	stream_reset(fnc->ibuf);
 }
 
 static void fpm_write(struct thread *t)

zebra/zebra_dplane.c

@@ -2785,7 +2785,7 @@ static int dplane_ctx_ns_init(struct zebra_dplane_ctx *ctx,
 int dplane_ctx_route_init_basic(struct zebra_dplane_ctx *ctx,
 				enum dplane_op_e op, struct route_entry *re,
 				const struct prefix *p,
-				const struct prefix *src_p, afi_t afi,
+				const struct prefix_ipv6 *src_p, afi_t afi,
 				safi_t safi)
 {
 	int ret = EINVAL;
@@ -2836,7 +2836,8 @@ int dplane_ctx_route_init(struct zebra_dplane_ctx *ctx, enum dplane_op_e op,
 	int ret = EINVAL;
 	const struct route_table *table = NULL;
 	const struct rib_table_info *info;
-	const struct prefix *p, *src_p;
+	const struct prefix *p;
+	const struct prefix_ipv6 *src_p;
 	struct zebra_ns *zns;
 	struct zebra_vrf *zvrf;
 	struct nexthop *nexthop;
@@ -2853,7 +2854,7 @@ int dplane_ctx_route_init(struct zebra_dplane_ctx *ctx, enum dplane_op_e op,
 	 */
 
 	/* Prefixes: dest, and optional source */
-	srcdest_rnode_prefixes(rn, &p, &src_p);
+	srcdest_rnode_prefixes(rn, &p, (const struct prefix **)&src_p);
 	table = srcdest_rnode_table(rn);
 	info = table->info;
 
@@ -6309,6 +6310,9 @@ void dplane_rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p,
 	if (!ctx)
 		rib_add_multipath(afi, safi, p, src_p, re, ng, startup);
 	else {
+		dplane_ctx_route_init_basic(ctx, dplane_ctx_get_op(ctx), re, p,
+					    src_p, afi, safi);
+		dplane_provider_enqueue_to_zebra(ctx);
 	}
 }
 

zebra/zebra_dplane.h

@@ -913,7 +913,7 @@ int dplane_ctx_route_init(struct zebra_dplane_ctx *ctx, enum dplane_op_e op,
 int dplane_ctx_route_init_basic(struct zebra_dplane_ctx *ctx,
 				enum dplane_op_e op, struct route_entry *re,
 				const struct prefix *p,
-				const struct prefix *src_p, afi_t afi,
+				const struct prefix_ipv6 *src_p, afi_t afi,
 				safi_t safi);
 
 /* Encode next hop information into data plane context. */

zebra/zebra_rib.c

@@ -2260,10 +2260,8 @@ static void rib_process_dplane_notify(struct zebra_dplane_ctx *ctx)
 	}
 
 	/* Ensure we clear the QUEUED flag */
-	if (!zrouter.asic_offloaded) {
-		UNSET_FLAG(re->status, ROUTE_ENTRY_QUEUED);
-		UNSET_FLAG(re->status, ROUTE_ENTRY_ROUTE_REPLACING);
-	}
+	UNSET_FLAG(re->status, ROUTE_ENTRY_QUEUED);
+	UNSET_FLAG(re->status, ROUTE_ENTRY_ROUTE_REPLACING);
 
 	/* Is this a notification that ... matters? We mostly care about
 	 * the route that is currently selected for installation; we may also
@@ -2306,6 +2304,19 @@ static void rib_process_dplane_notify(struct zebra_dplane_ctx *ctx)
 						dplane_ctx_get_type(ctx)));
 		}
 		goto done;
+	} else {
+		uint32_t flags = dplane_ctx_get_flags(ctx);
+
+		if (CHECK_FLAG(flags, ZEBRA_FLAG_OFFLOADED)) {
+			UNSET_FLAG(re->flags, ZEBRA_FLAG_OFFLOAD_FAILED);
+			SET_FLAG(re->flags, ZEBRA_FLAG_OFFLOADED);
+		}
+		if (CHECK_FLAG(flags, ZEBRA_FLAG_OFFLOAD_FAILED)) {
+			UNSET_FLAG(re->flags, ZEBRA_FLAG_OFFLOADED);
+			SET_FLAG(re->flags, ZEBRA_FLAG_OFFLOAD_FAILED);
+		}
+		if (CHECK_FLAG(flags, ZEBRA_FLAG_TRAPPED))
+			SET_FLAG(re->flags, ZEBRA_FLAG_TRAPPED);
 	}
 
 	/* We'll want to determine whether the installation status of the