diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 2903cd0342..85d80e16f5 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -381,6 +381,70 @@ void RouteOrch::detach(Observer *observer, const IpAddress& dstAddr, sai_object_ } } +void RouteOrch::updateDefaultRouteSwapSet(const NextHopGroupKey default_nhg_key, std::set& active_default_route_nhops) +{ + std::set current_default_route_nhops; + current_default_route_nhops.clear(); + + if (default_nhg_key.getSize() == 1) + { + current_default_route_nhops.insert(*default_nhg_key.getNextHops().begin()); + } + else + { + auto nhgm = m_syncdNextHopGroups[default_nhg_key].nhopgroup_members; + for (auto nhop = nhgm.begin(); nhop != nhgm.end(); ++nhop) + { + current_default_route_nhops.insert(nhop->first); + } + } + + active_default_route_nhops.clear(); + std::copy(current_default_route_nhops.begin(), current_default_route_nhops.end(), std::inserter(active_default_route_nhops, active_default_route_nhops.begin())); +} + +bool RouteOrch::addDefaultRouteNexthopsInNextHopGroup(NextHopGroupEntry& original_next_hop_group, std::set& default_route_next_hop_set) +{ + SWSS_LOG_ENTER(); + sai_object_id_t nexthop_group_member_id; + sai_status_t status; + + for (auto it : default_route_next_hop_set) + { + vector nhgm_attrs; + sai_attribute_t nhgm_attr; + nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID; + nhgm_attr.value.oid = original_next_hop_group.next_hop_group_id; + nhgm_attrs.push_back(nhgm_attr); + + nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID; + nhgm_attr.value.oid = m_neighOrch->getNextHopId(it); + nhgm_attrs.push_back(nhgm_attr); + + status = sai_next_hop_group_api->create_next_hop_group_member(&nexthop_group_member_id, gSwitchId, + (uint32_t)nhgm_attrs.size(), + nhgm_attrs.data()); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Default Route Swap Failed to add next hop member to group %" PRIx64 ": %d\n", + original_next_hop_group.next_hop_group_id, status); + task_process_status handle_status = handleSaiCreateStatus(SAI_API_NEXT_HOP_GROUP, status); + if (handle_status != task_success) + { + return parseHandleSaiStatusFailure(handle_status); + } + } + // Increment the Default Route Active NH Reference Count + m_neighOrch->increaseNextHopRefCount(it); + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); + original_next_hop_group.default_route_nhopgroup_members[it].next_hop_id = nexthop_group_member_id; + original_next_hop_group.default_route_nhopgroup_members[it].seq_id = 0; + original_next_hop_group.is_default_route_nh_swap = true; + } + return true; +} + bool RouteOrch::validnexthopinNextHopGroup(const NextHopKey &nexthop, uint32_t& count) { SWSS_LOG_ENTER(); @@ -398,6 +462,13 @@ bool RouteOrch::validnexthopinNextHopGroup(const NextHopKey &nexthop, uint32_t& continue; } + // Route NHOP Group is swapped by default route nh memeber . do not add Nexthop again. + // Wait for Nexthop Group Cleanup + if (nhopgroup->second.is_default_route_nh_swap) + { + continue; + } + vector nhgm_attrs; sai_attribute_t nhgm_attr; @@ -444,6 +515,7 @@ bool RouteOrch::validnexthopinNextHopGroup(const NextHopKey &nexthop, uint32_t& ++count; gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); nhopgroup->second.nhopgroup_members[nexthop].next_hop_id = nexthop_id; + nhopgroup->second.nh_member_install_count++; } if (!m_fgNhgOrch->validNextHopInNextHopGroup(nexthop)) @@ -484,7 +556,22 @@ bool RouteOrch::invalidnexthopinNextHopGroup(const NextHopKey &nexthop, uint32_t return parseHandleSaiStatusFailure(handle_status); } } - + if (nhopgroup->second.nh_member_install_count) + { + nhopgroup->second.nh_member_install_count--; + } + + if (nhopgroup->second.nh_member_install_count == 0 && nhopgroup->second.eligible_for_default_route_nh_swap && !nhopgroup->second.is_default_route_nh_swap) + { + if(nexthop.ip_address.isV4()) + { + addDefaultRouteNexthopsInNextHopGroup(nhopgroup->second, v4_active_default_route_nhops); + } + else + { + addDefaultRouteNexthopsInNextHopGroup(nhopgroup->second, v6_active_default_route_nhops); + } + } ++count; gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); } @@ -632,6 +719,7 @@ void RouteOrch::doTask(Consumer& consumer) string srv6_segments; string srv6_source; bool srv6_nh = false; + bool fallback_to_default_route = false; for (auto i : kfvFieldsValues(t)) { @@ -673,6 +761,10 @@ void RouteOrch::doTask(Consumer& consumer) { ctx.protocol = fvValue(i); } + if (fvField(i) == "fallback_to_default_route") + { + fallback_to_default_route = fvValue(i) == "true"; + } } /* @@ -687,6 +779,7 @@ void RouteOrch::doTask(Consumer& consumer) } ctx.nhg_index = nhg_index; + ctx.fallback_to_default_route = fallback_to_default_route; /* * If the nexthop_group is empty, create the next hop group key @@ -975,6 +1068,8 @@ void RouteOrch::doTask(Consumer& consumer) // Go through the bulker results auto it_prev = consumer.m_toSync.begin(); m_bulkNhgReducedRefCnt.clear(); + NextHopGroupKey v4_default_nhg_key; + NextHopGroupKey v6_default_nhg_key; while (it_prev != it) { KeyOpFieldsValuesTuple t = it_prev->second; @@ -1032,6 +1127,18 @@ void RouteOrch::doTask(Consumer& consumer) it_prev = consumer.m_toSync.erase(it_prev); else it_prev++; + + if (ip_prefix.isDefaultRoute() && vrf_id == gVirtualRouterId) + { + if (ip_prefix.isV4()) + { + v4_default_nhg_key = getSyncdRouteNhgKey(gVirtualRouterId, ip_prefix); + } + else + { + v6_default_nhg_key = getSyncdRouteNhgKey(gVirtualRouterId, ip_prefix); + } + } } } else if (op == DEL_COMMAND) @@ -1067,9 +1174,23 @@ void RouteOrch::doTask(Consumer& consumer) } else if (m_syncdNextHopGroups[it_nhg.first].ref_count == 0) { - removeNextHopGroup(it_nhg.first); + // Pass the flag to indicate if the NextHop Group as Default Route NH Members as swapped. + removeNextHopGroup(it_nhg.first, m_syncdNextHopGroups[it_nhg.first].is_default_route_nh_swap); } } + + if (!(v4_default_nhg_key.getSize()) && !(v6_default_nhg_key.getSize())) + { + return; + } + if (v4_default_nhg_key.getSize()) + { + updateDefaultRouteSwapSet(v4_default_nhg_key, v4_active_default_route_nhops); + } + if (v6_default_nhg_key.getSize()) + { + updateDefaultRouteSwapSet(v6_default_nhg_key, v6_active_default_route_nhops); + } } } @@ -1364,6 +1485,7 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) NextHopGroupEntry next_hop_group_entry; next_hop_group_entry.next_hop_group_id = next_hop_group_id; + next_hop_group_entry.nh_member_install_count = 0; size_t npid_count = next_hop_ids.size(); vector nhgm_ids(npid_count); @@ -1434,6 +1556,7 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) { next_hop_group_entry.nhopgroup_members[nhopgroup_members_set.find(nhid)->second].next_hop_id = nhgm_id; next_hop_group_entry.nhopgroup_members[nhopgroup_members_set.find(nhid)->second].seq_id = ((uint32_t)i) + 1; + next_hop_group_entry.nh_member_install_count++; } } @@ -1451,7 +1574,7 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) return true; } -bool RouteOrch::removeNextHopGroup(const NextHopGroupKey &nexthops) +bool RouteOrch::removeNextHopGroup(const NextHopGroupKey &nexthops, const bool is_default_route_nh_swap) { SWSS_LOG_ENTER(); @@ -1472,10 +1595,10 @@ bool RouteOrch::removeNextHopGroup(const NextHopGroupKey &nexthops) SWSS_LOG_NOTICE("Delete next hop group %s", nexthops.to_string().c_str()); vector next_hop_ids; - auto& nhgm = next_hop_group_entry->second.nhopgroup_members; + auto& nhgm = is_default_route_nh_swap ? next_hop_group_entry->second.default_route_nhopgroup_members : next_hop_group_entry->second.nhopgroup_members; for (auto nhop = nhgm.begin(); nhop != nhgm.end();) { - if (m_neighOrch->isNextHopFlagSet(nhop->first, NHFLAGS_IFDOWN)) + if (m_neighOrch->isNextHopFlagSet(nhop->first, NHFLAGS_IFDOWN) && (!is_default_route_nh_swap)) { SWSS_LOG_WARN("NHFLAGS_IFDOWN set for next hop group member %s with next_hop_id %" PRIx64, nhop->first.to_string().c_str(), nhop->second.next_hop_id); @@ -1566,6 +1689,16 @@ bool RouteOrch::removeNextHopGroup(const NextHopGroupKey &nexthops) } } + // Decrement Nexthop Reference Count for Default Route NH Member used as swapped + if (is_default_route_nh_swap) + { + auto& nhgm = next_hop_group_entry->second.default_route_nhopgroup_members; + for (auto nhop = nhgm.begin(); nhop != nhgm.end(); ++nhop) + { + m_neighOrch->decreaseNextHopRefCount(nhop->first); + } + } + m_syncdNextHopGroups.erase(nexthops); return true; @@ -1718,6 +1851,11 @@ void RouteOrch::addTempRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextH (*it).to_string().c_str(), ipPrefix.to_string().c_str()); it = next_hop_set.erase(it); } + else if(m_neighOrch->isNextHopFlagSet(*it, NHFLAGS_IFDOWN)) + { + SWSS_LOG_INFO("Interface down for NH %s, skip this NH", (*it).to_string().c_str()); + it = next_hop_set.erase(it); + } else it++; } @@ -1951,6 +2089,11 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) /* Return false since the original route is not successfully added */ return false; } + else + { + m_syncdNextHopGroups[nextHops].eligible_for_default_route_nh_swap = ctx.fallback_to_default_route; + m_syncdNextHopGroups[nextHops].is_default_route_nh_swap = false; + } } next_hop_id = m_syncdNextHopGroups[nextHops].next_hop_group_id; @@ -2506,6 +2649,15 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) updateDefRouteState(ipPrefix.to_string()); SWSS_LOG_INFO("Set route %s next hop ID to NULL", ipPrefix.to_string().c_str()); + + if (ipPrefix.isV4()) + { + v4_active_default_route_nhops.clear(); + } + else + { + v6_active_default_route_nhops.clear(); + } } else { diff --git a/orchagent/routeorch.h b/orchagent/routeorch.h index 595af46081..e745b2319f 100644 --- a/orchagent/routeorch.h +++ b/orchagent/routeorch.h @@ -40,6 +40,10 @@ struct NextHopGroupEntry sai_object_id_t next_hop_group_id; // next hop group id int ref_count; // reference count NextHopGroupMembers nhopgroup_members; // ids of members indexed by + NextHopGroupMembers default_route_nhopgroup_members; // ids of members indexed by + uint32_t nh_member_install_count; + bool eligible_for_default_route_nh_swap; + bool is_default_route_nh_swap; }; struct NextHopUpdate @@ -122,13 +126,15 @@ struct RouteBulkContext bool excp_intfs_flag; // using_temp_nhg will track if the NhgOrch's owned NHG is temporary or not bool using_temp_nhg; + bool fallback_to_default_route; std::string key; // Key in database table std::string protocol; // Protocol string bool is_set; // True if set operation RouteBulkContext(const std::string& key, bool is_set) - : key(key), excp_intfs_flag(false), using_temp_nhg(false), is_set(is_set) + : key(key), excp_intfs_flag(false), using_temp_nhg(false), is_set(is_set), + fallback_to_default_route(false) { } @@ -146,6 +152,7 @@ struct RouteBulkContext using_temp_nhg = false; key.clear(); protocol.clear(); + fallback_to_default_route = false; } }; @@ -197,12 +204,13 @@ class RouteOrch : public Orch, public Subject bool isRefCounterZero(const NextHopGroupKey&) const; bool addNextHopGroup(const NextHopGroupKey&); - bool removeNextHopGroup(const NextHopGroupKey&); + bool removeNextHopGroup(const NextHopGroupKey&, const bool is_default_route_nh_swap=false); void addNextHopRoute(const NextHopKey&, const RouteKey&); void removeNextHopRoute(const NextHopKey&, const RouteKey&); bool updateNextHopRoutes(const NextHopKey&, uint32_t&); bool getRoutesForNexthop(std::set&, const NextHopKey&); + bool swapnexthopinNextHopGroup(sai_object_id_t next_hop_group_id, sai_object_id_t default_next_hop_id); bool validnexthopinNextHopGroup(const NextHopKey&, uint32_t&); bool invalidnexthopinNextHopGroup(const NextHopKey&, uint32_t&); @@ -242,6 +250,8 @@ class RouteOrch : public Orch, public Subject unsigned int m_maxNextHopGroupCount; bool m_resync; + std::set v4_active_default_route_nhops; + std::set v6_active_default_route_nhops; shared_ptr m_stateDb; unique_ptr m_stateDefaultRouteTb; @@ -288,6 +298,8 @@ class RouteOrch : public Orch, public Subject bool isVipRoute(const IpPrefix &ipPrefix, const NextHopGroupKey &nextHops); void createVipRouteSubnetDecapTerm(const IpPrefix &ipPrefix); void removeVipRouteSubnetDecapTerm(const IpPrefix &ipPrefix); + bool addDefaultRouteNexthopsInNextHopGroup(NextHopGroupEntry& original_next_hop_group, std::set& default_route_next_hop_set); + void updateDefaultRouteSwapSet(const NextHopGroupKey default_nhg_key, std::set& active_default_route_nhops); }; #endif /* SWSS_ROUTEORCH_H */ diff --git a/tests/conftest.py b/tests/conftest.py index abf9955cd7..29d8191fa6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -742,13 +742,19 @@ def start_fpmsyncd(self): self.runcmd(['sh', '-c', 'supervisorctl start fpmsyncd']) # Let's give fpmsyncd a chance to connect to Zebra. - time.sleep(5) + time.sleep(10) # deps: warm_reboot def stop_fpmsyncd(self): self.runcmd(['sh', '-c', 'pkill -x fpmsyncd']) time.sleep(1) + def disable_fpmsyncd(self): + self.runcmd(['sh', '-c', 'supervisorctl stop fpmsyncd']) + + # Let's give fpmsyncd a chance to connect to Zebra. + time.sleep(5) + # deps: warm_reboot def SubscribeAppDbObject(self, objpfx): r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.APPL_DB, diff --git a/tests/test_nhg.py b/tests/test_nhg.py index b9e125f760..2b4b70a050 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -125,22 +125,38 @@ def port_name(self, i): def port_ip(self, i): return "10.0.0." + str(i * 2) + def port_ipv6(self, i): + return "fc00::" + str(hex((i * 2)))[2:] + def port_ipprefix(self, i): return self.port_ip(i) + "/31" + def port_ipv6prefix(self, i): + return self.port_ipv6(i) + "/126" + def peer_ip(self, i): return "10.0.0." + str(i * 2 + 1) + def peer_ipv6(self, i): + return "fc00::" + str(hex((i * 2 + 1)))[2:] + def port_mac(self, i): return "00:00:00:00:00:0" + str(i + 1) - def config_intf(self, i): + def config_intf(self, i, is_ipv6_needed=False): fvs = {'NULL': 'NULL'} self.config_db.create_entry("INTERFACE", self.port_name(i), fvs) self.config_db.create_entry("INTERFACE", "{}|{}".format(self.port_name(i), self.port_ipprefix(i)), fvs) + if is_ipv6_needed: + self.config_db.create_entry("INTERFACE", "{}|{}".format(self.port_name(i), self.port_ipv6prefix(i)), fvs) + self.dvs.port_admin_set(self.port_name(i), "up") self.dvs.runcmd("arp -s {} {}".format(self.peer_ip(i), self.port_mac(i))) + if is_ipv6_needed: + command = "ip -6 neighbor replace {} lladdr {} dev {}".format(self.peer_ipv6(i), + self.port_mac(i), self.port_name(i)) + self.dvs.runcmd(command) assert self.dvs.servers[i].runcmd("ip link set down dev eth0") == 0 assert self.dvs.servers[i].runcmd("ip link set up dev eth0") == 0 @@ -188,7 +204,7 @@ def update_bfd_session_state(self, dvs, session, state): ntf.send("bfd_session_state_change", ntf_data, fvp) # BFD utilities for static route BFD and ecmp acceleration -- end - def init_test(self, dvs, num_intfs): + def init_test(self, dvs, num_intfs, is_ipv6_needed=False): self.dvs = dvs self.app_db = self.dvs.get_app_db() self.asic_db = self.dvs.get_asic_db() @@ -205,7 +221,7 @@ def init_test(self, dvs, num_intfs): self.dvs.setReadOnlyAttr('SAI_OBJECT_TYPE_SWITCH', 'SAI_SWITCH_ATTR_MAX_NUMBER_OF_FORWARDING_CLASSES', '63') for i in range(num_intfs): - self.config_intf(i) + self.config_intf(i, is_ipv6_needed) self.asic_nhgs_count = len(self.asic_db.get_keys(self.ASIC_NHG_STR)) self.asic_nhgms_count = len(self.asic_db.get_keys(self.ASIC_NHGM_STR)) @@ -1825,6 +1841,232 @@ def test_nhgorch_label_route(self, dvs, testlog): self.nhg_ps._del("group1") self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) + def test_route_fallback_to_default_bothv4v6(self, dvs, dvs_route, testlog): + self.init_test(dvs, 6, True) + rtprefix_v6 = "2603:10b0::1/120" + defaultprefix_v6 = "::/0" + nexthop_str_v6 = "fc00::1,fc00::3,fc00::5" + default_nexthop_str_v6 = "fc00::7,fc00::9,fc00::b" + rtprefix = "3.3.3.0/24" + defaultprefix = "0.0.0.0/0" + nexthop_str = "10.0.0.1,10.0.0.3,10.0.0.5" + default_nexthop_str = "10.0.0.7,10.0.0.9,10.0.0.11" + + dvs_route.check_asicdb_deleted_route_entries([rtprefix, rtprefix_v6]) + + try: + dvs.disable_fpmsyncd() + # Program Regular Rouute with fallback to default + fvs = swsscommon.FieldValuePairs([("nexthop",nexthop_str), + ("ifname", "Ethernet0,Ethernet4,Ethernet8"), + ("fallback_to_default_route", "true")]) + self.rt_ps.set(rtprefix, fvs) + time.sleep(1) + + fvs = swsscommon.FieldValuePairs([("nexthop",nexthop_str_v6), + ("ifname", "Ethernet0,Ethernet4,Ethernet8"), + ("fallback_to_default_route", "true")]) + self.rt_ps.set(rtprefix_v6, fvs) + time.sleep(1) + + # Program default route + fvs = swsscommon.FieldValuePairs([("nexthop", default_nexthop_str), + ("ifname", "Ethernet12,Ethernet16,Ethernet20")]) + + self.rt_ps.set(defaultprefix, fvs) + time.sleep(1) + + fvs = swsscommon.FieldValuePairs([("nexthop", default_nexthop_str_v6), + ("ifname", "Ethernet12,Ethernet16,Ethernet20")]) + + self.rt_ps.set(defaultprefix_v6, fvs) + time.sleep(1) + + # check if route was propagated to ASIC DB + rtkeys = dvs_route.check_asicdb_route_entries([rtprefix, rtprefix_v6]) + + # check if default route was propagated to ASIC DB + defaultrtkeys = dvs_route.check_asicdb_route_entries([defaultprefix, defaultprefix_v6]) + + default_nhgid = [] + default_nhops = set() + default_nhgmids = set() + default_nhopsids = set() + default_parentnhgid = set() + + rt_nhgid = [] + rt_nhops = set() + rt_nhopsids = set() + rt_nhgmids = set() + rt_parentnhgid = set() + + # assert the route points to next hop group + flat_list = [] + for x in rtkeys: + flat_list.append(x) + for x in defaultrtkeys: + flat_list.append(x) + for idx, rtkey in enumerate(flat_list): + fvs = self.asic_db.get_entry(self.ASIC_RT_STR, rtkey) + if idx in [0,1]: + rt_nhgid.append(fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"]) + else: + default_nhgid.append(fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"]) + + nhgid = fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] + + fvs = self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid) + + assert bool(fvs) + + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) + + assert len(keys) == 12 + + for k in keys: + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) + nhid = fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID"] + nh_fvs = self.asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", nhid) + + if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] in rt_nhgid: + rt_nhopsids.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID"]) + rt_nhops.add(nh_fvs["SAI_NEXT_HOP_ATTR_IP"]) + rt_nhgmids.add(k) + rt_parentnhgid.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"]) + elif fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] in default_nhgid: + default_nhopsids.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID"]) + default_nhops.add(nh_fvs["SAI_NEXT_HOP_ATTR_IP"]) + default_nhgmids.add(k) + default_parentnhgid.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"]) + + assert len(rt_parentnhgid) == 2 + assert len(rt_nhopsids) == 6 + assert len(rt_nhops) == 6 + assert len(rt_nhgmids) == 6 + + assert len(default_parentnhgid) == 2 + assert len(default_nhopsids) == 6 + assert len(default_nhops) == 6 + assert len(default_nhgmids) == 6 + + assert rt_nhops != default_nhops + assert rt_nhopsids != default_nhopsids + assert rt_parentnhgid != default_parentnhgid + assert rt_nhgmids != default_nhgmids + + # bring links down one-by-one + for i in [0, 1, 2]: + self.flap_intf(i, 'down') + time.sleep(1) + + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) + + if i != 2: + assert len(keys) == 10 - (i * 2) + else: + # Last Link down so we will fallback to default eoute 3 members + assert len(keys) == 12 + rt_nhops.clear() + rt_nhgmids.clear() + rt_nhopsids.clear() + for k in keys: + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) + nhid = fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID"] + nh_fvs = self.asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", nhid) + + if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] in rt_nhgid: + rt_nhopsids.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID"]) + rt_nhops.add(nh_fvs["SAI_NEXT_HOP_ATTR_IP"]) + rt_nhgmids.add(k) + rt_parentnhgid.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"]) + elif fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] in default_nhgid: + default_nhopsids.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID"]) + default_nhops.add(nh_fvs["SAI_NEXT_HOP_ATTR_IP"]) + default_nhgmids.add(k) + default_parentnhgid.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"]) + + assert len(rt_parentnhgid) == 2 + assert len(rt_nhopsids) == 6 + assert len(rt_nhops) == 6 + assert len(rt_nhgmids) == 6 + + assert len(default_parentnhgid) == 2 + assert len(default_nhopsids) == 6 + assert len(default_nhops) == 6 + assert len(default_nhgmids) == 6 + + assert rt_nhops == default_nhops + assert rt_nhopsids == default_nhopsids + assert rt_nhgmids != default_nhgmids + + # bring links up one-by-one + # Bring link up in random order to verify sequence id is as per order + for i, val in enumerate([2,1,0]): + self.flap_intf(i, 'up') + time.sleep(1) + + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) + + assert len(keys) == 12 + + for k in keys: + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, k) + nhid = fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID"] + nh_fvs = self.asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", nhid) + + if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] in rt_nhgid: + rt_nhopsids.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID"]) + rt_nhops.add(nh_fvs["SAI_NEXT_HOP_ATTR_IP"]) + rt_nhgmids.add(k) + rt_parentnhgid.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"]) + elif fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] in default_nhgid: + default_nhopsids.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID"]) + default_nhops.add(nh_fvs["SAI_NEXT_HOP_ATTR_IP"]) + default_nhgmids.add(k) + default_parentnhgid.add(fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"]) + + assert len(rt_parentnhgid) == 2 + assert len(rt_nhopsids) == 6 + assert len(rt_nhops) == 6 + assert len(rt_nhgmids) == 6 + + assert len(default_parentnhgid) == 2 + assert len(default_nhopsids) == 6 + assert len(default_nhops) == 6 + assert len(default_nhgmids) == 6 + + assert rt_nhops == default_nhops + assert rt_nhopsids == default_nhopsids + assert rt_nhgmids != default_nhgmids + + # Remove route 2.2.2.0/24 + self.rt_ps._del(rtprefix) + time.sleep(1) + + self.rt_ps._del(rtprefix_v6) + time.sleep(1) + + # Wait for route 2.2.2.0/24 to be removed + dvs_route.check_asicdb_deleted_route_entries([rtprefix, rtprefix_v6]) + + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) + + assert len(keys) == 6 + + # Remove route 0.0.0.0/0 + self.rt_ps._del(defaultprefix) + time.sleep(1) + + self.rt_ps._del(defaultprefix_v6) + time.sleep(1) + + keys = self.asic_db.get_keys(self.ASIC_NHGM_STR) + + assert len(keys) == 0 + + finally: + dvs.start_fpmsyncd() + class TestCbfNextHopGroup(TestNextHopGroupBase): MAX_NHG_MAP_COUNT = 512