From aa63de0bba03fc875e244a452881311e7c3a40fa Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 21 Aug 2024 10:23:50 +0000 Subject: [PATCH] [nvidia] Skip SAI discovery on ports on fast-boot Given that modern systems have lots of ports, performing SAI discovery takes very long time, e.g. 8 sec for 256 port system. This has a big impact of fast-boot downtime and the discovery itself is not required for Nvidia platform fast-boot. Same applies to Nvidia fastfast-boot (aka warm-boot), yet needs to be tested separately. Signed-off-by: Stepan Blyschak --- configure.ac | 1 + saiasiccmp/SaiSwitchAsic.cpp | 3 +- saiasiccmp/SaiSwitchAsic.h | 3 +- syncd/Makefile.am | 4 +++ syncd/SaiSwitch.cpp | 56 +++++++++++++++++++----------------- syncd/SaiSwitch.h | 5 ++-- syncd/SaiSwitchInterface.h | 3 +- syncd/Syncd.cpp | 22 ++++++++++++-- syncd/Syncd.h | 2 ++ 9 files changed, 66 insertions(+), 33 deletions(-) diff --git a/configure.ac b/configure.ac index 43fe79fab..dc569000a 100644 --- a/configure.ac +++ b/configure.ac @@ -17,6 +17,7 @@ AX_ADD_AM_MACRO_STATIC([]) AM_CONDITIONAL(SONIC_ASIC_PLATFORM_BAREFOOT, test x$CONFIGURED_PLATFORM = xbarefoot) AM_CONDITIONAL(SONIC_ASIC_PLATFORM_BROADCOM, test x$CONFIGURED_PLATFORM = xbroadcom) +AM_CONDITIONAL(SONIC_ASIC_PLATFORM_MELLANOX, test x$CONFIGURED_PLATFORM = xmellanox) AC_ARG_ENABLE(debug, [ --enable-debug turn on debugging], diff --git a/saiasiccmp/SaiSwitchAsic.cpp b/saiasiccmp/SaiSwitchAsic.cpp index 8ee0356b6..eef601c44 100644 --- a/saiasiccmp/SaiSwitchAsic.cpp +++ b/saiasiccmp/SaiSwitchAsic.cpp @@ -272,7 +272,8 @@ std::set SaiSwitchAsic::getWarmBootDiscoveredVids() const void SaiSwitchAsic::onPostPortCreate( _In_ sai_object_id_t port_rid, - _In_ sai_object_id_t port_vid) + _In_ sai_object_id_t port_vid, + _In_ bool discoverPortObjects) { SWSS_LOG_ENTER(); diff --git a/saiasiccmp/SaiSwitchAsic.h b/saiasiccmp/SaiSwitchAsic.h index 479001751..346fd2ff6 100644 --- a/saiasiccmp/SaiSwitchAsic.h +++ b/saiasiccmp/SaiSwitchAsic.h @@ -66,7 +66,8 @@ namespace saiasiccmp virtual void onPostPortCreate( _In_ sai_object_id_t port_rid, - _In_ sai_object_id_t port_vid) override; + _In_ sai_object_id_t port_vid, + _In_ bool discoverPortObjects = true) override; virtual void postPortRemove( _In_ sai_object_id_t portRid) override; diff --git a/syncd/Makefile.am b/syncd/Makefile.am index 3c546fb74..3211eeac6 100644 --- a/syncd/Makefile.am +++ b/syncd/Makefile.am @@ -82,6 +82,10 @@ if SONIC_ASIC_PLATFORM_BROADCOM libSyncd_a_CXXFLAGS += -DMDIO_ACCESS_USE_NPU endif +if SONIC_ASIC_PLATFORM_MELLANOX +libSyncd_a_CPPFLAGS += -DSKIP_SAI_PORT_DISCOVERY_ON_FAST_BOOT +endif + libSyncdRequestShutdown_a_SOURCES = \ RequestShutdown.cpp \ RequestShutdownCommandLineOptions.cpp \ diff --git a/syncd/SaiSwitch.cpp b/syncd/SaiSwitch.cpp index fb711db4d..4fd873f16 100644 --- a/syncd/SaiSwitch.cpp +++ b/syncd/SaiSwitch.cpp @@ -948,46 +948,50 @@ void SaiSwitch::redisUpdatePortLaneMap( void SaiSwitch::onPostPortCreate( _In_ sai_object_id_t port_rid, - _In_ sai_object_id_t port_vid) + _In_ sai_object_id_t port_vid, + _In_ bool discoverPortObjects) { SWSS_LOG_ENTER(); - SaiDiscovery sd(m_vendorSai); + if (discoverPortObjects) + { + SaiDiscovery sd(m_vendorSai); - auto discovered = sd.discover(port_rid); + auto discovered = sd.discover(port_rid); - auto defaultOidMap = sd.getDefaultOidMap(); + auto defaultOidMap = sd.getDefaultOidMap(); - // we need to merge default oid maps + // we need to merge default oid maps - for (auto& kvp: defaultOidMap) - { - for (auto& it: kvp.second) + for (auto& kvp: defaultOidMap) { - m_defaultOidMap[kvp.first][it.first] = it.second; + for (auto& it: kvp.second) + { + m_defaultOidMap[kvp.first][it.first] = it.second; + } } - } - SWSS_LOG_NOTICE("discovered %zu new objects (including port) after creating port VID: %s", - discovered.size(), - sai_serialize_object_id(port_vid).c_str()); + SWSS_LOG_NOTICE("discovered %zu new objects (including port) after creating port VID: %s", + discovered.size(), + sai_serialize_object_id(port_vid).c_str()); - m_discovered_rids.insert(discovered.begin(), discovered.end()); + m_discovered_rids.insert(discovered.begin(), discovered.end()); - SWSS_LOG_NOTICE("putting ALL new discovered objects to redis for port %s", - sai_serialize_object_id(port_vid).c_str()); + SWSS_LOG_NOTICE("putting ALL new discovered objects to redis for port %s", + sai_serialize_object_id(port_vid).c_str()); - for (sai_object_id_t rid: discovered) - { - /* - * We also could thing of optimizing this since it's one call to redis - * per rid, and probably this should be ATOMIC. - * - * NOTE: We are also storing read only object's here, like default - * virtual router, CPU, default trap group, etc. - */ + for (sai_object_id_t rid: discovered) + { + /* + * We also could thing of optimizing this since it's one call to redis + * per rid, and probably this should be ATOMIC. + * + * NOTE: We are also storing read only object's here, like default + * virtual router, CPU, default trap group, etc. + */ - redisSetDummyAsicStateForRealObjectId(rid); + redisSetDummyAsicStateForRealObjectId(rid); + } } redisUpdatePortLaneMap(port_rid); diff --git a/syncd/SaiSwitch.h b/syncd/SaiSwitch.h index 523e1fe1d..5d23fe847 100644 --- a/syncd/SaiSwitch.h +++ b/syncd/SaiSwitch.h @@ -184,11 +184,12 @@ namespace syncd * * Performs actions needed after port creation. Will discover new * queues, ipgs and scheduler groups that belong to new created port, - * and updated ASIC DB accordingly. + * and updated ASIC DB accordingly when discoverPortObjects is true. */ virtual void onPostPortCreate( _In_ sai_object_id_t port_rid, - _In_ sai_object_id_t port_vid) override; + _In_ sai_object_id_t port_vid, + _In_ bool discoverPortObjects = true) override; /** * @brief Post port remove. diff --git a/syncd/SaiSwitchInterface.h b/syncd/SaiSwitchInterface.h index a369f5e64..8b764d25a 100644 --- a/syncd/SaiSwitchInterface.h +++ b/syncd/SaiSwitchInterface.h @@ -89,7 +89,8 @@ namespace syncd virtual void onPostPortCreate( _In_ sai_object_id_t port_rid, - _In_ sai_object_id_t port_vid) = 0; + _In_ sai_object_id_t port_vid, + _In_ bool discoverPortObjects = true) = 0; virtual void postPortRemove( _In_ sai_object_id_t portRid) = 0; diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 9e2cd1971..7c5d5a9ad 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -2001,7 +2001,7 @@ sai_status_t Syncd::processBulkOidCreate( if (objectType == SAI_OBJECT_TYPE_PORT) { - m_switches.at(switchVid)->onPostPortCreate(objectRids[idx], objectVids[idx]); + m_switches.at(switchVid)->onPostPortCreate(objectRids[idx], objectVids[idx], shouldDiscoverPortObjects()); } } } @@ -3130,7 +3130,7 @@ sai_status_t Syncd::processOidCreate( if (objectType == SAI_OBJECT_TYPE_PORT) { - m_switches.at(switchVid)->onPostPortCreate(objectRid, objectVid); + m_switches.at(switchVid)->onPostPortCreate(objectRid, objectVid, shouldDiscoverPortObjects()); } } @@ -5296,3 +5296,21 @@ syncd_restart_type_t Syncd::handleRestartQuery( return RequestShutdownCommandLineOptions::stringToRestartType(op); } + +bool Syncd::shouldDiscoverPortObjects() const +{ +#ifdef SKIP_SAI_PORT_DISCOVERY_ON_FAST_BOOT + const bool discoverPortObjectsInFastBoot = false; +#else + const bool discoverPortObjectsInFastBoot = true; +#endif + + // Comparing with m_veryFirstRun, so that we only skip discovery when switch is fast booting + // and not after it finished fast boot (e.g. port breakout after fast-reboot). + if ((m_commandLineOptions->m_startType == SAI_START_TYPE_FAST_BOOT) && m_veryFirstRun) + { + return discoverPortObjectsInFastBoot; + } + + return true; +} diff --git a/syncd/Syncd.h b/syncd/Syncd.h index 06eb84a62..b01253400 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -334,6 +334,8 @@ namespace syncd syncd_restart_type_t handleRestartQuery( _In_ swss::NotificationConsumer &restartQuery); + bool shouldDiscoverPortObjects() const; + private: /**